diff --git a/src/graphics/host_gpu/renderer/cache/samplerCache.cpp b/src/graphics/host_gpu/renderer/cache/samplerCache.cpp index 8eeddbc47..259ec463e 100644 --- a/src/graphics/host_gpu/renderer/cache/samplerCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/samplerCache.cpp @@ -14,10 +14,10 @@ SamplerCache::~SamplerCache() { } } -vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) { +vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r, bool integer_border) { Common::LockGuard lock(m_mutex); - const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3]}; + const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3], integer_border}; if (auto iter = m_samplers.find(key); iter != m_samplers.end()) { return iter->second; } @@ -93,23 +93,23 @@ vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) { return vk::SamplerAddressMode::eClampToBorder; }; - vk::BorderColor border = vk::BorderColor::eIntTransparentBlack; + vk::BorderColor border = integer_border ? vk::BorderColor::eIntTransparentBlack + : vk::BorderColor::eFloatTransparentBlack; switch (static_cast(r.BorderColorType())) { - case Prospero::SamplerBorderColor::kTransBlack: - border = vk::BorderColor::eIntTransparentBlack; - break; + case Prospero::SamplerBorderColor::kTransBlack: break; case Prospero::SamplerBorderColor::kOpaqueBlack: - border = vk::BorderColor::eIntOpaqueBlack; + border = integer_border ? vk::BorderColor::eIntOpaqueBlack + : vk::BorderColor::eFloatOpaqueBlack; break; case Prospero::SamplerBorderColor::kOpaqueWhite: - border = vk::BorderColor::eIntOpaqueWhite; + border = integer_border ? vk::BorderColor::eIntOpaqueWhite + : vk::BorderColor::eFloatOpaqueWhite; break; case Prospero::SamplerBorderColor::kFromTable: LOGF( "temporary: approximating table border color as transparent black, index = %" PRIu16 "\n", r.BorderColorPtr()); - border = vk::BorderColor::eIntTransparentBlack; break; default: EXIT("unknown border color: %d", static_cast(r.BorderColorType())); } diff --git a/src/graphics/host_gpu/renderer/cache/samplerCache.h b/src/graphics/host_gpu/renderer/cache/samplerCache.h index 4121c29fb..9c8199136 100644 --- a/src/graphics/host_gpu/renderer/cache/samplerCache.h +++ b/src/graphics/host_gpu/renderer/cache/samplerCache.h @@ -24,10 +24,10 @@ public: ~SamplerCache(); KYTY_CLASS_NO_COPY(SamplerCache); - vk::Sampler GetSampler(const ShaderSamplerResource& r); + vk::Sampler GetSampler(const ShaderSamplerResource& r, bool integer_border); private: - using SamplerKey = std::array; + using SamplerKey = std::array; struct SamplerKeyHash { std::size_t operator()(const SamplerKey& key) const { diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index d986ddf19..cf13cbb9b 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -738,14 +738,15 @@ static vk::Sampler NativeSampler(RenderContext& context, const ShaderRecompiler::IR::CompiledShaderInfo& program, uint32_t index, const ShaderRecompiler::IR::DescriptorValue& value) { - auto descriptor = DecodeNativeDescriptor(value); - if (!program.info.samplers[index].depth_compare) { + auto descriptor = DecodeNativeDescriptor(value); + const auto& sampler = program.info.samplers[index]; + if (!sampler.depth_compare) { descriptor.fields[0] &= ~(0x7u << 12u); } - if (program.info.samplers[index].force_point_filtering) { + if (sampler.force_point_filtering) { descriptor.SetPointFiltering(); } - return context.GetSamplerCache().GetSampler(descriptor); + return context.GetSamplerCache().GetSampler(descriptor, sampler.integer_border); } static vk::DescriptorBufferInfo NativeUpload(RenderContext& context, diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index 19bcf051a..6d4179b97 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -116,33 +115,22 @@ void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash, } void DumpShaderOriginal(const char* stage_name, uint64_t shader_hash, - std::span code, const std::string& decoded_dump) { + std::span code) { if (!Config::GraphicsDebugDumpEnabled()) { return; } EXIT_IF(code.empty()); static std::atomic_int id = 0; - const auto base = Config::GetShaderLogFolder() / "original" / - fmt::format("{:04d}_new_shader_{}_{:016x}", id++, stage_name, shader_hash); - Common::File::CreateDirectories(base.parent_path()); - for (const auto& [suffix, data, size]: { - std::tuple {".bin", static_cast(code.data()), code.size_bytes()}, - std::tuple {".rdna2", static_cast(decoded_dump.data()), - decoded_dump.size()}, - }) { - if (size == 0) { - continue; - } - auto path = base; - path += suffix; - Common::File file(path); - if (file.IsInvalid()) { - const auto path_text = Common::PathToString(path); - LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str()); - } else { - file.Write(data, size); - } + const auto path = Config::GetShaderLogFolder() / "original" / + fmt::format("{:04d}_new_shader_{}_{:016x}.bin", id++, stage_name, shader_hash); + Common::File::CreateDirectories(path.parent_path()); + Common::File file(path); + if (file.IsInvalid()) { + const auto path_text = Common::PathToString(path); + LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str()); + return; } + file.Write(code.data(), code.size_bytes()); } bool ValidateShaderSpirv(const char* label, uint64_t shader_hash, @@ -225,25 +213,13 @@ struct PipelineCache::ProgramCache { static constexpr std::size_t MaxStaticKeyWords = 13 + ShaderVertexInputInfo::RES_MAX * 13; - Permutation CompilePermutation(const ShaderParams& params, + Permutation CompilePermutation(const char* stage_name, const ShaderRecompiler::CompileOptions& options, ShaderRecompiler::TranslateResult translated, ShaderRecompiler::IR::ResourceSpecialization specialization, uint32_t push_data_start_dword) { - const char* stage_name = nullptr; - switch (options.stage) { - case ShaderType::Vertex: stage_name = "vs"; break; - case ShaderType::Mesh: stage_name = "ms"; break; - case ShaderType::Local: stage_name = "ls"; break; - case ShaderType::TessellationControl: stage_name = "hs"; break; - case ShaderType::TessellationEvaluation: stage_name = "ds"; break; - case ShaderType::Pixel: stage_name = "ps"; break; - case ShaderType::Compute: stage_name = "cs"; break; - default: EXIT("invalid pipeline shader stage\n"); - } auto result = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization, push_data_start_dword); - DumpShaderOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump); if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv)) { DumpShaderSpirv(stage_name, options.shader_hash, result.spirv); EXIT("%s failed hash=0x%016" PRIx64 ": SPIR-V validation failed\n", options.dump_label, @@ -321,14 +297,15 @@ struct PipelineCache::ProgramCache { stage_input.compute = &input_info; } const char* label = nullptr; + const char* stage_name = nullptr; switch (stage) { - case ShaderType::Vertex: label = "ShaderRecompiler VS"; break; - case ShaderType::Mesh: label = "ShaderRecompiler MS"; break; - case ShaderType::Local: label = "ShaderRecompiler LS"; break; - case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; break; - case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; break; - case ShaderType::Pixel: label = "ShaderRecompiler PS"; break; - case ShaderType::Compute: label = "ShaderRecompiler CS"; break; + case ShaderType::Vertex: label = "ShaderRecompiler VS"; stage_name = "vs"; break; + case ShaderType::Mesh: label = "ShaderRecompiler MS"; stage_name = "ms"; break; + case ShaderType::Local: label = "ShaderRecompiler LS"; stage_name = "ls"; break; + case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; stage_name = "hs"; break; + case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; stage_name = "ds"; break; + case ShaderType::Pixel: label = "ShaderRecompiler PS"; stage_name = "ps"; break; + case ShaderType::Compute: label = "ShaderRecompiler CS"; stage_name = "cs"; break; default: EXIT("invalid pipeline shader stage\n"); } ShaderRecompiler::CompileOptions options; @@ -351,6 +328,7 @@ struct PipelineCache::ProgramCache { } else { options.wave_size = input_info.wave_size; } + DumpShaderOriginal(stage_name, options.shader_hash, params.code); auto translated = ShaderRecompiler::TranslateProgram(params.code, options); if (translated.skip_dispatch) { entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first; @@ -365,7 +343,7 @@ struct PipelineCache::ProgramCache { entry->second.specialization)); } entry->second.permutations.push_back(CompilePermutation( - params, options, std::move(translated), entry->second.specialization, push_data_cursor)); + stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor)); const auto& permutation = entry->second.permutations.back(); input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources}; permutation.program.bindings.AdvancePushData(push_data_cursor); diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index 879d315af..627e63395 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -575,12 +575,17 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo return state; } -static bool DrawHasActivePixelShader(const CommandBuffer& buffer) { - const auto& ctx = buffer.GetRegisters(); - const auto& sh_regs = ctx.GetShaderRegisters(); - const bool has_color_output = (ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask) != 0; - return buffer.GetShaders().GetPs().ps_regs.data_addr != 0 && - (has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs)); +static uint32_t DrawColorOutputMask(const HW::Context& ctx) { + const auto& sh_regs = ctx.GetShaderRegisters(); + const auto write_mask = ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask; + uint32_t output_mask = 0; + for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) { + if (sh_regs.target_output_mode[slot] != 0 && + render_target_mask_slot(write_mask, slot) != 0) { + output_mask |= 1u << slot; + } + } + return output_mask; } enum class CbColorMode : uint8_t { @@ -847,7 +852,7 @@ static bool ResolvePrimitiveRestart(const CommandBuffer& buffer, } static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, - DrawRenderState& state) { + uint32_t color_output_mask, DrawRenderState& state) { auto& ctx = buffer.GetRegisters(); auto& sh_ctx = buffer.GetShaders(); @@ -861,7 +866,7 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, target_export_mapping {}; for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) { const auto& rt = ctx.GetRenderTarget(slot); - if (rt.base.addr != 0 && render_target_mask_slot(ctx.GetRenderTargetMask(), slot) != 0) { + if ((color_output_mask & (1u << slot)) != 0 && rt.base.addr != 0) { target_export_mapping[slot] = TextureGetRenderTargetFormat(rt.info.format, rt.info.channel_type, rt.info.channel_order) @@ -880,8 +885,12 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCallInfo& draw, uint32_t render_target_slice_offset, DrawRenderState& state) { - state.ps_active = DrawHasActivePixelShader(buffer); - RefreshShaders(buffer, draw, state); + const auto& shader_regs = buffer.GetRegisters().GetShaderRegisters(); + const auto color_output_mask = DrawColorOutputMask(buffer.GetRegisters()); + state.ps_active = buffer.GetShaders().GetPs().ps_regs.data_addr != 0 && + (color_output_mask != 0 || + PixelShaderHasDepthOrCoverageSideEffects(shader_regs)); + RefreshShaders(buffer, draw, color_output_mask, state); uint32_t mrt_mask = 0; if (state.ps_active) { for (const auto& output: state.ps_input_info.stage.program->info.outputs) { @@ -890,6 +899,7 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal } } } + mrt_mask &= color_output_mask; if (draw.IsIndexed()) { LogDrawPhase(draw.Name(), "ResolveRenderColorTarget"); } diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 261786df6..b930791f1 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -15,7 +15,6 @@ #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" -#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include "graphics/shader/recompiler/ir/passes/SsaRewrite.h" #include @@ -548,30 +547,40 @@ TranslateResult TranslateProgram(std::span code, const CompileOp LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash); - auto cfg = CFG::BuildGraph(decoded); + auto native_cfg = CFG::BuildGraph(decoded); + CFG::Graph structured_cfg; + auto* selected_cfg = &native_cfg; LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64 " loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash, - static_cast(cfg.blocks.size()), static_cast(cfg.natural_loops.size()), - static_cast(cfg.back_edges.size()), phase_ms()); - if (cfg.irreducible) { - LogDispatcherFallback(options, cfg, "build"); + static_cast(native_cfg.blocks.size()), + static_cast(native_cfg.natural_loops.size()), + static_cast(native_cfg.back_edges.size()), phase_ms()); + if (native_cfg.irreducible) { + LogDispatcherFallback(options, native_cfg, "build"); } else { LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash); - if (!CFG::Structurize(cfg)) { - LogDispatcherFallback(options, cfg, "structurize"); + structured_cfg = CFG::Structurize(native_cfg); + if (structured_cfg.unsupported) { + native_cfg.unsupported = true; + native_cfg.failure_kind = structured_cfg.failure_kind; + native_cfg.failure_block = structured_cfg.failure_block; + native_cfg.unsupported_reason = structured_cfg.unsupported_reason; + LogDispatcherFallback(options, native_cfg, "structurize"); } else { + selected_cfg = &structured_cfg; LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options), - static_cast(cfg.blocks.size())); + static_cast(selected_cfg->blocks.size())); } LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64 " loops=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash, - static_cast(cfg.blocks.size()), - static_cast(cfg.natural_loops.size()), phase_ms()); + static_cast(selected_cfg->blocks.size()), + static_cast(selected_cfg->natural_loops.size()), phase_ms()); } + const auto& cfg = *selected_cfg; Frontend::EmbeddedFetchPlan embedded_fetch; if ((options.stage == ShaderType::Vertex || options.stage == ShaderType::Local) && options.input_info.vertex != nullptr && options.input_info.vertex->fetch_embedded) { @@ -614,15 +623,21 @@ TranslateResult TranslateProgram(std::span code, const CompileOp IR::EliminateDeadCode(ir.blocks); } LowerTessellationMemory(ir, options); - IR::BuildSrtPlan(ir); - IR::EliminateDeadCode(ir.blocks); - IR::TrackResources(ir); + std::string cfg_dump; + if (options.dump_ir) { + cfg_dump = CFG::GraphToString(cfg); + if (options.early_dump) { + LOGF("%s native IR before resource tracking:\n%s", GetDumpLabel(options), + MakeIrDump(cfg_dump, ir).c_str()); + } + } + IR::TrackResources(ir, decoded, native_cfg); IR::EliminateDeadCode(ir.blocks); TranslateResult result; result.program = std::move(ir); if (options.dump_ir) { result.decoded_dump = std::move(decoded_dump); - result.cfg_dump = CFG::GraphToString(cfg); + result.cfg_dump = std::move(cfg_dump); } return result; } @@ -634,6 +649,35 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o const auto emit_begin = std::chrono::steady_clock::now(); auto& ir = translated.program; IR::ApplyResourceSpecialization(ir, specialization); + // The resource plan owns host descriptor evaluation now. Keep only dependencies consumed + // by GPU memory operations; bound descriptor dwords must not retain shader instructions. + for (auto& inst: ir.value_storage) { + inst.Invalidate(); + } + for (auto* block: ir.blocks) { + for (auto& inst: *block) { + const auto op = inst.GetOpcode(); + uint32_t first = 0; + if (op == IR::ValueOpcode::GetBufferResource) { + if (std::ranges::any_of(inst.Uses(), [&](const IR::Use& use) { + return ir.memory_info[use.user->Flags().index].kind == + IR::ResourceKind::IndirectBuffer; + })) { + continue; + } + } else if (op == IR::ValueOpcode::GetImageResource) { + const auto resource = inst.Flags(); + first = resource < ir.info.images.size() && + ir.info.images[resource].indirect_root == resource ? 1u : 0u; + } else if (op != IR::ValueOpcode::GetSamplerResource) { + continue; + } + for (size_t index = first; index < inst.NumArgs(); index++) { + inst.SetArg(index, IR::Value(0u)); + } + } + } + ir.value_storage.clear(); IR::RemoveIdentities(ir.blocks); IR::EliminateDeadCode(ir.blocks); diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index e7b9b61d7..c45a0c9d0 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -62,12 +62,7 @@ void ValidateNativeProgram(const IR::Program& program) { Expect(Kind::BdaPagetable); Expect(Kind::FaultBuffer); } - const bool uses_flattened_runtime = - !program.srt_reads.empty() || - std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) { - return image.indirect_search_iterations != 0u; - }); - if (uses_flattened_runtime) { + if (IR::UsesFlattenedSrt(program)) { Expect(Kind::FlattenedSrt); } if (program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData()) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp index 273f224b4..1ccb434b1 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp @@ -203,9 +203,7 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) { const auto bits = state.builder.AllocateId(); const auto abs_bits = state.builder.AllocateId(); const auto sign_bits = state.builder.AllocateId(); - const auto non_zero = state.builder.AllocateId(); const auto subnormal = state.builder.AllocateId(); - const auto flush = state.builder.AllocateId(); const auto selected = state.builder.AllocateId(); const auto ret = state.builder.AllocateId(); state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value); @@ -213,12 +211,9 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) { ConstantU32(state, 0x7fffffffu)); state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)); - state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), non_zero, abs_bits, - ConstantU32(state, 0)); state.builder.AddFunction(spv::OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)); - state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal); - state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, flush, sign_bits, bits); + state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, subnormal, sign_bits, bits); state.builder.AddFunction(spv::OpBitcast, TypeF32(state), ret, selected); return ret; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 32652e02a..c49b1d6b1 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -636,6 +636,29 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Arg(inst, 1)); } +uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst) { + if (ctx.other_half == nullptr) return ctx.Arg(inst, 0); + // A native scalar branch makes one decision for both emulated wave halves. + if (ctx.half != 0) return ctx.other_half->Def(IR::Value(&inst)); + const auto kind = inst.Flags(); + if (kind == CFG::BranchCondition::ScalarInstruction) return ctx.Arg(inst, 0); + const auto ballot = ctx.Ballot(inst.Arg(0)); + const auto low = ctx.state.builder.AllocateId(); + const auto high = ctx.state.builder.AllocateId(); + const auto combined = ctx.state.builder.AllocateId(); + const auto result = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0); + ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1); + const bool zero = kind == CFG::BranchCondition::ExecZero || + kind == CFG::BranchCondition::VccZero || kind == CFG::BranchCondition::SccZero; + ctx.state.builder.AddFunction(zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, + TypeU32(ctx.state), combined, low, high); + ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual, + TypeBool(ctx.state), result, combined, + ConstantU32(ctx.state, zero ? ~0u : 0u)); + return result; +} + uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) { return ctx.Ballot(predicate); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index 53dde198b..26d1f9bdc 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -2,6 +2,8 @@ #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" +#include + namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { // Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values // (IR::Value), and instructions whose metadata or operand loads must stay lazy. @@ -112,18 +114,34 @@ EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalNot, OpLogicalNot, U1, uint32_t) -EMIT_NATIVE(FPOrdEqual32, OpFOrdEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordEqual32, OpFUnordEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPOrdNotEqual32, OpFOrdNotEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordNotEqual32, OpFUnordNotEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPOrdLessThan32, OpFOrdLessThan, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordLessThan32, OpFUnordLessThan, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPOrdGreaterThan32, OpFOrdGreaterThan, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordGreaterThan32, OpFUnordGreaterThan, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t) +template +uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) { + // SPIR-V's DenormFlushToZero mode does not require comparison operands to flush. + const bool flush = inst.Flags().flush_input_denorms; + const auto operand = [&](size_t index) { + const auto value = inst.Arg(index); + if (flush && value.IsImmediate()) { + const auto bits = std::bit_cast(value.F32Value()); + return ConstantF32(ctx.state, (bits & 0x7fffffffu) < 0x00800000u + ? bits & 0x80000000u : bits); + } + const auto id = ctx.Arg(inst, index); + return flush ? EmitFlushF32DenormToSignedZero(ctx.state, id) : id; + }; + return EmitNative(ctx.state, operand(0), operand(1)); +} +inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPOrdLessThan32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordLessThan32 = EmitFloatCompare32; +inline constexpr auto EmitFPOrdGreaterThan32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordGreaterThan32 = EmitFloatCompare32; +inline constexpr auto EmitFPOrdLessThanEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordLessThanEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPOrdGreaterThanEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordGreaterThanEqual32 = EmitFloatCompare32; uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0); inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32; EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t) @@ -182,6 +200,7 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& in uint32_t EmitWqmU64(EmitterState& state, uint32_t value); uint32_t EmitLaneId(EmitterState& state); uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate); +uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 736d5514b..3d21e0f4b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -146,7 +146,12 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem ctx.Fail(inst, "has no address base pair"); return ConstantDeviceAddress(state, 0); } - const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1)); + auto base_low = ctx.Arg(*handle, 0); + if (mem.kind == IR::ResourceKind::ScalarAddress) { + base_low = Binary(state, spv::OpBitwiseAnd, TypeU32(state), base_low, + ConstantU32(state, ~3u)); + } + const auto base = DeviceAddressFromWords(state, base_low, ctx.Arg(*handle, 1)); address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base, Unary(state, spv::OpUConvert, TypeScalarU64(state), low)); } @@ -1099,16 +1104,27 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) { if (ctx.half == 1) { return ctx.other_half->Def(IR::Value(const_cast(&inst))); } - const auto m0 = ctx.Arg(inst, 0); - const auto base = - Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16)); - const auto size = - Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu)); - const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base, - ConstantU32(state, ctx.Memory(inst).offset)); + const auto mem = ctx.Memory(inst); + const auto offset = mem.offset & 0xfffcu; + auto address = ConstantU32(state, offset); + uint32_t region_bounds = 0; + if (mem.kind == IR::ResourceKind::Gds) { + const auto m0 = ctx.Arg(inst, 0); + const auto base = + Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16)); + const auto size = + Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu)); + address = Binary(state, spv::OpIAdd, TypeU32(state), base, address); + // The entire M0 region must fit the PS5's 48 KiB GDS partition. + region_bounds = AndCondition( + state, Binary(state, spv::OpULessThan, TypeBool(state), + ConstantU32(state, offset + 3u), size), + Binary(state, spv::OpULessThanEqual, TypeBool(state), + Binary(state, spv::OpIAdd, TypeU32(state), base, size), + ConstantU32(state, 0xc000u))); + } const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); - const auto mem = ctx.Memory(inst); const auto access = PrepareMemoryResourceAccess(state, mem); const auto index = EmitMemoryElementIndex(state, access, raw_index); const auto exec = ctx.Arg(inst, 1); @@ -1127,19 +1143,17 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) { : first; const auto is_first = Binary(state, spv::OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), source_lane); - const auto storage_bounds = EmitMemoryElementInBounds(state, access, index); - const auto m0_bounds = - mem.kind == IR::ResourceKind::Gds - ? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0)) - : Binary(state, spv::OpULessThan, TypeBool(state), - ConstantU32(state, ctx.Memory(inst).offset + 3u), size); + auto bounds = EmitMemoryElementInBounds(state, access, index); + if (mem.kind == IR::ResourceKind::Gds) { + bounds = AndCondition(state, bounds, region_bounds); + } const auto condition = AndCondition( state, is_first, AndCondition(state, state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count, ConstantU32(state, 0)) : exec, - AndCondition(state, storage_bounds, m0_bounds))); + bounds)); const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() { const auto value = state.builder.AllocateId(); state.builder.AddFunction(append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state), @@ -1173,10 +1187,11 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) { if (mem.planning_only) return; auto& state = ctx.state; mem.kind = IR::ResourceKind::ScalarBuffer; - const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1), - ConstantU32(state, mem.offset)); - const auto index = - Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); + auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1), + ConstantU32(state, 2)); + if (mem.offset >= 4u) { + index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u)); + } const auto access = PrepareMemoryResourceAccess(state, mem); const auto element = EmitMemoryElementIndex(state, access, index); const auto condition = EmitMemoryElementInBounds(state, access, element); @@ -1198,6 +1213,8 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) { value = LoadWideBuffer(ctx, inst, buffer_components); else if (shared_components > 1u) value = LoadWideShared(ctx, inst, shared_components); + else if (mem.kind == IR::ResourceKind::ScalarAddress) + value = LoadBdaDword(ctx, GuestAddress(ctx, inst, mem)); else if (address_info.access == IR::AddressAccess::Read && mem.kind != IR::ResourceKind::Scratch) value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1), diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index d8b9df738..6f3b06322 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -97,30 +97,6 @@ void EmitReturn(ValueEmitContext& ctx) { ctx.state.builder.AddFunction(spv::OpReturn); } -uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) { - // Scalar-instruction conditions already test the full wave's raw register values. - if (ctx.other_half == nullptr || - info.terminator.condition == CFG::BranchCondition::ScalarInstruction || - info.terminator.condition == CFG::BranchCondition::GotoVariable) { - return ctx.Def(info.condition); - } - const auto ballot = ctx.Ballot(info.condition); - const auto low = ctx.state.builder.AllocateId(); - const auto high = ctx.state.builder.AllocateId(); - const auto result = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0); - ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1); - const auto kind = info.terminator.condition; - const bool zero = kind == CFG::BranchCondition::ExecZero || - kind == CFG::BranchCondition::VccZero || - kind == CFG::BranchCondition::SccZero; - const auto combined = - EmitBinaryU32(ctx.state, zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, low, high); - ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual, TypeBool(ctx.state), - result, combined, ConstantU32(ctx.state, zero ? ~0u : 0u)); - return result; -} - void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block, const IR::BlockInfo& info) { const auto& program = ctx.state.program; @@ -160,7 +136,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block, EmitReturn(ctx); return; } - const auto condition = BranchCondition(ctx, info); + const auto condition = ctx.Def(info.condition); emit_merge(); ctx.state.builder.AddFunction(spv::OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)); @@ -193,7 +169,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta EmitDispatcherTarget(ctx, dispatcher, block, term.false_block); const auto selected = ctx.state.builder.AllocateId(); ctx.state.builder.AddFunction( - spv::OpSelect, TypeU32(ctx.state), selected, BranchCondition(ctx, info), + spv::OpSelect, TypeU32(ctx.state), selected, ctx.Def(info.condition), ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.false_block)); return selected; } diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp index 9c279baf7..8729678f3 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp @@ -5,10 +5,11 @@ #include #include #include +#include +#include #include #include #include -#include namespace Libs::Graphics::ShaderRecompiler::CFG { namespace { @@ -598,39 +599,6 @@ bool Contains(const std::vector& values, uint32_t value) { return std::find(values.begin(), values.end(), value) != values.end(); } -bool ReplaceValue(std::vector& values, uint32_t old_value, uint32_t new_value) { - bool changed = false; - for (auto& value: values) { - if (value == old_value) { - value = new_value; - changed = true; - } - } - if (changed) { - SortUnique(values); - } - return changed; -} - -bool RemoveValue(std::vector& values, uint32_t value) { - const auto old_size = values.size(); - values.erase(std::remove(values.begin(), values.end(), value), values.end()); - return values.size() != old_size; -} - -bool ReplaceTerminatorTarget(Terminator& terminator, uint32_t old_value, uint32_t new_value) { - bool changed = false; - if (terminator.true_block == old_value) { - terminator.true_block = new_value; - changed = true; - } - if (terminator.false_block == old_value) { - terminator.false_block = new_value; - changed = true; - } - return changed; -} - uint32_t RemapId(uint32_t id, const std::vector& id_map) { return id != UINT32_MAX && id < id_map.size() ? id_map[id] : id; } @@ -700,7 +668,6 @@ void PruneUnreachableBlocks(Graph& graph) { RemapIds(block.successors, id_map); block.predecessors.clear(); block.dominators.clear(); - block.post_dominators.clear(); auto& terminator = block.terminator; terminator.true_block = RemapId(terminator.true_block, id_map); terminator.false_block = RemapId(terminator.false_block, id_map); @@ -753,37 +720,6 @@ void ComputeDominators(Graph& graph) { } } -void ComputePostDominators(Graph& graph) { - const auto count = static_cast(graph.blocks.size()); - const auto all = AllBlockIds(count); - - for (auto& block: graph.blocks) { - block.post_dominators = block.successors.empty() ? std::vector {block.id} : all; - } - - bool changed = true; - while (changed) { - changed = false; - for (auto& block: graph.blocks) { - std::vector next; - if (block.successors.empty()) { - next = {block.id}; - } else { - next = graph.blocks[block.successors.front()].post_dominators; - for (uint32_t i = 1; i < block.successors.size(); i++) { - next = IntersectSorted(next, graph.blocks[block.successors[i]].post_dominators); - } - AddUnique(next, block.id); - SortUnique(next); - } - if (next != block.post_dominators) { - block.post_dominators = std::move(next); - changed = true; - } - } - } -} - void ComputeBackEdges(Graph& graph) { graph.back_edges.clear(); for (const auto& block: graph.blocks) { @@ -841,31 +777,9 @@ void ComputeNaturalLoops(Graph& graph) { edge.natural = natural; NaturalLoop loop; - loop.header = edge.to; - loop.latch = edge.from; - loop.continue_block = edge.from; - loop.body_blocks = std::move(body); - - for (auto block_id: loop.body_blocks) { - const auto* block = graph.FindBlock(block_id); - if (block == nullptr) { - continue; - } - for (auto succ: block->successors) { - if (!Contains(loop.body_blocks, succ)) { - AddUnique(loop.exit_blocks, succ); - } - } - } - SortUnique(loop.exit_blocks); - - if (!loop.exit_blocks.empty()) { - uint32_t merge = loop.exit_blocks.front(); - for (uint32_t i = 1; i < loop.exit_blocks.size(); i++) { - merge = graph.FindNearestCommonPostDominator(merge, loop.exit_blocks[i]); - } - loop.merge = merge; - } + loop.header = edge.to; + loop.latch = edge.from; + loop.body_blocks = std::move(body); graph.natural_loops.push_back(std::move(loop)); } } @@ -965,7 +879,6 @@ void ComputeComponents(Graph& graph) { void RecomputeAnalyses(Graph& graph) { ComputeDominators(graph); - ComputePostDominators(graph); ComputeBackEdges(graph); ComputeNaturalLoops(graph); ComputeComponents(graph); @@ -984,7 +897,6 @@ std::vector ApplyBlockOrder(Graph& graph, std::vector bloc RemapIds(block.predecessors, id_map); RemapIds(block.successors, id_map); RemapIds(block.dominators, id_map); - RemapIds(block.post_dominators, id_map); block.terminator.true_block = RemapId(block.terminator.true_block, id_map); block.terminator.false_block = RemapId(block.terminator.false_block, id_map); block.terminator.merge_block = RemapId(block.terminator.merge_block, id_map); @@ -994,562 +906,6 @@ std::vector ApplyBlockOrder(Graph& graph, std::vector bloc return id_map; } -uint32_t MoveBlockBefore(Graph& graph, uint32_t block_id, uint32_t before_id) { - if (block_id == before_id || block_id >= graph.blocks.size() || - before_id >= graph.blocks.size()) { - return block_id; - } - - const auto block_pos = block_id; - const auto before_pos = before_id; - std::vector old_blocks = std::move(graph.blocks); - std::vector new_blocks; - new_blocks.reserve(old_blocks.size()); - - for (uint32_t i = 0; i < old_blocks.size(); i++) { - if (i == before_pos) { - new_blocks.push_back(std::move(old_blocks[block_pos])); - } - if (i != block_pos) { - new_blocks.push_back(std::move(old_blocks[i])); - } - } - - const auto id_map = ApplyBlockOrder(graph, std::move(new_blocks)); - - return RemapId(block_id, id_map); -} - -std::vector DominatedBlocks(const Graph& graph, uint32_t header, - uint32_t stop_block = UINT32_MAX) { - std::vector blocks; - std::vector stack = {header}; - blocks.reserve(graph.blocks.size()); - stack.reserve(graph.blocks.size()); - - while (!stack.empty()) { - const auto block_id = stack.back(); - stack.pop_back(); - if (block_id == stop_block || Contains(blocks, block_id) || - !graph.Dominates(header, block_id)) { - continue; - } - - const auto* block = graph.FindBlock(block_id); - if (block == nullptr) { - continue; - } - - AddUnique(blocks, block_id); - for (auto succ: block->successors) { - if (succ != stop_block && graph.Dominates(header, succ)) { - stack.push_back(succ); - } - } - } - - SortUnique(blocks); - return blocks; -} - -uint32_t AppendSyntheticBranchBlock(Graph& graph, uint32_t target) { - const auto* target_block = graph.FindBlock(target); - - BasicBlock block; - block.id = static_cast(graph.blocks.size()); - block.start_pc = target_block != nullptr ? target_block->start_pc : 0u; - block.end_pc = block.start_pc; - block.inst_begin = target_block != nullptr ? target_block->inst_begin : 0u; - block.inst_end = block.inst_begin; - block.successors = {target}; - block.terminator.kind = TerminatorKind::Branch; - block.terminator.condition = BranchCondition::Always; - block.terminator.true_block = target; - graph.blocks.push_back(std::move(block)); - return graph.blocks.back().id; -} - -bool IsolateSemanticLoopHeader(Graph& graph, uint32_t old_header) { - const auto* header = graph.FindBlock(old_header); - if (header == nullptr || header->inst_begin == header->inst_end) { - return false; - } - - const auto predecessors = header->predecessors; - const auto new_header = AppendSyntheticBranchBlock(graph, old_header); - for (auto pred: predecessors) { - auto* block = graph.FindBlock(pred); - if (block != nullptr) { - ReplaceValue(block->successors, old_header, new_header); - ReplaceTerminatorTarget(block->terminator, old_header, new_header); - } - } - if (graph.entry_block == old_header) { - graph.entry_block = new_header; - } - MoveBlockBefore(graph, new_header, old_header); - RebuildPredecessors(graph); - RecomputeAnalyses(graph); - return true; -} - -const NaturalLoop* FindInnermostContainingLoop(const Graph& graph, uint32_t block_id) { - const NaturalLoop* innermost = nullptr; - for (const auto& loop: graph.natural_loops) { - if (Contains(loop.body_blocks, block_id) && - (innermost == nullptr || loop.body_blocks.size() < innermost->body_blocks.size())) { - innermost = &loop; - } - } - return innermost; -} - -bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) { - return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block && - graph.Dominates(loop.header, block_id) && !graph.Dominates(loop.merge, block_id); -} - -bool IsLoopControlGateway(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) { - const auto* block = graph.FindBlock(block_id); - if (block == nullptr || block->terminator.kind != TerminatorKind::ConditionalBranch) { - return false; - } - const auto is_control_target = [&](uint32_t target) { - return target == loop.merge || target == loop.continue_block; - }; - return is_control_target(block->terminator.true_block) && - is_control_target(block->terminator.false_block); -} - -bool HasLinearPathToTerminal(const Graph& graph, uint32_t start) { - std::vector visited(graph.blocks.size(), false); - for (auto block_id = start;;) { - const auto* block = graph.FindBlock(block_id); - if (block == nullptr || block_id >= visited.size() || visited[block_id]) { - return false; - } - if (block->successors.empty()) { - return true; - } - if (block->successors.size() != 1) { - return false; - } - visited[block_id] = true; - block_id = block->successors.front(); - } -} - -bool IsEnclosingLinearExit(const Graph& graph, uint32_t header, uint32_t block_id) { - // AGC commonly lowers nested early returns through a terminal epilogue shared with an - // enclosing conditional. Such a path is an exit boundary, not part of the inner selection. - if (!HasLinearPathToTerminal(graph, block_id)) { - return false; - } - const auto* block = graph.FindBlock(block_id); - while (block != nullptr && graph.Dominates(header, block->id) && - block->successors.size() == 1u) { - block = graph.FindBlock(block->successors.front()); - } - if (block == nullptr || graph.Dominates(header, block->id) || - block->predecessors.empty()) { - return false; - } - return std::ranges::all_of(block->predecessors, [&](uint32_t predecessor) { - return graph.Dominates(header, predecessor) || graph.Dominates(predecessor, header); - }); -} - -bool CanReachBefore(const Graph& graph, uint32_t start, uint32_t target, uint32_t stop) { - std::vector pending = {start}; - std::vector visited(graph.blocks.size(), false); - while (!pending.empty()) { - const auto block_id = pending.back(); - pending.pop_back(); - if (block_id == target) { - return true; - } - if (block_id == stop || block_id >= visited.size() || visited[block_id]) { - continue; - } - visited[block_id] = true; - const auto* block = graph.FindBlock(block_id); - if (block != nullptr) { - pending.insert(pending.end(), block->successors.begin(), block->successors.end()); - } - } - return false; -} - -uint32_t FindSelectionMerge(const Graph& graph, const BasicBlock& block) { - const auto global_merge = graph.FindNearestCommonPostDominator(block.terminator.true_block, - block.terminator.false_block); - const auto* loop = FindInnermostContainingLoop(graph, block.id); - if (loop == nullptr) { - const auto* global_block = graph.FindBlock(global_merge); - const auto true_target = block.terminator.true_block; - const auto false_target = block.terminator.false_block; - if (global_block == nullptr || global_block->successors.empty()) { - const bool false_reaches_true = - CanReachBefore(graph, false_target, true_target, global_merge); - const bool true_reaches_false = - CanReachBefore(graph, true_target, false_target, global_merge); - if (false_reaches_true != true_reaches_false) { - return false_reaches_true ? true_target : false_target; - } - if (global_merge == UINT32_MAX) { - // An enclosing selection's shared return needs its own inner merge - // gateway; the other arm can return directly without joining live state. - if (IsEnclosingLinearExit(graph, block.id, true_target)) { - return true_target; - } - if (IsEnclosingLinearExit(graph, block.id, false_target)) { - return false_target; - } - // A return can leave a selection without reaching its merge. Keep the - // continuing arm as the merge. Shared continuations receive a dedicated - // gateway in SplitSharedMergeBlock before structured control is emitted. - if (HasLinearPathToTerminal(graph, true_target)) { - return false_target; - } - if (HasLinearPathToTerminal(graph, false_target)) { - return true_target; - } - } - } - return global_merge; - } - - // A selection can join at the loop-control gateway even when another arm breaks directly to - // the enclosing loop merge. Global post-dominance sees that break and incorrectly expands the - // selection through the backedge. - const auto true_target = block.terminator.true_block; - const auto false_target = block.terminator.false_block; - if (IsLoopControlGateway(graph, *loop, true_target) && graph.Dominates(block.id, true_target) && - IsInsideLoopConstruct(graph, *loop, false_target)) { - return true_target; - } - if (IsLoopControlGateway(graph, *loop, false_target) && - graph.Dominates(block.id, false_target) && - IsInsideLoopConstruct(graph, *loop, true_target)) { - return false_target; - } - return global_merge; -} - -bool IsInnermostLoopControlConditional(const Graph& graph, const BasicBlock& block) { - if (block.terminator.kind != TerminatorKind::ConditionalBranch) { - return false; - } - const auto* loop = FindInnermostContainingLoop(graph, block.id); - if (loop == nullptr || loop->merge == UINT32_MAX || loop->continue_block == UINT32_MAX) { - return false; - } - const auto true_target = block.terminator.true_block; - const auto false_target = block.terminator.false_block; - if (block.id == loop->continue_block) { - const auto is_repeat_target = [&](uint32_t target) { - return target == loop->header || target == loop->merge; - }; - return is_repeat_target(true_target) && is_repeat_target(false_target); - } - const bool true_in_body = Contains(loop->body_blocks, true_target); - const bool false_in_body = Contains(loop->body_blocks, false_target); - if (true_in_body != false_in_body) { - return true; - } - const auto is_control_target = [&](uint32_t target) { - return target == loop->merge || target == loop->continue_block; - }; - return (is_control_target(true_target) && - (is_control_target(false_target) || - IsInsideLoopConstruct(graph, *loop, false_target))) || - (is_control_target(false_target) && IsInsideLoopConstruct(graph, *loop, true_target)); -} - -bool MergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) { - for (const auto& loop: graph.natural_loops) { - if (loop.header != header && IsInsideLoopConstruct(graph, loop, header) && - !IsInsideLoopConstruct(graph, loop, merge)) { - return true; - } - } - return false; -} - -bool CanonicalizeNaturalLoops(Graph& graph) { - const auto rewrite_budget = graph.blocks.size() * 2u + 16u; - for (size_t rewrite = 0; rewrite < rewrite_budget; rewrite++) { - bool changed = false; - for (const auto& loop: graph.natural_loops) { - std::vector latches; - for (const auto& edge: graph.back_edges) { - if (edge.to == loop.header) { - AddUnique(latches, edge.from); - } - } - if (latches.size() <= 1u) { - continue; - } - - const auto continue_block = AppendSyntheticBranchBlock(graph, loop.header); - for (auto latch: latches) { - auto* block = graph.FindBlock(latch); - if (block != nullptr) { - ReplaceValue(block->successors, loop.header, continue_block); - ReplaceTerminatorTarget(block->terminator, loop.header, continue_block); - } - } - RebuildPredecessors(graph); - RecomputeAnalyses(graph); - changed = true; - break; - } - if (changed) { - continue; - } - - for (const auto& loop: graph.natural_loops) { - const auto* header = graph.FindBlock(loop.header); - const auto is_loop_control_target = [&](uint32_t target) { - return target == loop.merge || target == loop.continue_block; - }; - if (header == nullptr || header->terminator.kind != TerminatorKind::ConditionalBranch || - is_loop_control_target(header->terminator.true_block) || - is_loop_control_target(header->terminator.false_block) || - !Contains(loop.body_blocks, header->terminator.true_block) || - !Contains(loop.body_blocks, header->terminator.false_block)) { - continue; - } - - IsolateSemanticLoopHeader(graph, loop.header); - changed = true; - break; - } - if (!changed) { - return true; - } - } - - SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block, - "CFG loop canonicalization exceeded rewrite budget"); - return false; -} - -bool IsolateSemanticLoopHeaders(Graph& graph) { - const auto isolation_budget = graph.natural_loops.size() + 1u; - for (size_t isolation = 0; isolation < isolation_budget; isolation++) { - const auto loop = std::find_if( - graph.natural_loops.begin(), graph.natural_loops.end(), [&](const auto& value) { - const auto* header = graph.FindBlock(value.header); - return header != nullptr && header->inst_begin != header->inst_end; - }); - if (loop == graph.natural_loops.end()) { - return true; - } - - // SPIR-V requires OpLoopMerge and its branch to remain in the loop header's - // physical block. Keep that header as a dedicated control node, exactly like - // a separate Loop node in a structured AST. Guest instructions live - // in the body so later translation may introduce bounds/EXEC control flow without - // displacing OpLoopMerge into a helper-created block. - if (!IsolateSemanticLoopHeader(graph, loop->header)) { - SetFailure(graph, FailureKind::StructuredControlFlow, loop->header, - fmt::format("failed to isolate semantic loop header {}", loop->header)); - return false; - } - } - - SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block, - "CFG semantic loop-header isolation exceeded rewrite budget"); - return false; -} - -bool SplitSharedMergeBlock(Graph& graph, uint32_t merge, - const std::vector& construct_blocks, - bool force_split = false) { - if (merge == UINT32_MAX || merge >= graph.blocks.size() || Contains(construct_blocks, merge)) { - return false; - } - - const auto* merge_block = graph.FindBlock(merge); - if (merge_block == nullptr) { - return false; - } - - std::vector construct_predecessors; - bool has_external_predecessor = false; - for (auto pred: merge_block->predecessors) { - if (Contains(construct_blocks, pred)) { - AddUnique(construct_predecessors, pred); - } else { - has_external_predecessor = true; - } - } - - if (construct_predecessors.empty() || (!force_split && !has_external_predecessor)) { - return false; - } - if (construct_predecessors.size() == 1u) { - const auto* predecessor = graph.FindBlock(construct_predecessors.front()); - if (predecessor != nullptr && predecessor->inst_begin == predecessor->inst_end && - predecessor->terminator.kind == TerminatorKind::Branch && - predecessor->terminator.true_block == merge && - predecessor->start_pc == merge_block->start_pc) { - return false; - } - } - const auto synthetic_merge = AppendSyntheticBranchBlock(graph, merge); - auto* synthetic_block = graph.FindBlock(synthetic_merge); - if (synthetic_block != nullptr) { - synthetic_block->predecessors = construct_predecessors; - SortUnique(synthetic_block->predecessors); - } - - for (auto pred: construct_predecessors) { - auto* block = graph.FindBlock(pred); - if (block == nullptr) { - continue; - } - ReplaceValue(block->successors, merge, synthetic_merge); - ReplaceTerminatorTarget(block->terminator, merge, synthetic_merge); - } - - auto* old_merge = graph.FindBlock(merge); - if (old_merge != nullptr) { - for (auto pred: construct_predecessors) { - RemoveValue(old_merge->predecessors, pred); - } - AddUnique(old_merge->predecessors, synthetic_merge); - SortUnique(old_merge->predecessors); - } - - MoveBlockBefore(graph, synthetic_merge, merge); - return true; -} - -bool SplitOneLoopMerge(Graph& graph) { - const auto& loops = graph.natural_loops; - for (const auto& loop: loops) { - const auto construct_blocks = DominatedBlocks(graph, loop.header, loop.merge); - const auto force_split = MergeLeavesContainingLoop(graph, loop.header, loop.merge); - if (SplitSharedMergeBlock(graph, loop.merge, construct_blocks, force_split)) { - return true; - } - } - return false; -} - -std::vector SelectionRegion(const Graph& graph, const BasicBlock& header, - uint32_t merge) { - std::vector region; - std::vector pending = {header.terminator.true_block, header.terminator.false_block}; - const auto* loop = FindInnermostContainingLoop(graph, header.id); - while (!pending.empty()) { - const auto block_id = pending.back(); - pending.pop_back(); - if (block_id == merge || Contains(region, block_id) || - (loop != nullptr && (block_id == loop->merge || block_id == loop->continue_block))) { - continue; - } - const auto* block = graph.FindBlock(block_id); - if (block == nullptr) { - continue; - } - // A return terminates its own block; a branch to a shared return still has to - // obey selection entry/exit rules, just like any other branch. - AddUnique(region, block_id); - pending.insert(pending.end(), block->successors.begin(), block->successors.end()); - } - SortUnique(region); - return region; -} - -bool SplitOneSelectionMerge(Graph& graph) { - std::vector loop_headers; - loop_headers.reserve(graph.natural_loops.size()); - for (const auto& loop: graph.natural_loops) { - AddUnique(loop_headers, loop.header); - } - - std::vector selection_headers; - for (const auto& block: graph.blocks) { - if (block.terminator.kind == TerminatorKind::ConditionalBranch && - !Contains(loop_headers, block.id)) { - selection_headers.push_back(block.id); - } - } - std::sort(selection_headers.begin(), selection_headers.end(), [&](uint32_t lhs, uint32_t rhs) { - const auto* lhs_block = graph.FindBlock(lhs); - const auto* rhs_block = graph.FindBlock(rhs); - const auto lhs_depth = lhs_block != nullptr ? lhs_block->dominators.size() : 0u; - const auto rhs_depth = rhs_block != nullptr ? rhs_block->dominators.size() : 0u; - return lhs_depth != rhs_depth ? lhs_depth > rhs_depth : lhs < rhs; - }); - - for (const auto block_id: selection_headers) { - const auto* block = graph.FindBlock(block_id); - if (block == nullptr) { - continue; - } - if (IsInnermostLoopControlConditional(graph, *block)) { - continue; - } - - const auto merge = FindSelectionMerge(graph, *block); - if (merge == UINT32_MAX || graph.FindBlock(merge) == nullptr) { - continue; - } - const auto region = SelectionRegion(graph, *block, merge); - const auto external = std::find_if(region.begin(), region.end(), [&](uint32_t member) { - const auto* member_block = graph.FindBlock(member); - return member_block != nullptr && - std::ranges::any_of(member_block->predecessors, [&](uint32_t predecessor) { - return predecessor != block_id && !Contains(region, predecessor); - }); - }); - if (external != region.end()) { - SetFailure( - graph, FailureKind::StructuredControlFlow, block_id, - fmt::format("selection header block {} has externally entered region block {}; " - "semantic block cloning is disabled", - block_id, *external)); - return false; - } - const auto construct_blocks = DominatedBlocks(graph, block_id, merge); - const auto force_split = MergeLeavesContainingLoop(graph, block_id, merge); - if (SplitSharedMergeBlock(graph, merge, construct_blocks, force_split)) { - return true; - } - } - return false; -} - -bool SplitSharedMergeBlocks(Graph& graph) { - const auto original_block_count = static_cast(graph.blocks.size()); - const auto split_budget = std::max(16u, original_block_count * 4u); - for (uint32_t splits = 0; splits < split_budget; splits++) { - if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph)) { - return !graph.unsupported; - } - RebuildPredecessors(graph); - RecomputeAnalyses(graph); - } - SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block, - fmt::format("CFG shared merge splitting exceeded budget: original_blocks={} " - "current_blocks={} split_budget={}", - original_block_count, static_cast(graph.blocks.size()), - split_budget)); - return false; -} - -void ClearStructuredTerminators(Graph& graph) { - for (auto& block: graph.blocks) { - block.terminator.merge_block = UINT32_MAX; - block.terminator.continue_block = UINT32_MAX; - block.terminator.loop_header = false; - } -} - std::string VectorToString(const std::vector& values) { std::string text; for (uint32_t i = 0; i < values.size(); i++) { @@ -1597,292 +953,495 @@ bool Graph::Dominates(uint32_t dominator, uint32_t block) const { return target != nullptr && Contains(target->dominators, dominator); } -bool Graph::PostDominates(uint32_t post_dominator, uint32_t block) const { - const auto* target = FindBlock(block); - return target != nullptr && Contains(target->post_dominators, post_dominator); -} - -uint32_t Graph::FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const { - const auto* a = FindBlock(block_a); - const auto* b = FindBlock(block_b); - if (a == nullptr || b == nullptr) { - return UINT32_MAX; - } - - const auto common = IntersectSorted(a->post_dominators, b->post_dominators); - for (auto candidate: common) { - bool nearest = true; - for (auto other: common) { - if (other != candidate && !PostDominates(other, candidate)) { - nearest = false; - break; - } - } - if (nearest) { - return candidate; - } - } - return common.empty() ? UINT32_MAX : common.front(); -} - namespace { -struct GotoRouteBlocks { - uint32_t before = UINT32_MAX; - std::vector blocks; -}; +// Eliminate gotos by moving them through a structured statement tree. Every Code node +// retains exactly one guest block; only Boolean route values cross construct boundaries. +// This follows the outward/inward/lifting transformations used by shadPS4's GotoPass. +class GotoStructurizer { + using ExprOp = ConditionExpression::Op; + enum class Kind { Root, Code, Label, Goto, If, Loop, Break, Return, Set }; + struct Node; + using List = std::list; + struct Node { + Kind kind; + Node* parent = nullptr; + Node* target = nullptr; + List children; + List::iterator position; + uint32_t condition = 1; + uint32_t id = UINT32_MAX; + }; -void PlaceGotoRouteBlocks(Graph& graph, uint32_t original_block_count, - const GotoRouteBlocks& route) { - std::vector old_blocks = std::move(graph.blocks); - std::vector new_blocks; - new_blocks.reserve(old_blocks.size()); - for (uint32_t block_id = 0; block_id < original_block_count; block_id++) { - if (block_id == route.before) { - for (const auto route_block: route.blocks) { - new_blocks.push_back(std::move(old_blocks[route_block])); - } +public: + explicit GotoStructurizer(const Graph& source): m_source(source) { + m_graph.expressions = {{ExprOp::Constant, 0}, {ExprOp::Constant, 1}}; + m_graph.code_table_load_pcs = source.code_table_load_pcs; + m_root = New(Kind::Root); + std::vector labels; + for (const auto& block: source.blocks) { + auto* label = Insert(m_root, m_root->children.end(), Kind::Label); + label->id = block.id; + labels.push_back(label); } - new_blocks.push_back(std::move(old_blocks[block_id])); - } - - ApplyBlockOrder(graph, std::move(new_blocks)); -} - -uint32_t AppendGotoSelectBlock(Graph& graph, uint32_t route_variable, uint32_t true_target, - uint32_t false_target) { - const auto* target = graph.FindBlock(false_target); - BasicBlock block; - block.id = static_cast(graph.blocks.size()); - block.start_pc = target != nullptr ? target->start_pc : 0u; - block.end_pc = block.start_pc; - block.inst_begin = target != nullptr ? target->inst_begin : 0u; - block.inst_end = block.inst_begin; - block.successors = {true_target, false_target}; - block.terminator.kind = TerminatorKind::ConditionalBranch; - block.terminator.condition = BranchCondition::GotoVariable; - block.terminator.true_block = true_target; - block.terminator.false_block = false_target; - block.terminator.goto_variable = route_variable; - graph.blocks.push_back(std::move(block)); - return graph.blocks.back().id; -} - -uint32_t AppendGotoSetBlock(Graph& graph, uint32_t route_variable, bool value, uint32_t target) { - const auto block_id = AppendSyntheticBranchBlock(graph, target); - auto* block = graph.FindBlock(block_id); - block->terminator.goto_variable = route_variable; - block->terminator.goto_value = value ? 1 : 0; - return block_id; -} - -void AppendEnclosingRouteBlocks(Graph& graph, uint32_t original_block_count, - uint32_t route_variable, uint32_t shared, uint32_t other, - uint32_t route_select, uint32_t route_root, - std::vector& route_blocks) { - for (uint32_t depth = 0; depth < original_block_count; depth++) { - uint32_t predecessor = UINT32_MAX; - uint32_t leaf = UINT32_MAX; - for (uint32_t candidate = 0; candidate < original_block_count; candidate++) { - const auto* candidate_block = graph.FindBlock(candidate); - if (candidate_block == nullptr || - candidate_block->terminator.kind != TerminatorKind::ConditionalBranch || - IsInnermostLoopControlConditional(graph, *candidate_block)) { - continue; - } - const auto candidate_true = candidate_block->terminator.true_block; - const auto candidate_false = candidate_block->terminator.false_block; - if (candidate_true == route_root && - (candidate_false == shared || candidate_false == other)) { - predecessor = candidate; - leaf = candidate_false; - break; - } - if (candidate_false == route_root && - (candidate_true == shared || candidate_true == other)) { - predecessor = candidate; - leaf = candidate_true; - break; + // Give every natural loop one backedge before eliminating gotos. Otherwise + // multiple latches become nested loops that retest the same exit predicate. + struct Continue { uint32_t header; uint32_t after; size_t body_size; Node* label; }; + std::vector continues; + for (const auto& loop: source.natural_loops) { + if (std::ranges::any_of(continues, [&](const auto& value) { return value.header == loop.header; }) || + std::ranges::count(source.natural_loops, loop.header, &NaturalLoop::header) < 2) continue; + std::vector body; + for (const auto& member: source.natural_loops) { + if (member.header == loop.header) + body.insert(body.end(), member.body_blocks.begin(), member.body_blocks.end()); } + SortUnique(body); + auto* label = New(Kind::Label); + label->id = static_cast(labels.size()); + labels.push_back(label); + continues.push_back({loop.header, body.back(), body.size(), label}); } - if (predecessor == UINT32_MAX) { - break; - } - - const auto routed_leaf = - AppendGotoSetBlock(graph, route_variable, leaf == other, route_select); - auto* predecessor_block = graph.FindBlock(predecessor); - ReplaceValue(predecessor_block->successors, leaf, routed_leaf); - ReplaceTerminatorTarget(predecessor_block->terminator, leaf, routed_leaf); - route_blocks.push_back(routed_leaf); - route_root = predecessor; - } -} - -// Turn H0 -> shared/H1, H1 -> shared/other into nested selections whose empty -// forwarding blocks set one typed SSA route value. Guest semantic blocks remain unique. -// Preserve loop-control branches just as SplitOneSelectionMerge does. Moving a loop -// exit through the route selector can create an inner cycle whose merge is also the -// enclosing loop merge, so splitting that shared exit can never separate the loops. -bool RouteOneSharedArm(Graph& graph, uint32_t original_block_count, uint32_t outer_id, - uint32_t route_variable, GotoRouteBlocks& route) { - auto* outer = graph.FindBlock(outer_id); - if (outer == nullptr || outer->inst_begin == outer->inst_end || - outer->terminator.kind != TerminatorKind::ConditionalBranch || - IsInnermostLoopControlConditional(graph, *outer)) { - return false; - } - - for (const bool inner_is_true: {true, false}) { - const auto inner_id = - inner_is_true ? outer->terminator.true_block : outer->terminator.false_block; - const auto shared = - inner_is_true ? outer->terminator.false_block : outer->terminator.true_block; - const auto* inner = graph.FindBlock(inner_id); - if (inner == nullptr || inner->inst_begin == inner->inst_end || - inner->predecessors.size() != 1u || inner->predecessors.front() != outer_id || - inner->terminator.kind != TerminatorKind::ConditionalBranch || - IsInnermostLoopControlConditional(graph, *inner)) { - continue; - } - - const bool true_is_shared = inner->terminator.true_block == shared; - const bool false_is_shared = inner->terminator.false_block == shared; - if (true_is_shared != false_is_shared) { - const auto other = - true_is_shared ? inner->terminator.false_block : inner->terminator.true_block; - if (other == outer_id || other == inner_id || other == shared) { - continue; - } - const auto first_arm = std::min(shared, other); - if (first_arm >= original_block_count || outer_id >= inner_id || - inner_id >= first_arm) { - continue; - } - - const auto route_select = AppendGotoSelectBlock(graph, route_variable, other, shared); - const auto inner_merge = AppendSyntheticBranchBlock(graph, route_select); - const auto outer_shared = - AppendGotoSetBlock(graph, route_variable, false, route_select); - const auto inner_shared = AppendGotoSetBlock(graph, route_variable, false, inner_merge); - const auto inner_other = AppendGotoSetBlock(graph, route_variable, true, inner_merge); - - outer = graph.FindBlock(outer_id); - auto* mutable_inner = graph.FindBlock(inner_id); - outer->terminator.true_block = inner_is_true ? inner_id : outer_shared; - outer->terminator.false_block = inner_is_true ? outer_shared : inner_id; - outer->successors = {outer->terminator.true_block, outer->terminator.false_block}; - mutable_inner->terminator.true_block = true_is_shared ? inner_shared : inner_other; - mutable_inner->terminator.false_block = true_is_shared ? inner_other : inner_shared; - mutable_inner->successors = {mutable_inner->terminator.true_block, - mutable_inner->terminator.false_block}; - - std::vector route_blocks = {inner_shared, inner_other, inner_merge, - outer_shared}; - AppendEnclosingRouteBlocks(graph, original_block_count, route_variable, shared, other, - route_select, outer_id, route_blocks); - route_blocks.push_back(route_select); - route = {first_arm, std::move(route_blocks)}; - return true; - } - - // H0 -> arm/H1, H1 -> exit/body, and both arms reach a common continuation. - // Route exit/continuation after joining H0 and H1 so neither has a nonlocal edge. - for (const bool exit_is_true: {true, false}) { - const auto other = - exit_is_true ? inner->terminator.true_block : inner->terminator.false_block; - const auto body = - exit_is_true ? inner->terminator.false_block : inner->terminator.true_block; - if (other == outer_id || other == inner_id || other == shared || body == outer_id || - body == inner_id || body == shared || !graph.Dominates(inner_id, body)) { - continue; - } - - const auto continuation = graph.FindNearestCommonPostDominator(shared, body); - const auto* continuation_block = graph.FindBlock(continuation); - if (continuation_block == nullptr || continuation == other || - CanReachBefore(graph, other, continuation, UINT32_MAX)) { - continue; - } - std::vector outer_predecessors; - std::vector inner_predecessors; - bool external_predecessor = false; - for (const auto predecessor: continuation_block->predecessors) { - if (predecessor == outer_id || graph.Dominates(shared, predecessor)) { - outer_predecessors.push_back(predecessor); - } else if (predecessor == inner_id || graph.Dominates(body, predecessor)) { - inner_predecessors.push_back(predecessor); - } else { - external_predecessor = true; + std::ranges::sort(continues, [](const auto& a, const auto& b) { + return a.after != b.after ? a.after < b.after : a.body_size < b.body_size; + }); + m_route_expressions.resize(labels.size(), UINT32_MAX); + for (const auto& block: source.blocks) { + auto* label = labels.at(block.id); + const auto end = std::next(label->position); + auto* code = Insert(m_root, end, Kind::Code); + code->id = block.id; + const auto add_goto = [&](uint32_t condition, uint32_t target) { + auto* node = Insert(m_root, end, Kind::Goto); + node->condition = condition; + node->target = labels.at(target); + for (const auto& value: continues) { + if (value.header == target && source.Dominates(target, block.id)) + node->target = value.label; } - } - const auto first_arm = std::min(continuation, other); - if (outer_predecessors.empty() || inner_predecessors.empty() || - external_predecessor || first_arm >= original_block_count || - outer_id >= inner_id || inner_id >= first_arm) { - continue; - } - - const auto route_select = - AppendGotoSelectBlock(graph, route_variable, other, continuation); - const auto inner_merge = AppendSyntheticBranchBlock(graph, route_select); - const auto outer_continue = - AppendGotoSetBlock(graph, route_variable, false, route_select); - const auto inner_continue = - AppendGotoSetBlock(graph, route_variable, false, inner_merge); - const auto inner_other = AppendGotoSetBlock(graph, route_variable, true, inner_merge); - - auto* mutable_inner = graph.FindBlock(inner_id); - if (exit_is_true) { - mutable_inner->terminator.true_block = inner_other; + m_gotos.push_back(node); + }; + const auto& term = block.terminator; + if (term.kind == TerminatorKind::ConditionalBranch) { + // Capture at the semantic block, before movement can pass an SCC/EXEC write. + const auto condition = Expression(ExprOp::Variable, CaptureVariable(block.id)); + add_goto(condition, term.true_block); + add_goto(1, term.false_block); + } else if (term.kind == TerminatorKind::Branch) { + add_goto(1, term.true_block); + } else if (term.kind == TerminatorKind::Return) { + Insert(m_root, end, Kind::Return); } else { - mutable_inner->terminator.false_block = inner_other; + Fail("indirect control flow requires dispatcher lowering"); } - ReplaceValue(mutable_inner->successors, other, inner_other); - for (const auto predecessor: outer_predecessors) { - auto* predecessor_block = graph.FindBlock(predecessor); - ReplaceValue(predecessor_block->successors, continuation, outer_continue); - ReplaceTerminatorTarget(predecessor_block->terminator, continuation, - outer_continue); + for (const auto& value: continues) { + if (value.after != block.id) continue; + value.label->parent = m_root; + value.label->position = m_root->children.insert(end, value.label); + auto* repeat = Insert(m_root, end, Kind::Goto); + repeat->target = labels[value.header]; + m_gotos.push_back(repeat); } - for (const auto predecessor: inner_predecessors) { - auto* predecessor_block = graph.FindBlock(predecessor); - ReplaceValue(predecessor_block->successors, continuation, inner_continue); - ReplaceTerminatorTarget(predecessor_block->terminator, continuation, - inner_continue); - } - - std::vector route_blocks = {inner_continue, inner_other, inner_merge, - outer_continue}; - if (continuation == shared) { - AppendEnclosingRouteBlocks(graph, original_block_count, route_variable, shared, - other, route_select, outer_id, route_blocks); - } - route_blocks.push_back(route_select); - route = {first_arm, std::move(route_blocks)}; - return true; } } - return false; -} -bool RouteSharedSelectionArm(Graph& graph, uint32_t route_variable) { - if (graph.irreducible) { + Graph Run() { + for (auto it = m_gotos.rbegin(); it != m_gotos.rend() && m_failure == nullptr; ++it) { + Eliminate(*it); + } + if (m_failure != nullptr) return Failed(); + const auto entry = Allocate(); + m_graph.entry_block = entry; + Place(entry); + Branch(Visit(m_root, entry, UINT32_MAX), UINT32_MAX); + if (m_failure != nullptr) return Failed(); + std::vector ordered; + ordered.reserve(m_order.size()); + for (const auto id: m_order) { + ordered.push_back(std::move(m_graph.blocks[id])); + } + ApplyBlockOrder(m_graph, std::move(ordered)); + RebuildPredecessors(m_graph); + if (std::ranges::any_of(m_graph.blocks, [&](const auto& block) { + return block.id != m_graph.entry_block && block.predecessors.empty(); + })) { + Fail("structured control flow requires an unreachable merge"); + return Failed(); + } + RecomputeAnalyses(m_graph); + if (m_graph.irreducible) { + Fail("goto elimination produced irreducible control flow"); + return Failed(); + } + return std::move(m_graph); + } + +private: + void Fail(const char* reason) { m_failure = reason; } + Graph Failed() { + SetFailure(m_graph, FailureKind::StructuredControlFlow, UINT32_MAX, m_failure); + return std::move(m_graph); + } + uint32_t CaptureVariable(uint32_t block) const { + return static_cast(m_route_expressions.size()) + block; + } + Node* New(Kind kind) { + m_nodes.emplace_back(); + m_nodes.back().kind = kind; + return &m_nodes.back(); + } + Node* Insert(Node* parent, List::iterator before, Kind kind) { + auto* node = New(kind); + node->parent = parent; + node->position = parent->children.insert(before, node); + return node; + } + void Erase(Node* node) { node->parent->children.erase(node->position); } + uint32_t Expression(ExprOp op, uint32_t lhs, uint32_t rhs = UINT32_MAX) { + if (op == ExprOp::Not) { + const auto& inner = m_graph.expressions[lhs]; + if (inner.op == ExprOp::Constant) return inner.lhs ? 0 : 1; + if (inner.op == ExprOp::Not) return inner.lhs; + } + if (op == ExprOp::Or) { + if (lhs == 1 || rhs == 1) return 1; + if (lhs == 0 || lhs == rhs) return rhs; + if (rhs == 0) return lhs; + } + const auto result = static_cast(m_graph.expressions.size()); + m_graph.expressions.push_back({op, lhs, rhs}); + return result; + } + uint32_t RouteVariable(Node* label) { + auto& expression = m_route_expressions[label->id]; + if (expression == UINT32_MAX) { + expression = Expression(ExprOp::Variable, label->id); + // SSA initializes route variables to false; clear them again on each label visit. + auto* reset = Insert(label->parent, std::next(label->position), Kind::Set); + reset->id = label->id; + reset->condition = 0; + } + return expression; + } + static size_t Depth(Node* node) { + size_t depth = 0; + for (; node->parent != nullptr; node = node->parent) ++depth; + return depth; + } + static bool Related(Node* a, Node* b) { + auto a_depth = Depth(a); + auto b_depth = Depth(b); + while (a_depth > b_depth) { a = a->parent; --a_depth; } + while (b_depth > a_depth) { b = b->parent; --b_depth; } + return a->parent == b->parent; + } + static bool Ordered(Node* a, Node* b) { + for (auto it = a->position; it != a->parent->children.end(); ++it) { + if (*it == b) return true; + } return false; } - const auto block_count = static_cast(graph.blocks.size()); - // Route later/deeper candidates first so earlier shared arms remain structured. - for (uint32_t block_id = block_count; block_id-- > 0;) { - GotoRouteBlocks route; - if (RouteOneSharedArm(graph, block_count, block_id, route_variable, route)) { - PlaceGotoRouteBlocks(graph, block_count, route); - RebuildPredecessors(graph); - RecomputeAnalyses(graph); - return true; + static Node* Sibling(Node* uncle, Node* nephew) { + while (nephew->parent != uncle->parent) nephew = nephew->parent; + return nephew; + } + static void Adopt(Node* parent) { + for (auto* child: parent->children) child->parent = parent; + } + void SetBefore(Node* node, uint32_t id, uint32_t expression) { + auto* set = Insert(node->parent, node->position, Kind::Set); + set->id = id; + set->condition = expression; + } + Node* GotoAfter(Node* construct, Node* label, uint32_t condition) { + auto* node = Insert(construct->parent, std::next(construct->position), Kind::Goto); + node->target = label; + node->condition = condition; + return node; + } + Node* Outward(Node* node) { + auto* parent = node->parent; + auto* target = node->target; + const auto variable = RouteVariable(target); + SetBefore(node, target->id, node->condition); + if (parent->kind == Kind::If) { + auto* rest = Insert(parent, node->position, Kind::If); + rest->condition = Expression(ExprOp::Not, variable); + rest->children.splice(rest->children.end(), parent->children, + std::next(node->position), parent->children.end()); + Adopt(rest); + } else if (parent->kind == Kind::Loop) { + auto* stop = Insert(parent, node->position, Kind::Break); + stop->condition = variable; + } else { + Fail("goto has no enclosing selection or loop"); + return node; + } + Erase(node); + return GotoAfter(parent, target, variable); + } + Node* Inward(Node* node) { + auto* parent = node->parent; + auto* target = node->target; + auto* nested = Sibling(node, target); + const auto variable = RouteVariable(target); + SetBefore(node, target->id, node->condition); + auto* rest = Insert(parent, node->position, Kind::If); + rest->condition = Expression(ExprOp::Not, variable); + rest->children.splice(rest->children.end(), parent->children, + std::next(node->position), nested->position); + Adopt(rest); + Erase(node); + if (nested->kind == Kind::If) { + nested->condition = Expression(ExprOp::Or, variable, nested->condition); + } else if (nested->kind != Kind::Loop) { + Fail("goto target has no enclosing construct"); + return node; + } + auto* moved = Insert(nested, nested->children.begin(), Kind::Goto); + moved->target = target; + moved->condition = variable; + return moved; + } + Node* Lift(Node* node) { + auto* parent = node->parent; + auto* target = node->target; + auto* nested = Sibling(node, target); + const auto variable = RouteVariable(target); + auto* loop = Insert(parent, node->position, Kind::Loop); + loop->condition = variable; + loop->children.splice(loop->children.end(), parent->children, + nested->position, loop->position); + Adopt(loop); + if (std::ranges::any_of(loop->children, [](auto* child) { return child->kind == Kind::Break; })) { + Fail("goto lifting would capture an enclosing loop exit"); + return node; + } + auto* moved = Insert(loop, loop->children.begin(), Kind::Goto); + moved->target = target; + moved->condition = variable; + auto* set = Insert(loop, loop->children.end(), Kind::Set); + set->id = target->id; + set->condition = node->condition; + Erase(node); + return moved; + } + void Eliminate(Node* node) { + auto* target = node->target; + while (node->parent != target->parent && m_failure == nullptr) { + if (!Related(node, target) || Depth(node) > Depth(target)) { + node = Outward(node); + } else if (Ordered(Sibling(node, target), node)) { + node = Lift(node); + } else { + node = Inward(node); + } + } + if (m_failure != nullptr) return; + if (std::next(node->position) == target->position) { + Erase(node); + return; + } + auto* parent = node->parent; + if (Ordered(node, target)) { + auto* selection = Insert(parent, node->position, Kind::If); + selection->condition = Expression(ExprOp::Not, node->condition); + selection->children.splice(selection->children.end(), parent->children, + std::next(node->position), target->position); + Adopt(selection); + } else { + auto* loop = Insert(parent, node->position, Kind::Loop); + loop->condition = node->condition; + loop->children.splice(loop->children.end(), parent->children, + target->position, loop->position); + Adopt(loop); + } + Erase(node); + } + + uint32_t Allocate() { + BasicBlock block; + block.id = static_cast(m_graph.blocks.size()); + m_graph.blocks.push_back(std::move(block)); + return m_graph.blocks.back().id; + } + void Place(uint32_t block) { m_order.push_back(block); } + void Branch(uint32_t from, uint32_t to) { + if (from == UINT32_MAX) return; + auto& block = m_graph.blocks[from]; + block.terminator.kind = to == UINT32_MAX ? TerminatorKind::Return : TerminatorKind::Branch; + block.terminator.true_block = to; + block.successors = to == UINT32_MAX ? std::vector {} : std::vector {to}; + } + void Conditional(uint32_t block_id, uint32_t condition, uint32_t yes, uint32_t no, + uint32_t merge = UINT32_MAX) { + auto& block = m_graph.blocks[block_id]; + auto& term = block.terminator; + term.kind = TerminatorKind::ConditionalBranch; + term.expression = condition; + term.condition = BranchCondition::Expression; + term.true_block = yes; + term.false_block = no; + term.merge_block = merge; + block.successors = {yes, no}; + } + uint32_t Advance(uint32_t current) { + const auto next = Allocate(); + Branch(current, next); + Place(next); + return next; + } + bool RefersTo(uint32_t expression, uint32_t variable) const { + const auto& value = m_graph.expressions[expression]; + switch (value.op) { + case ExprOp::Variable: return value.lhs == variable; + case ExprOp::Not: return RefersTo(value.lhs, variable); + case ExprOp::Or: return RefersTo(value.lhs, variable) || RefersTo(value.rhs, variable); + default: return false; } } - return false; -} + bool Modifies(Node* node, uint32_t expression) const { + if (node->kind == Kind::Set && RefersTo(expression, node->id)) return true; + if (node->kind == Kind::Code && + m_source.blocks[node->id].terminator.kind == TerminatorKind::ConditionalBranch && + RefersTo(expression, CaptureVariable(node->id))) return true; + return std::ranges::any_of(node->children, [&](auto* child) { return Modifies(child, expression); }); + } + bool Complements(uint32_t first, uint32_t second) const { + const auto& a = m_graph.expressions[first]; + const auto& b = m_graph.expressions[second]; + return (a.op == ExprOp::Not && a.lhs == second) || + (b.op == ExprOp::Not && b.lhs == first); + } + uint32_t Visit(Node* parent, uint32_t current, uint32_t break_block) { + for (auto it = parent->children.begin(); it != parent->children.end(); ++it) { + auto* node = *it; + if (node->kind == Kind::Label) continue; + if (current == UINT32_MAX) { + current = Allocate(); + Place(current); + } + switch (node->kind) { + case Kind::Code: { + const auto& source = m_source.blocks[node->id]; + const auto& previous = m_graph.blocks[current]; + // Route assignments cannot observe guest writes; native predicates must stay at their original Code. + const bool captures_native = std::ranges::any_of(previous.assignments, [&](const auto& assignment) { + return m_graph.expressions[assignment.expression].op == ExprOp::Native; + }); + if (captures_native || + (previous.inst_begin != previous.inst_end && previous.inst_end != source.inst_begin)) { + current = Advance(current); + } + auto& block = m_graph.blocks[current]; + if (block.inst_begin == block.inst_end) { + block.start_pc = source.start_pc; + block.inst_begin = source.inst_begin; + } + block.end_pc = source.end_pc; + block.inst_end = source.inst_end; + if (source.terminator.kind == TerminatorKind::ConditionalBranch) { + block.assignments.push_back({CaptureVariable(source.id), + Expression(ExprOp::Native, static_cast(source.terminator.condition))}); + } + break; + } + case Kind::Set: { + m_graph.blocks[current].assignments.push_back({node->id, node->condition}); + break; + } + case Kind::If: { + if (node->condition == 0 || node->children.empty()) break; + if (node->condition == 1) { + current = Visit(node, current, break_block); + break; + } + auto next = std::next(it); + while (next != parent->children.end() && (*next)->kind == Kind::Label) ++next; + Node* other = next != parent->children.end() ? *next : nullptr; + const bool paired = other != nullptr && other->kind == Kind::If && + Complements(node->condition, other->condition) && + !Modifies(node, node->condition); + const auto body = Allocate(); + const auto merge = Allocate(); + const auto alternative = paired ? Allocate() : merge; + Conditional(current, node->condition, body, alternative, merge); + Place(body); + Branch(Visit(node, body, break_block), merge); + if (paired) { + Place(alternative); + Branch(Visit(other, alternative, break_block), merge); + it = next; + } + Place(merge); + current = merge; + break; + } + case Kind::Loop: { + const auto& previous = m_graph.blocks[current]; + const bool empty = previous.inst_begin == previous.inst_end && + previous.assignments.empty(); + const auto header = empty ? current : Advance(current); + const auto body = Allocate(); + const auto merge = Allocate(); + Branch(header, body); + Place(body); + auto repeat = Visit(node, body, merge); + // A construct merge belongs to the loop body, not its continue construct. + if (repeat == UINT32_MAX || std::ranges::any_of(m_graph.blocks, [&](const auto& block) { + return block.terminator.merge_block == repeat; + })) { + repeat = Advance(repeat); + } + auto& term = m_graph.blocks[header].terminator; + term.loop_header = true; + term.merge_block = merge; + term.continue_block = repeat; + if (node->condition <= 1) Branch(repeat, node->condition ? header : merge); + else Conditional(repeat, node->condition, header, merge); + Place(merge); + current = merge; + break; + } + case Kind::Break: { + if (break_block == UINT32_MAX) { + Fail("break outside a loop"); + return UINT32_MAX; + } + if (node->condition == 0) break; + if (node->condition == 1) { + Branch(current, break_block); + current = UINT32_MAX; + } else { + const auto next = Allocate(); + Conditional(current, node->condition, break_block, next); + Place(next); + current = next; + } + break; + } + case Kind::Return: + Branch(current, UINT32_MAX); + current = UINT32_MAX; + break; + default: + Fail("uneliminated control-flow statement"); + return UINT32_MAX; + } + if (m_failure != nullptr) return UINT32_MAX; + } + return current; + } + + const char* m_failure = nullptr; + const Graph& m_source; + Graph m_graph; + std::deque m_nodes; + Node* m_root = nullptr; + std::vector m_gotos; + std::vector m_route_expressions; + std::vector m_order; +}; } // namespace @@ -2115,125 +1674,13 @@ Graph BuildGraph(const Decoder::Program& program) { return graph; } -namespace { - -bool StructurizeImpl(Graph& graph) { +Graph Structurize(const Graph& graph) { if (graph.unsupported || graph.irreducible) { - if (graph.unsupported_reason.empty()) { - graph.unsupported_reason = "unsupported CFG"; - } - return false; + Graph failed; + SetFailure(failed, graph.failure_kind, graph.failure_block, graph.unsupported_reason); + return failed; } - - if (!CanonicalizeNaturalLoops(graph)) { - return false; - } - if (!SplitSharedMergeBlocks(graph)) { - return false; - } - if (!IsolateSemanticLoopHeaders(graph)) { - return false; - } - ClearStructuredTerminators(graph); - - std::map merge_headers; - auto reserve_merge_block = [&](uint32_t header, uint32_t merge) { - const auto [it, inserted] = merge_headers.emplace(merge, header); - if (!inserted && it->second != header) { - SetFailure( - graph, FailureKind::StructuredControlFlow, header, - fmt::format( - "duplicate structured merge block {} for header {} (already used by header {})", - merge, header, it->second)); - return false; - } - return true; - }; - - for (const auto& loop: graph.natural_loops) { - auto* header = graph.FindBlock(loop.header); - if (header == nullptr || loop.merge == UINT32_MAX || loop.continue_block == UINT32_MAX) { - SetFailure( - graph, FailureKind::StructuredControlFlow, loop.header, - fmt::format("loop at block {} has no structured merge/continue", loop.header)); - return false; - } - if (header->inst_begin != header->inst_end || - header->terminator.kind != TerminatorKind::Branch) { - SetFailure( - graph, FailureKind::StructuredControlFlow, loop.header, - fmt::format("loop header {} is not a dedicated empty control block", loop.header)); - return false; - } - if (!reserve_merge_block(loop.header, loop.merge)) { - return false; - } - header->terminator.loop_header = true; - header->terminator.merge_block = loop.merge; - header->terminator.continue_block = loop.continue_block; - } - - for (auto& block: graph.blocks) { - if (block.terminator.kind != TerminatorKind::ConditionalBranch || - block.terminator.loop_header) { - continue; - } - if (IsInnermostLoopControlConditional(graph, block)) { - continue; - } - - const auto merge = FindSelectionMerge(graph, block); - if (merge == UINT32_MAX) { - SetFailure(graph, FailureKind::StructuredControlFlow, block.id, - fmt::format("conditional block {} has no structured merge", block.id)); - return false; - } - - if (!reserve_merge_block(block.id, merge)) { - return false; - } - block.terminator.merge_block = merge; - } - - return true; -} - -} // namespace - -bool Structurize(Graph& graph) { - Graph structured = graph; - if (StructurizeImpl(structured)) { - graph = std::move(structured); - return true; - } - - const auto failure_kind = structured.failure_kind; - // Structurization inserts and renumbers blocks. Recover source identity for a - // semantic block; a synthetic block has no corresponding original diagnostic ID. - const auto* failed = structured.FindBlock(structured.failure_block); - const auto original = std::ranges::find_if(graph.blocks, [&](const BasicBlock& block) { - return failed != nullptr && failed->inst_begin != failed->inst_end && - block.inst_begin == failed->inst_begin && block.inst_end == failed->inst_end && - block.start_pc == failed->start_pc && block.end_pc == failed->end_pc; - }); - const auto failure_block = original != graph.blocks.end() ? original->id : UINT32_MAX; - auto failure_reason = std::move(structured.unsupported_reason); - Graph routed = graph; - const auto route_budget = static_cast(graph.blocks.size()); - // Apply one route at a time and retry. Eagerly routing every matching diamond can - // rewrite unrelated selections that were already structurally valid. - for (uint32_t route_variable = 0; route_variable < route_budget; route_variable++) { - if (!RouteSharedSelectionArm(routed, route_variable)) { - break; - } - structured = routed; - if (StructurizeImpl(structured)) { - graph = std::move(structured); - return true; - } - } - SetFailure(graph, failure_kind, failure_block, failure_reason); - return false; + return GotoStructurizer(graph).Run(); } std::string BranchConditionToString(BranchCondition condition) { @@ -2246,7 +1693,7 @@ std::string BranchConditionToString(BranchCondition condition) { case BranchCondition::ExecZero: return "execz"; case BranchCondition::ExecNonZero: return "execnz"; case BranchCondition::ScalarInstruction: return "scalar_instruction"; - case BranchCondition::GotoVariable: return "goto_variable"; + case BranchCondition::Expression: return "expression"; default: return "unknown"; } } @@ -2283,13 +1730,11 @@ std::string GraphToString(const Graph& graph) { text += fmt::format(" predecessors=[{}] successors=[{}]\n", VectorToString(block.predecessors).c_str(), VectorToString(block.successors).c_str()); - text += fmt::format(" dominators=[{}] post_dominators=[{}]\n", - VectorToString(block.dominators).c_str(), - VectorToString(block.post_dominators).c_str()); + text += fmt::format(" dominators=[{}]\n", VectorToString(block.dominators).c_str()); text += fmt::format( " terminator={} condition={} true={} false={} merge={} continue={} loop_header={} " "indirect_sgpr={} indirect_selector={} indirect_targets=[{}] selector_values=[{}] " - "goto_variable={} goto_value={}\n", + "expression={}\n", static_cast(block.terminator.kind), BranchConditionToString(block.terminator.condition).c_str(), block.terminator.true_block, block.terminator.false_block, block.terminator.merge_block, @@ -2297,7 +1742,14 @@ std::string GraphToString(const Graph& graph) { block.terminator.indirect_pc_sgpr, block.terminator.indirect_selector_code, VectorToString(block.terminator.indirect_targets).c_str(), VectorToString(block.terminator.indirect_selector_values).c_str(), - block.terminator.goto_variable, block.terminator.goto_value); + block.terminator.expression); + for (const auto& assignment: block.assignments) { + text += fmt::format(" variable_{} = expression_{}", assignment.variable, assignment.expression); + const auto& expression = graph.expressions[assignment.expression]; + if (expression.op == ConditionExpression::Op::Native) + text += fmt::format(" condition={}", BranchConditionToString(static_cast(expression.lhs))); + text += "\n"; + } } for (const auto& edge: graph.back_edges) { @@ -2305,10 +1757,8 @@ std::string GraphToString(const Graph& graph) { edge.natural ? 1u : 0u); } for (const auto& loop: graph.natural_loops) { - text += fmt::format("loop header={} latch={} merge={} continue={} body=[{}] exits=[{}]\n", - loop.header, loop.latch, loop.merge, loop.continue_block, - VectorToString(loop.body_blocks).c_str(), - VectorToString(loop.exit_blocks).c_str()); + text += fmt::format("loop header={} latch={} body=[{}]\n", loop.header, loop.latch, + VectorToString(loop.body_blocks).c_str()); } for (const auto& component: graph.components) { text += fmt::format("scc blocks=[{}] entries=[{}] irreducible={}\n", diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h index f85c093fc..a051bf2f5 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h @@ -18,10 +18,17 @@ enum class BranchCondition { ExecZero, ExecNonZero, ScalarInstruction, - GotoVariable, + Expression, Unknown }; +struct ConditionExpression { + enum class Op { Constant, Variable, Native, Not, Or }; + Op op; + uint32_t lhs; + uint32_t rhs = UINT32_MAX; +}; + enum class TerminatorKind { Branch, ConditionalBranch, IndirectBranch, Return, Unsupported }; enum class FailureKind { @@ -48,8 +55,7 @@ struct Terminator { std::vector indirect_targets; std::vector indirect_selector_values; std::vector indirect_selector_targets; - uint32_t goto_variable = UINT32_MAX; - int32_t goto_value = -1; + uint32_t expression = UINT32_MAX; bool loop_header = false; }; @@ -59,10 +65,11 @@ struct BasicBlock { uint32_t end_pc = 0; uint32_t inst_begin = 0; uint32_t inst_end = 0; + struct Assignment { uint32_t variable; uint32_t expression; }; + std::vector assignments; std::vector predecessors; std::vector successors; std::vector dominators; - std::vector post_dominators; Terminator terminator; }; @@ -73,12 +80,9 @@ struct BackEdge { }; struct NaturalLoop { - uint32_t header = UINT32_MAX; - uint32_t latch = UINT32_MAX; - uint32_t merge = UINT32_MAX; - uint32_t continue_block = UINT32_MAX; + uint32_t header = UINT32_MAX; + uint32_t latch = UINT32_MAX; std::vector body_blocks; - std::vector exit_blocks; }; struct StronglyConnectedComponent { @@ -89,6 +93,7 @@ struct StronglyConnectedComponent { struct Graph { std::vector blocks; + std::vector expressions; std::vector back_edges; std::vector natural_loops; std::vector components; @@ -105,14 +110,12 @@ struct Graph { const BasicBlock* FindBlockByPc(uint32_t pc) const; BasicBlock* FindBlockByPc(uint32_t pc); bool Dominates(uint32_t dominator, uint32_t block) const; - bool PostDominates(uint32_t post_dominator, uint32_t block) const; - uint32_t FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const; }; Graph BuildGraph(const Decoder::Program& program); -// Commits structured control flow on success; preserves the original graph with -// failure diagnostics on failure. failure_block is an original block ID or UINT32_MAX. -bool Structurize(Graph& graph); +// Returns structured control flow or failure diagnostics without changing the native graph. +// On failure, failure_block is an original block ID or UINT32_MAX. +Graph Structurize(const Graph& graph); std::string BranchConditionToString(BranchCondition condition); std::string FailureKindToString(FailureKind kind); std::string GraphToString(const Graph& graph); diff --git a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp index bc0ce8f17..e864e9c23 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp @@ -44,7 +44,8 @@ void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpc half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32); const auto rhs = half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32); - EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx); + const IR::FPCompareFlags flags{.flush_input_denorms = !half && flush_f32_inputs}; + EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx); } void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 95e12c0c0..db7de4002 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -235,6 +235,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in return index == 0u ? decoded.src0 : MakeImmediate(decoded.offset & 0xffffu); case Decoder::Opcode::DS_CONSUME: case Decoder::Opcode::DS_APPEND: + return decoded.gds ? MakeM0Operand() : MakeImmediate(0); case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand(); case Decoder::Opcode::DS_WRITE_ADDTID_B32: return index == 0u ? decoded.src1 : MakeM0Operand(); @@ -845,8 +846,7 @@ void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { const auto memory = MemoryInfoFromDecoded(inst); WriteOperand(inst.dst, ir.Emit(opcode, - {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec(), ir.GetExecLo(), - ir.GetExecHi()}, + {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, AddMemoryInfo(memory, inst.pc))); } diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 3edb31f67..cfdbbfca6 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -317,6 +317,7 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) { case Decoder::OperandKind::Sgpr: { const auto reg = static_cast(operand.reg); ir.SetScalarReg(reg, value); + program.scalar_writes.push_back({current_pc, reg}); ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false))); if (IR::RegIndex(reg) > 0u) { ir.SetScalarMaskTag(static_cast(IR::RegIndex(reg) - 1u), @@ -699,11 +700,13 @@ std::array Translator::WriteMask(const Decoder::Operand& operand, IR IR::U1(IR::Value(false))); } ir.SetScalarReg(reg, mask[0]); + program.scalar_writes.push_back({current_pc, reg}); // A wave32 VALU mask destination must not overwrite the neighboring SGPR. if ((write_64 || program.wave_size == 64u) && IR::RegIndex(reg) + 1u < IR::NumScalarRegs) { const auto high = static_cast(IR::RegIndex(reg) + 1u); ir.SetScalarReg(high, mask[1]); + program.scalar_writes.push_back({current_pc, high}); ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false))); ir.SetScalarMaskTag(high, IR::U1(IR::Value(false))); } @@ -746,55 +749,61 @@ void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 valu WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value)); } -void Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info) { - if (source.terminator.goto_value >= 0) { - if (source.terminator.goto_variable == UINT32_MAX) { - EXIT("block %u sets an invalid goto variable", source.id); +void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source, + IR::BlockInfo& info) { + const auto native_condition = [&](CFG::BranchCondition kind) -> IR::U1 { + IR::U1 condition; + switch (kind) { + case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break; + case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break; + case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break; + case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break; + case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break; + case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break; + case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break; + case CFG::BranchCondition::ScalarInstruction: + EXIT_IF(instruction_branch_condition.IsEmpty()); + condition = instruction_branch_condition; + break; + default: EXIT("block %u has an invalid native branch condition", source.id); } - ir.SetGotoVariable(source.terminator.goto_variable, - IR::U1(IR::Value(source.terminator.goto_value != 0))); + return IR::U1(ir.Emit(IR::ValueOpcode::ConditionRef, {condition}, kind)); + }; + const auto expression = [&](auto&& self, uint32_t index) -> IR::U1 { + const auto& value = graph.expressions.at(index); + switch (value.op) { + case CFG::ConditionExpression::Op::Constant: return IR::U1(IR::Value(value.lhs != 0)); + case CFG::ConditionExpression::Op::Variable: return ir.GetGotoVariable(value.lhs); + case CFG::ConditionExpression::Op::Native: + return native_condition(static_cast(value.lhs)); + case CFG::ConditionExpression::Op::Not: return ir.LogicalNot(self(self, value.lhs)); + case CFG::ConditionExpression::Op::Or: + return ir.LogicalOr(self(self, value.lhs), self(self, value.rhs)); + } + EXIT("invalid CFG condition expression"); + }; + for (const auto& assignment: source.assignments) { + ir.SetGotoVariable(assignment.variable, expression(expression, assignment.expression)); } - if (source.terminator.kind == CFG::TerminatorKind::IndirectBranch) { - if (source.terminator.indirect_selector_code != UINT32_MAX) { - info.indirect_target = ReadScalarCode(source.terminator.indirect_selector_code); - } else if (source.terminator.indirect_pc_sgpr != UINT32_MAX) { - info.indirect_target = - ir.GetScalarReg(static_cast(source.terminator.indirect_pc_sgpr)); + const auto& term = source.terminator; + if (term.kind == CFG::TerminatorKind::IndirectBranch) { + if (term.indirect_selector_code != UINT32_MAX) { + info.indirect_target = ReadScalarCode(term.indirect_selector_code); + } else if (term.indirect_pc_sgpr != UINT32_MAX) { + info.indirect_target = ir.GetScalarReg(static_cast(term.indirect_pc_sgpr)); } else { EXIT("block %u has no indirect branch selector", source.id); } ir.Emit(IR::ValueOpcode::ReferenceU32, {info.indirect_target}); return; } - if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) { - return; + if (term.kind == CFG::TerminatorKind::ConditionalBranch) { + const auto condition = term.expression != UINT32_MAX + ? expression(expression, term.expression) + : native_condition(term.condition); + info.condition = condition; + ir.Emit(IR::ValueOpcode::Reference, {condition}); } - // EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive - // invocations leave the region without reconstructing a host-subgroup mask. - IR::U1 condition; - switch (source.terminator.condition) { - case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break; - case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break; - case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break; - case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break; - case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break; - case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break; - case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break; - case CFG::BranchCondition::ScalarInstruction: - EXIT_IF(instruction_branch_condition.IsEmpty()); - condition = instruction_branch_condition; - break; - case CFG::BranchCondition::GotoVariable: - if (source.terminator.goto_variable == UINT32_MAX) { - EXIT("block %u reads an invalid goto variable", source.id); - } - condition = ir.GetGotoVariable(source.terminator.goto_variable); - break; - case CFG::BranchCondition::Unknown: - EXIT("block %u has an unknown branch condition", source.id); - } - info.condition = condition; - ir.Emit(IR::ValueOpcode::Reference, {condition}); } namespace { @@ -1272,9 +1281,11 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& builtin(IR::StageInputKind::InstanceIndex)); } } + const bool flush_f32_inputs = options.stage == ShaderType::Compute && + (options.input_info.compute->float_mode & 0x10u) == 0; for (const auto& cfg_block: cfg.blocks) { const auto typed_index = block_indices.at(cfg_block.id); - Translator translator(result, result.blocks[typed_index], vector_limit); + Translator translator(result, result.blocks[typed_index], vector_limit, flush_f32_inputs); for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) { const auto& instruction = decoded.instructions[index]; if (IsCodeTableLoad(cfg, instruction.pc)) { @@ -1297,7 +1308,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& } translator.TranslateInstruction(instruction); } - translator.AddBranchCondition(cfg_block, result.block_info[typed_index]); + translator.AddBranchCondition(cfg, cfg_block, result.block_info[typed_index]); } IR::ValidateProgram(result, false); return result; diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 88afad485..76a1ddc80 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -9,13 +9,15 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend { class Translator { public: - Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit) - : program(program), ir(block), current_vector_limit(vector_limit) {} + Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit, + bool flush_f32_inputs) + : program(program), ir(block), current_vector_limit(vector_limit), + flush_f32_inputs(flush_f32_inputs) {} void TranslateInstruction(const Decoder::Instruction& inst); void TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute, uint32_t component_count, const ShaderBufferResource& resource); - void AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info); + void AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source, IR::BlockInfo& info); private: const Decoder::Operand& SourceAt(const Decoder::Instruction& inst, uint32_t index); @@ -261,6 +263,7 @@ private: Decoder::Opcode current_opcode = Decoder::Opcode::UNKNOWN; uint32_t current_pc = 0; uint32_t current_vector_limit = 1; + bool flush_f32_inputs; }; } // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/ir/Program.cpp b/src/graphics/shader/recompiler/ir/Program.cpp index 1fe47e8f5..51ce341f0 100644 --- a/src/graphics/shader/recompiler/ir/Program.cpp +++ b/src/graphics/shader/recompiler/ir/Program.cpp @@ -81,7 +81,8 @@ bool EquivalentValue(const ResourcePlan& program, Value left, Value right, program.memory_info[li] != program.memory_info[ri]) { return false; } - } else if (lhs->Flags() != rhs->Flags()) { + } else if (lhs->GetOpcode() != ValueOpcode::ReadConst && + lhs->Flags() != rhs->Flags()) { return false; } for (size_t index = 0; index < lhs->NumArgs(); index++) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index e9827605b..65dd66328 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -147,6 +147,7 @@ struct SamplerResource { uint32_t first_use_pc = 0; bool force_point_filtering = false; bool depth_compare = false; + bool integer_border = false; bool operator==(const SamplerResource& other) const = default; }; @@ -582,10 +583,11 @@ struct Program: ResourcePlan { CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None; std::string fallback_reason; std::vector block_info; + struct ScalarWrite { uint32_t pc; ScalarReg reg; }; + std::vector scalar_writes; // Typed memory and export instructions reference shader-local metadata by dense index. // Decoder-only details (such as NSA register numbers) have already become IR operands. std::vector export_info; - std::vector dynamic_reads; bool shader_info_complete = false; BindingLayout bindings; bool binding_layout_complete = false; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h index c5da6bef9..b853d99ba 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h @@ -52,6 +52,10 @@ struct PermlaneFlags { static_assert(sizeof(PermlaneFlags) <= sizeof(uint64_t)); static_assert(std::is_trivially_copyable_v); +struct FPCompareFlags { + bool flush_input_denorms = false; +}; + struct MemoryFlags { uint32_t index = 0; uint32_t pc = 0; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 7a903b492..768679402 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -4,6 +4,7 @@ VALUE_OPCODE(Identity, Opaque, Opaque) VALUE_OPCODE(Void, Void) VALUE_OPCODE(Reference, Void, U1) VALUE_OPCODE(ReferenceU32, Void, U32) +VALUE_OPCODE(ConditionRef, U1, U1) // Translation and register-state pseudo operations. VALUE_OPCODE(GetUserData, U32, ScalarReg) @@ -274,8 +275,8 @@ VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1) -VALUE_OPCODE(DataAppend, U32, U32, U1, U32, U32) -VALUE_OPCODE(DataConsume, U32, U32, U1, U32, U32) +VALUE_OPCODE(DataAppend, U32, U32, U1) +VALUE_OPCODE(DataConsume, U32, U32, U1) VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1) VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress) diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp index 022591569..70ddbf4cd 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp @@ -85,6 +85,16 @@ bool CollectMemoryResources(const Program& program, std::vector& buffe return uses_gds; } +bool UsesFlattenedSrt(const Program& program) { + return std::ranges::any_of(program.blocks, [](const Block* block) { + return std::ranges::any_of(*block, [](const Inst& inst) { + return inst.GetOpcode() == ValueOpcode::ReadConst; + }); + }) || std::ranges::any_of(program.info.images, [](const ImageResource& image) { + return image.indirect_search_iterations != 0u; + }); +} + void AllocateBindings(Program& program, uint32_t push_data_start_dword) { if (!program.shader_info_complete || program.binding_layout_complete) { EXIT("shader binding layout failed: %s", !program.shader_info_complete @@ -145,12 +155,7 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword) { AddBinding(next, DescriptorBindingKind::BdaPagetable); AddBinding(next, DescriptorBindingKind::FaultBuffer); } - const bool uses_flattened_runtime = - !program.srt_reads.empty() || - std::ranges::any_of(program.info.images, [](const ImageResource& image) { - return image.indirect_search_iterations != 0u; - }); - if (uses_flattened_runtime) { + if (UsesFlattenedSrt(program)) { AddBinding(next, DescriptorBindingKind::FlattenedSrt); } diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.h b/src/graphics/shader/recompiler/ir/passes/BindingLayout.h index 3ccdf33a0..c7ca51b9f 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.h +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.h @@ -9,6 +9,7 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword = 0); // Collect live direct buffer indices and report whether the shader accesses GDS. bool CollectMemoryResources(const Program& program, std::vector& buffers); +bool UsesFlattenedSrt(const Program& program); const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind); diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index 011ab562c..f4952e470 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -630,6 +630,11 @@ void FoldInstruction(Block& block, Block::iterator instruction, } } return; + case ValueOpcode::ConditionRef: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U1)) Replace(inst, value); + return; + } case ValueOpcode::LogicalNot: { const auto value = Arg(inst, 0); if (IsImmediate(value, Type::U1)) { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index b6590f644..57b30d2f4 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -107,14 +107,16 @@ Prospero::BufferFormat ImageConversionFormat(Prospero::BufferFormat format) { : Prospero::BufferFormat::kInvalid; } -bool RequiresPointSampler(const ImageResource& image) { - return image.numeric_class == Prospero::TextureNumericClass::Sint || - image.conversion_format != Prospero::BufferFormat::kInvalid; -} +enum class SamplerClass : uint8_t { Float, Integer, PointInteger }; -bool RequiresPointSampler(const ResourceSpecialization::Image& image) { - return image.numeric_class == Prospero::TextureNumericClass::Sint || - image.conversion_format != Prospero::BufferFormat::kInvalid; +template +SamplerClass ClassifySampler(const Image& image) { + if (image.numeric_class == Prospero::TextureNumericClass::Sint || + image.conversion_format != Prospero::BufferFormat::kInvalid) { + return SamplerClass::PointInteger; + } + return image.numeric_class == Prospero::TextureNumericClass::Uint ? SamplerClass::Integer + : SamplerClass::Float; } bool DescriptorIsCube(const DescriptorValue& descriptor) { @@ -223,9 +225,9 @@ void MarkCleanFlatSlots(const ResourcePlan& program, const DescriptorSource* sou } } -bool ReadScalarTable(uint64_t base, uint64_t size, uint32_t dynamic_offset, +bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset, const SrtRuntime& runtime, std::span words) { - const auto offset = static_cast(dynamic_offset) & ~uint64_t {3}; + const auto offset = dynamic_offset & ~uint64_t {3}; const auto count = std::min(words.size(), offset < size ? (size - offset) / 4u : 0u); std::ranges::fill(words.subspan(count), 0u); if (count == 0u) { @@ -304,26 +306,24 @@ bool MaterializeIndirectImage(const ResourcePlan& program, material.Stride() != indirect.selector_stride) { return false; } - // Enumerate every wrapped scalar-buffer offset that can pass the descriptor bounds. - const auto step = std::gcd(indirect.selector_stride, uint64_t {1} << 32u); - const auto residue = static_cast(indirect.selector_offset) % step; - const auto limit = std::min(UINT32_MAX, material.GetSize() + 3u); - const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u; + // The first aligned offset includes the immediate added after shader U32 arithmetic. + const auto step = std::max(4u, + std::gcd(indirect.selector_stride, uint64_t {1} << 32u)); + const uint64_t first = indirect.selector_offset; + const auto size = material.GetSize(); + const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u); + const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u; if (probe_count > MaxIndirectImageProbes) { return false; } keys.reserve(static_cast(probe_count) + 1u); keys.push_back(0u); - for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) { + for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) { uint32_t key = 0; - if (!ReadScalarTable(material.Base48(), material.GetSize(), - static_cast(offset), runtime, {&key, 1})) { + if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) { return false; } keys.push_back(key); - if (limit - offset < step) { - break; - } } std::ranges::sort(keys); keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); @@ -380,8 +380,13 @@ bool MaterializeIndirectImage(const ResourcePlan& program, } // namespace struct SamplerPlan { - std::array point_sampler {}; - uint32_t sampler_count = 0; + struct Binding { + uint32_t source; + SamplerClass type; + }; + std::array, ShaderInfo::MaxSamplers> mapping; + std::array bindings; + uint32_t sampler_count = 0; }; struct ImageRemap { @@ -595,11 +600,9 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna if (!BuildSamplerPlan(program.info, specialization.images, sampler_plan)) { return SpecializationFail("specialized sampler layout exceeds its resource limit"); } - for (uint32_t index = 0; index < program.info.samplers.size(); index++) { - const auto target = sampler_plan.point_sampler[index]; - if (target != UINT32_MAX && target >= program.info.samplers.size()) { - snapshot.samplers.push_back(snapshot.samplers[index]); - } + for (uint32_t index = static_cast(program.info.samplers.size()); + index < sampler_plan.sampler_count; index++) { + snapshot.samplers.push_back(snapshot.samplers[sampler_plan.bindings[index].source]); } ImageRemap(specialization).Apply(snapshot.images); return true; @@ -607,29 +610,29 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna template bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan) { - if (base.samplers.size() > plan.point_sampler.size()) { + if (base.samplers.size() > plan.mapping.size()) { return false; } std::array usage {}; - plan.point_sampler.fill(UINT32_MAX); plan.sampler_count = static_cast(base.samplers.size()); for (const auto& pair: base.sampled_pairs) { if (pair.image >= images.size() || pair.sampler >= base.samplers.size()) { return false; } - usage[pair.sampler] |= RequiresPointSampler(images[pair.image]) ? 2u : 1u; + usage[pair.sampler] |= 1u << static_cast(ClassifySampler(images[pair.image])); } for (uint32_t index = 0; index < base.samplers.size(); index++) { - if ((usage[index] & 2u) == 0u) { - continue; - } - if ((usage[index] & 1u) == 0u) { - plan.point_sampler[index] = index; - } else { - if (plan.sampler_count >= ShaderInfo::MaxSamplers) { - return false; - } - plan.point_sampler[index] = plan.sampler_count++; + auto& mapping = plan.mapping[index]; + mapping.fill(UINT32_MAX); + const auto classes = usage[index] == 0u ? 1u : usage[index]; + bool first = true; + for (uint32_t type = 0; type < mapping.size(); type++) { + if ((classes & (1u << type)) == 0u) continue; + const auto target = first ? index : plan.sampler_count++; + if (target >= ShaderInfo::MaxSamplers) return false; + mapping[type] = target; + plan.bindings[target] = {index, static_cast(type)}; + first = false; } } return true; @@ -1105,25 +1108,18 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& auto samplers = program.info.samplers; auto sampled_pairs = program.info.sampled_pairs; samplers.reserve(sampler_plan.sampler_count); - for (uint32_t index = 0; index < program.info.samplers.size(); index++) { - const auto target = sampler_plan.point_sampler[index]; - if (target == UINT32_MAX) { - continue; - } - if (target == index) { - samplers[index].force_point_filtering = true; - } else { - EXIT_IF(target != samplers.size()); - auto sampler = samplers[index]; - sampler.force_point_filtering = true; - samplers.push_back(std::move(sampler)); + for (uint32_t index = 0; index < sampler_plan.sampler_count; index++) { + const auto& binding = sampler_plan.bindings[index]; + if (index >= program.info.samplers.size()) { + samplers.push_back(program.info.samplers[binding.source]); } + samplers[index].force_point_filtering = binding.type == SamplerClass::PointInteger; + samplers[index].integer_border = binding.type != SamplerClass::Float; } for (auto& pair: sampled_pairs) { - if (RequiresPointSampler(images[pair.image])) { - EXIT_IF(sampler_plan.point_sampler[pair.sampler] == UINT32_MAX); - pair.sampler = sampler_plan.point_sampler[pair.sampler]; - } + const auto type = static_cast(ClassifySampler(images[pair.image])); + pair.sampler = sampler_plan.mapping[pair.sampler][type]; + EXIT_IF(pair.sampler == UINT32_MAX); samplers[pair.sampler].depth_compare |= images[pair.image].depth_compare; } @@ -1201,10 +1197,11 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& inst.ReplaceUsesWith(Value(&*result)); continue; } - if (image_opcode.needs_sampler && RequiresPointSampler(image) && + if (image_opcode.needs_sampler && memory.sampler < program.info.samplers.size()) { - EXIT_IF(sampler_plan.point_sampler[memory.sampler] == UINT32_MAX); - memory.sampler = sampler_plan.point_sampler[memory.sampler]; + const auto type = static_cast(ClassifySampler(image)); + memory.sampler = sampler_plan.mapping[memory.sampler][type]; + EXIT_IF(memory.sampler == UINT32_MAX); } EXIT_IF(image.indirect_root == memory.resource && inst.GetOpcode() != ValueOpcode::ImageSampleRaw); diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 809b16050..ce795034e 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -2,11 +2,16 @@ #include "common/assert.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" +#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include #include +#include +#include +#include #include +#include #include namespace Libs::Graphics::ShaderRecompiler::IR { @@ -80,9 +85,175 @@ uint32_t ByteExtent(const MemoryInfo& memory) { return end > UINT32_MAX ? UINT32_MAX : static_cast(end); } +// Prove a loop cannot continue after its bound fails, both on entry and after an +// arbitrary previous iteration. Header Phis are substituted simultaneously; all +// unsupported expressions remain unconstrained. This state exists only while tracking. +class LoopBoundProof { +public: + LoopBoundProof(const Program& program, const Inst& induction, const Inst& bound) + : m_program(program), m_induction(induction), m_bound(bound) {} + + bool Excludes(Value condition, bool positive) { + for (uint32_t incoming = 0; incoming < m_induction.NumArgs(); ++incoming) { + m_incoming = m_induction.PhiBlock(incoming); + m_values[1].clear(); + if (Evaluate(condition, true) != (positive ? 0u : 1u)) return false; + } + return true; + } + +private: + struct Node { + uint32_t variable = UINT32_MAX; + uint32_t low = 0; + uint32_t high = 0; + }; + + uint32_t NodeFor(uint32_t variable, uint32_t low, uint32_t high) { + if (low == high) return low; + const auto [it, inserted] = m_nodes_by_key.try_emplace( + std::array {variable, low, high}, static_cast(m_nodes.size())); + if (inserted) m_nodes.push_back({variable, low, high}); + return it->second; + } + + uint32_t Unknown(Type type) { + if (type == Type::U1) return NodeFor(m_variables++, 0u, 1u); + m_nodes.emplace_back(); + return static_cast(m_nodes.size() - 1u); + } + + uint32_t Select(uint32_t condition, uint32_t yes, uint32_t no) { + if (condition == 0u) return no; + if (condition == 1u || yes == no) return yes; + if (yes == 1u && no == 0u) return condition; + const std::array key {condition, yes, no}; + if (const auto found = m_choices.find(key); found != m_choices.end()) return found->second; + const auto variable = std::min({m_nodes[condition].variable, m_nodes[yes].variable, + m_nodes[no].variable}); + const auto arm = [&](uint32_t value, bool high) { + const auto node = m_nodes[value]; + return node.variable == variable ? (high ? node.high : node.low) : value; + }; + const auto low = Select(arm(condition, false), arm(yes, false), arm(no, false)); + const auto high = Select(arm(condition, true), arm(yes, true), arm(no, true)); + const auto result = NodeFor(variable, low, high); + m_choices.emplace(key, result); + return result; + } + + uint32_t Compare(const Inst& inst, uint32_t left, uint32_t right) { + const auto key = std::tuple {inst.GetOpcode(), inst.Flags(), left, right}; + if (const auto found = m_predicates.find(key); found != m_predicates.end()) return found->second; + const auto variable = std::min(m_nodes[left].variable, m_nodes[right].variable); + uint32_t result; + if (variable == UINT32_MAX) { + result = Unknown(Type::U1); + } else { + const auto lhs = m_nodes[left]; + const auto rhs = m_nodes[right]; + const auto low = Compare(inst, lhs.variable == variable ? lhs.low : left, + rhs.variable == variable ? rhs.low : right); + const auto high = Compare(inst, lhs.variable == variable ? lhs.high : left, + rhs.variable == variable ? rhs.high : right); + result = Select(NodeFor(variable, 0u, 1u), high, low); + } + m_predicates.emplace(key, result); + return result; + } + + uint32_t Evaluate(Value value, bool current) { + value = value.Resolve(); + if (value.IsImmediate()) { + if (value.GetType() == Type::U1) return value.U1() ? 1u : 0u; + for (const auto& [literal, node]: m_literals) { + if (literal == value) return node; + } + const auto node = Unknown(value.GetType()); + m_literals.emplace_back(value, node); + return node; + } + const auto* inst = value.TryInstruction(); + if (current && inst == &m_bound) return 0u; + auto& values = m_values[current]; + if (const auto found = values.find(inst); found != values.end()) { + if (found->second == UINT32_MAX) found->second = Unknown(value.GetType()); + return found->second; + } + const auto cached = values.emplace(inst, UINT32_MAX).first; + auto result = UINT32_MAX; + const auto arg = [&](uint32_t index) { return Evaluate(inst->Arg(index), current); }; + switch (inst->GetOpcode()) { + case ValueOpcode::Phi: { + const auto invariant = ResolveInvariantPhi(m_program, value); + if (!invariant.IsEmpty()) { + result = Evaluate(invariant, current); + } else if (inst->Parent() == m_induction.Parent()) { + if (current) { + for (uint32_t i = 0; i < inst->NumArgs(); ++i) { + if (inst->PhiBlock(i) == m_incoming) result = Evaluate(inst->Arg(i), false); + } + } + } else if (inst->NumArgs() != 0u) { + result = arg(0); + for (uint32_t i = 1; i < inst->NumArgs(); ++i) { + if (arg(i) != result) { result = UINT32_MAX; break; } + } + } + break; + } + case ValueOpcode::LogicalNot: result = Select(arg(0), 0u, 1u); break; + case ValueOpcode::LogicalAnd: { + const auto left = arg(0); + result = left == 0u ? 0u : Select(left, arg(1), 0u); + break; + } + case ValueOpcode::LogicalOr: { + const auto left = arg(0); + result = left == 1u ? 1u : Select(left, 1u, arg(1)); + break; + } + case ValueOpcode::SelectU1: + case ValueOpcode::SelectU32: { + const auto condition = arg(0); + result = condition == 0u ? arg(2) : condition == 1u ? arg(1) + : Select(condition, arg(1), arg(2)); + break; + } + default: + if (inst->GetType() == Type::U1 && inst->NumArgs() == 2u && + inst->Arg(0).GetType() == Type::U32 && inst->Arg(1).GetType() == Type::U32) + result = Compare(*inst, arg(0), arg(1)); + break; + } + // Only unsupported values and cycles need free variables. + if (result == UINT32_MAX) { + if (cached->second == UINT32_MAX) cached->second = Unknown(value.GetType()); + return cached->second; + } + cached->second = result; + return result; + } + + const Program& m_program; + const Inst& m_induction; + const Inst& m_bound; + const Block* m_incoming = nullptr; + uint32_t m_variables = 0; + std::vector m_nodes {{}, {}}; + std::vector> m_literals; + std::array, 2> m_values; + std::map, uint32_t> m_nodes_by_key; + std::map, uint32_t> m_choices; + std::map, uint32_t> m_predicates; +}; + class Tracker { public: - explicit Tracker(Program& program): m_program(program), m_info(program.info) { + Tracker(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg) + : m_program(program), m_decoded(decoded), m_native_cfg(native_cfg), + m_scalar_writes(std::move(program.scalar_writes)), m_info(program.info) { + std::ranges::sort(m_scalar_writes, {}, &Program::ScalarWrite::pc); m_info.buffers.clear(); m_info.images.clear(); m_info.samplers.clear(); @@ -95,9 +266,8 @@ public: if (m_program.resource_tracking_complete) { Fail(0, "resources already tracked"); } - if (!m_program.srt_plan_complete) { - Fail(0, "SRT plan is not ready"); - } + PlanScalarReads(); + EliminateDeadCode(m_program.blocks); PlanIndirectImages(); for (auto* block: m_program.blocks) { for (auto& inst: *block) { @@ -127,13 +297,6 @@ public: m_program.memory_info[index].planning_only = true; } } - std::erase_if(m_program.dynamic_reads, [&](Value value) { - const auto* inst = value.Resolve().TryInstruction(); - return std::any_of(m_indirect_images.begin(), m_indirect_images.end(), - [&](const IndirectImagePlan& plan) { - return std::ranges::find(plan.reads, inst) != plan.reads.end(); - }); - }); m_program.descriptor_sources = std::move(m_sources); m_program.info = std::move(m_info); m_program.resource_tracking_complete = true; @@ -152,6 +315,13 @@ private: bool has_sampler = false; }; + struct ResolvedHandle { + const Inst* handle; + uint32_t pc; + DescriptorSource source; + Value planning_handle; + }; + struct IndirectImagePlan { Inst* handle = nullptr; uint32_t source = 0; @@ -169,7 +339,109 @@ private: std::abort(); } - Value LowerDescriptorPhi(Value value, const Block* use) { + Value NativeDescriptorSource(Value value, uint32_t reg, uint32_t use_pc) const { + value = value.Resolve(); + const auto* phi = value.TryInstruction(); + if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi || m_native_cfg.blocks.empty()) + return value; + std::vector candidates; + std::vector visited; + std::vector pending {phi}; + while (!pending.empty()) { + const auto* inst = pending.back(); + pending.pop_back(); + if (std::ranges::find(visited, inst) != visited.end()) continue; + visited.push_back(inst); + if (inst->GetOpcode() == ValueOpcode::Phi) { + for (size_t i = 0; i < inst->NumArgs(); ++i) { + const auto* arg = inst->Arg(i).Resolve().TryInstruction(); + if (arg != nullptr) pending.push_back(arg); + } + } else if (inst->GetOpcode() == ValueOpcode::ReadConst || + inst->GetOpcode() == ValueOpcode::LoadAddressU32 || + inst->GetOpcode() == ValueOpcode::ReadConstBuffer || + inst->GetOpcode() == ValueOpcode::GetUserData) { + candidates.push_back(inst); + } + } + if (candidates.empty()) return value; + const auto source_at = [&](uint32_t pc) { + Value source; + const auto native = std::ranges::lower_bound(m_decoded.instructions, pc, {}, + &Decoder::Instruction::pc); + for (const auto* candidate: candidates) { + if (pc == UINT32_MAX) { + if (candidate->GetOpcode() != ValueOpcode::GetUserData || + RegIndex(candidate->Arg(0).ScalarRegister()) != reg) + continue; + } else { + if (candidate->GetOpcode() != ValueOpcode::ReadConst && + candidate->GetOpcode() != ValueOpcode::LoadAddressU32 && + candidate->GetOpcode() != ValueOpcode::ReadConstBuffer) continue; + const auto flags = candidate->Flags(); + if (flags.pc != pc || flags.index >= m_program.memory_info.size()) continue; + if (native == m_decoded.instructions.end() || native->pc != pc || + native->dst.kind != Decoder::OperandKind::Sgpr || + native->dst.reg + m_program.memory_info[flags.index].component_index != reg) + continue; + } + const Value current(const_cast(candidate)); + if (!source.IsEmpty() && !EquivalentValue(m_program, source, current)) return Value {}; + source = current; + } + return source; + }; + const auto use = std::ranges::find_if(m_native_cfg.blocks, [&](const auto& block) { + return block.start_pc <= use_pc && use_pc < block.end_pc; + }); + if (use == m_native_cfg.blocks.end()) return value; + struct Position { uint32_t block; uint32_t before; }; + std::vector positions {{use->id, use_pc}}; + std::vector reached(m_native_cfg.blocks.size()); + Value selected; + uint32_t selected_pc = UINT32_MAX; + const auto select = [&](uint32_t pc) { + const auto source = source_at(pc); + if (source.IsEmpty()) return false; + if (!selected.IsEmpty()) { + const auto op = source.TryInstruction()->GetOpcode(); + if ((op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) && + selected_pc != pc) return false; + if (!EquivalentValue(m_program, selected, source)) return false; + } + selected = source; + selected_pc = pc; + return true; + }; + while (!positions.empty()) { + const auto position = positions.back(); + positions.pop_back(); + const auto& block = m_native_cfg.blocks[position.block]; + // A backedge may revisit the use block after a write later than the original use. + if (position.before == block.end_pc) { + if (reached[block.id]) continue; + reached[block.id] = true; + } + auto write = std::ranges::lower_bound(m_scalar_writes, position.before, {}, + &Program::ScalarWrite::pc); + bool found = false; + while (write != m_scalar_writes.begin()) { + --write; + if (write->pc < block.start_pc) break; + if (RegIndex(write->reg) != reg) continue; + if (!select(write->pc)) return value; + found = true; + break; + } + if (found) continue; + if (block.id == m_native_cfg.entry_block && !select(UINT32_MAX)) return value; + for (const auto pred: block.predecessors) + positions.push_back({pred, m_native_cfg.blocks[pred].end_pc}); + } + return selected.IsEmpty() ? value : selected; + } + + Value LowerDescriptorPhi(Value value) { value = value.Resolve(); const auto* phi = value.TryInstruction(); if (m_shader_writes || phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi || @@ -183,42 +455,6 @@ private: branch == phi->PhiBlock(1)) { return value; } - // Structurization can merge the descriptor and its use predicate in parallel Phis. - // Match their incoming blocks to exclude only edges that cannot reach this use. - if (use != nullptr && use->ImmPredecessors().size() == 1u && - use->ImmPredecessors()[0] == merge) { - const auto merge_it = std::ranges::find(m_program.blocks, merge); - const auto use_it = std::ranges::find(m_program.blocks, use); - if (merge_it != m_program.blocks.end() && use_it != m_program.blocks.end()) { - const auto& info = m_program.block_info[merge_it - m_program.blocks.begin()]; - const auto& term = info.terminator; - const auto id = m_program.block_info[use_it - m_program.blocks.begin()].id; - const auto* condition = info.condition.Resolve().TryInstruction(); - if (term.kind == CFG::TerminatorKind::ConditionalBranch && - term.true_block != term.false_block && - (id == term.true_block || id == term.false_block) && condition != nullptr && - condition->GetOpcode() == ValueOpcode::Phi && condition->GetType() == Type::U1 && - condition->Parent() == merge && condition->NumArgs() == 2u && - condition->NumPhiBlocks() == 2u) { - const bool taken = id == term.true_block; - for (uint32_t skipped = 0; skipped < 2u; skipped++) { - const auto excluded = condition->Arg(skipped).Resolve(); - const auto included = condition->Arg(skipped ^ 1u).Resolve(); - if (!excluded.IsImmediate() || excluded.GetType() != Type::U1 || - excluded.U1() == taken || - (included.IsImmediate() && included.U1() != taken)) { - continue; - } - for (uint32_t selected = 0; selected < 2u; selected++) { - if (phi->PhiBlock(selected) == condition->PhiBlock(skipped ^ 1u) && - phi->PhiBlock(selected ^ 1u) == condition->PhiBlock(skipped)) { - return phi->Arg(selected); - } - } - } - } - } - } for (const auto& [original, selected]: m_descriptor_selections) { if (original == phi) { return selected; @@ -276,14 +512,23 @@ private: } void MakeSource(const Inst& handle, uint32_t width, bool sampler, bool sample_adjust, - DescriptorSource& descriptor, uint32_t pc) { + uint32_t base_reg, DescriptorSource& descriptor, uint32_t pc) { + const auto resolved = std::ranges::find_if(m_resolved_handles, [&](const auto& entry) { + return entry.handle == &handle && entry.pc == pc; + }); + if (resolved != m_resolved_handles.end()) { + descriptor = resolved->source; + return; + } if (handle.NumArgs() != width) { Fail(pc, fmt::format("{} has {} descriptor dwords, expected {}", ValueOpcodeName(handle.GetOpcode()), handle.NumArgs(), width)); } descriptor.dword_count = width; for (uint32_t i = 0; i < width; i++) { - descriptor.dwords[i] = LowerDescriptorPhi(handle.Arg(i), handle.Parent()); + const auto value = base_reg != UINT32_MAX + ? NativeDescriptorSource(handle.Arg(i), base_reg + i, pc) : handle.Arg(i); + descriptor.dwords[i] = LowerDescriptorPhi(value); } if (sample_adjust) { descriptor.dwords[3] = CanonicalizeSampleAdjustDword3(descriptor.dwords[3]); @@ -294,6 +539,174 @@ private: // Border color and its table index are unused unless a clamp axis selects border mode. descriptor.dwords[3] = Value(0u); } + m_resolved_handles.push_back({&handle, pc, descriptor, {}}); + } + + uint32_t ScalarReadBase(const Inst& read) const { + const auto flags = read.Flags(); + if (m_program.memory_info[flags.index].kind == ResourceKind::ScalarBuffer) + return m_program.memory_info[flags.index].resource * 4u; + const auto native = std::ranges::lower_bound(m_decoded.instructions, flags.pc, {}, + &Decoder::Instruction::pc); + return native != m_decoded.instructions.end() && native->pc == flags.pc && + native->src0.kind == Decoder::OperandKind::Sgpr + ? native->src0.reg : UINT32_MAX; + } + + void CollectScalarRead(Value value, uint32_t use_pc) { + value = value.Resolve(); + if (value.IsImmediate()) return; + auto* inst = value.TryInstruction(); + if (inst == nullptr) Fail(use_pc, "invalid typed planning value"); + const auto cycle = std::ranges::find(m_srt_visiting, inst); + if (cycle != m_srt_visiting.end()) { + if (std::any_of(cycle, m_srt_visiting.end(), [](const Inst* value) { + return value->GetOpcode() == ValueOpcode::Phi; + })) return; + Fail(use_pc, "cyclic typed planning value without a phi"); + } + if (std::ranges::find(m_srt_visited, inst) != m_srt_visited.end()) return; + m_srt_visiting.push_back(inst); + uint32_t memory_index = 0; + const auto* memory = ScalarReadMemory(*inst, memory_index); + DescriptorSource source; + if (memory != nullptr) { + const auto* handle = inst->Arg(0).Resolve().TryInstruction(); + const auto width = memory->kind == ResourceKind::ScalarBuffer ? 4u : 2u; + if (handle == nullptr || handle->GetOpcode() != + (width == 4u ? ValueOpcode::GetBufferResource : ValueOpcode::GetAddressResource)) + Fail(use_pc, "scalar read has an invalid resource handle"); + MakeSource(*handle, width, false, false, ScalarReadBase(*inst), source, + inst->Flags().pc); + for (uint32_t word = 0; word < width; ++word) + CollectScalarRead(source.dwords[word], inst->Flags().pc); + for (size_t arg = 1; arg < inst->NumArgs(); ++arg) + CollectScalarRead(inst->Arg(arg), use_pc); + } else { + for (size_t arg = 0; arg < inst->NumArgs(); ++arg) + CollectScalarRead(inst->Arg(arg), use_pc); + } + m_srt_visiting.pop_back(); + m_srt_visited.push_back(inst); + if (memory == nullptr) return; + const auto offset = inst->Arg(1).Resolve(); + if (!offset.IsImmediate() || offset.GetType() != Type::U32) return; + m_scalar_reads.push_back(inst); + } + + void PlanScalarReads() { + m_program.srt_plan_complete = false; + m_program.srt_reads.clear(); + for (auto* block: m_program.blocks) { + for (auto& inst: *block) { + const auto op = inst.GetOpcode(); + const auto image = ImageOpcodeInfoOf(op); + if (BufferAccessOf(op) == BufferAccess::None && + AddressOpcodeInfoOf(op).access == AddressAccess::None && + image.access == ImageAccess::None) continue; + const auto flags = inst.Flags(); + if (flags.index >= m_program.memory_info.size()) + Fail(flags.pc, "memory metadata index is out of range"); + if (inst.NumArgs() < (image.needs_sampler ? 2u : 1u)) + Fail(flags.pc, "memory operation has no resource handle"); + const auto& memory = m_program.memory_info[flags.index]; + if ((op == ValueOpcode::LoadAddressU32 && memory.kind == ResourceKind::ScalarBuffer) || + (op == ValueOpcode::ReadConstBuffer && memory.kind == ResourceKind::ScalarAddress)) + Fail(flags.pc, "scalar read has incompatible scalar memory metadata"); + for (uint32_t arg = 0; arg < (image.needs_sampler ? 2u : 1u); ++arg) { + const auto* handle = inst.Arg(arg).Resolve().TryInstruction(); + if (handle == nullptr) continue; + const auto kind = handle->GetOpcode(); + const bool sampler = kind == ValueOpcode::GetSamplerResource; + const uint32_t width = kind == ValueOpcode::GetImageResource ? 8u + : kind == ValueOpcode::GetAddressResource ? 2u + : kind == ValueOpcode::GetBufferResource || sampler ? 4u : 0u; + if (width == 0u) continue; + const auto base = width == 2u + ? (memory.kind == ResourceKind::ScalarAddress ? ScalarReadBase(inst) : UINT32_MAX) + : (sampler ? memory.sampler : memory.resource) * 4u; + DescriptorSource source; + MakeSource(*handle, width, sampler, + sampler && (memory.image_sample_flags & Decoder::ImageSampleFlagAdjust) != 0, + base, source, flags.pc); + for (uint32_t word = 0; word < width; ++word) + CollectScalarRead(source.dwords[word], flags.pc); + } + } + } + for (auto* block: m_program.blocks) { + for (auto& inst: *block) { + uint32_t index = 0; + if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && + ScalarReadMemory(inst, index) != nullptr && inst.Arg(1).Resolve().IsImmediate() && + ValidateRuntimeValue(m_program, Value(&inst))) + CollectScalarRead(Value(&inst), inst.Flags().pc); + } + } + for (auto* read: m_scalar_reads) { + const auto flags = read->Flags(); + auto& memory = m_program.memory_info[flags.index]; + memory.planning_only = true; + const auto* handle = read->Arg(0).Resolve().TryInstruction(); + auto resolved = std::ranges::find_if(m_resolved_handles, [&](const auto& entry) { + return entry.handle == handle && entry.pc == flags.pc; + }); + EXIT_IF(resolved == m_resolved_handles.end()); + uint32_t slot = 0; + for (; slot < m_program.srt_reads.size(); ++slot) { + const auto* other = m_program.srt_reads[slot].value.Resolve().TryInstruction(); + if (other->GetOpcode() != read->GetOpcode() || + m_program.memory_info[other->Flags().index] != memory) continue; + const auto* other_handle = other->Arg(0).Resolve().TryInstruction(); + bool same = true; + for (uint32_t word = 0; word < resolved->source.dword_count; ++word) + same &= EquivalentValue(m_program, resolved->source.dwords[word], other_handle->Arg(word)); + for (size_t arg = 1; arg < read->NumArgs(); ++arg) + same &= EquivalentValue(m_program, read->Arg(arg), other->Arg(arg)); + if (same) break; + } + const bool keep = slot == m_program.srt_reads.size(); + if (keep) m_program.srt_reads.push_back({Value(read), slot}); + auto* block = read->Parent(); + auto& list = block->Instructions(); + const auto where = std::ranges::find_if(list, [&](const Inst& inst) { + return &inst == read; + }); + const auto resource = Value(&*block->PrependNewInst(where, ValueOpcode::GetSrtResource)); + const auto flat = Value(&*block->PrependNewInst(where, ValueOpcode::ReadConst, + {resource, Value(slot)}, read->Flags())); + const auto uses = read->Uses(); + for (const auto& use: uses) use.user->SetArg(use.operand, flat); + for (auto& entry: m_resolved_handles) { + for (uint32_t word = 0; word < entry.source.dword_count; ++word) + if (entry.source.dwords[word].Resolve() == Value(read)) + entry.source.dwords[word] = flat; + } + for (auto& info: m_program.block_info) { + if (info.condition.Resolve() == Value(read)) info.condition = flat; + if (info.indirect_target.Resolve() == Value(read)) info.indirect_target = flat; + } + if (keep) { + if (resolved->planning_handle.IsEmpty()) { + bool unchanged = true; + for (uint32_t word = 0; word < resolved->source.dword_count; ++word) + unchanged &= handle->Arg(word).Resolve() == resolved->source.dwords[word].Resolve(); + resolved->planning_handle = read->Arg(0); + if (!unchanged) { + auto& retained = m_program.value_storage.emplace_back(handle->GetOpcode()); + for (uint32_t word = 0; word < resolved->source.dword_count; ++word) + retained.SetArg(word, resolved->source.dwords[word]); + resolved->planning_handle = Value(&retained); + } + } + read->SetArg(0, resolved->planning_handle); + read->SetParent(nullptr); + m_program.value_storage.splice(m_program.value_storage.end(), list, where); + } else { + list.erase(where); + } + } + m_program.srt_plan_complete = true; } bool ValidateSource(const DescriptorSource& descriptor, uint32_t& bad_dword) const { @@ -397,14 +810,19 @@ private: return true; } - bool MakeRuntimeTableSource(const Inst& handle, uint32_t pc, - DescriptorSource& descriptor) { - const auto width = handle.GetOpcode() == ValueOpcode::GetBufferResource ? 4u - : handle.GetOpcode() == ValueOpcode::GetAddressResource ? 2u : 0u; + bool MakeRuntimeTableSource(const Inst& read, DescriptorSource& descriptor) { + const auto* handle = read.Arg(0).Resolve().TryInstruction(); + if (handle == nullptr) return false; + const auto width = handle->GetOpcode() == ValueOpcode::GetBufferResource ? 4u + : handle->GetOpcode() == ValueOpcode::GetAddressResource ? 2u : 0u; if (width == 0u) { return false; } - MakeSource(handle, width, false, false, descriptor, pc); + const auto flags = read.Flags(); + const auto kind = m_program.memory_info[flags.index].kind; + const auto base = kind == ResourceKind::ScalarAddress || kind == ResourceKind::ScalarBuffer + ? ScalarReadBase(read) : UINT32_MAX; + MakeSource(*handle, width, false, false, base, descriptor, flags.pc); uint32_t bad_dword = 0; return ValidateSource(descriptor, bad_dword); } @@ -443,6 +861,13 @@ private: selector_inst->GetOpcode() == ValueOpcode::ReadFirstLane; } + enum class LaneQuantifier { Any, All }; + struct EdgePredicate { + Value condition; + bool positive; + LaneQuantifier lanes = LaneQuantifier::All; + }; + bool NonzeroOnEntry(Value value, const Block* block) const { if (m_program.blocks.size() != m_program.block_info.size()) { return false; @@ -454,25 +879,12 @@ private: return false; } const auto* previous = block->ImmPredecessors()[0]; - const auto current_it = std::ranges::find(m_program.blocks, block); - const auto previous_it = std::ranges::find(m_program.blocks, previous); - if (current_it == m_program.blocks.end() || previous_it == m_program.blocks.end()) { - return false; - } - const auto& info = m_program.block_info[previous_it - m_program.blocks.begin()]; - const auto id = m_program.block_info[current_it - m_program.blocks.begin()].id; - const auto& term = info.terminator; - if (term.kind == CFG::TerminatorKind::ConditionalBranch && - (term.true_block == id) != (term.false_block == id)) { - bool positive = term.true_block == id; - const auto* test = info.condition.Resolve().TryInstruction(); - while (test != nullptr && test->GetOpcode() == ValueOpcode::LogicalNot) { - positive = !positive; - test = test->Arg(0).Resolve().TryInstruction(); - } + const auto edge = ConditionalEdge(previous, block); + if (edge && edge->lanes == LaneQuantifier::All) { + const auto* test = edge->condition.TryInstruction(); if (test != nullptr && test->NumArgs() == 2u && - ((test->GetOpcode() == ValueOpcode::INotEqual32 && positive) || - (test->GetOpcode() == ValueOpcode::IEqual32 && !positive))) { + ((test->GetOpcode() == ValueOpcode::INotEqual32 && edge->positive) || + (test->GetOpcode() == ValueOpcode::IEqual32 && !edge->positive))) { for (uint32_t arg = 0; arg < 2u; ++arg) { uint32_t immediate; if (ImmediateU32(test->Arg(arg), immediate) && immediate == 0u && @@ -515,55 +927,71 @@ private: } } - const Block* FindBlock(uint32_t id) const { - const auto info = std::ranges::find(m_program.block_info, id, &BlockInfo::id); - return info == m_program.block_info.end() - ? nullptr - : m_program.blocks[info - m_program.block_info.begin()]; - } - - bool CanReach(const Block* start, const Block* target, const Block* avoid) const { - std::vector pending {start}; - std::vector visited; - while (!pending.empty()) { - const auto* block = pending.back(); - pending.pop_back(); - if (block == nullptr || block == avoid || - std::ranges::find(visited, block) != visited.end()) continue; - if (block == target) return true; - visited.push_back(block); - for (const auto* next: block->ImmSuccessors()) pending.push_back(next); - } - return false; - } - - bool ConditionalEdge(const Block* from, const Block* to, Value& condition, - bool& positive) const { + std::optional ConditionalEdge(const Block* from, const Block* to) const { const auto position = std::ranges::find(m_program.blocks, from); const auto target = std::ranges::find(m_program.blocks, to); - if (position == m_program.blocks.end() || target == m_program.blocks.end()) return false; + if (position == m_program.blocks.end() || target == m_program.blocks.end()) return {}; const auto& info = m_program.block_info[position - m_program.blocks.begin()]; const auto& term = info.terminator; const auto id = m_program.block_info[target - m_program.blocks.begin()].id; if (term.kind != CFG::TerminatorKind::ConditionalBranch || - (term.true_block == id) == (term.false_block == id)) return false; - condition = info.condition; - positive = term.true_block == id; - while (const auto* inst = condition.Resolve().TryInstruction()) { - if (inst->GetOpcode() != ValueOpcode::LogicalNot) break; - condition = inst->Arg(0); - positive = !positive; + (term.true_block == id) == (term.false_block == id)) return {}; + EdgePredicate edge {info.condition, term.true_block == id}; + while (const auto* inst = edge.condition.Resolve().TryInstruction()) { + if (inst->GetOpcode() == ValueOpcode::LogicalNot) { + edge.positive = !edge.positive; + } else if (inst->GetOpcode() == ValueOpcode::ConditionRef) { + const auto kind = inst->Flags(); + const bool scalar = kind == CFG::BranchCondition::SccZero || + kind == CFG::BranchCondition::SccNonZero; + const bool zero = kind == CFG::BranchCondition::ExecZero || + kind == CFG::BranchCondition::VccZero; + const bool nonzero = kind == CFG::BranchCondition::ExecNonZero || + kind == CFG::BranchCondition::VccNonZero; + if (!scalar && !zero && !nonzero) break; + // SCC is uniform. Negating a lane reduction exchanges all and any. + edge.lanes = scalar || (zero == edge.positive) ? LaneQuantifier::All + : LaneQuantifier::Any; + } else { + break; + } + edge.condition = inst->Arg(0); } - condition = condition.Resolve(); - return true; + edge.condition = edge.condition.Resolve(); + return edge; } - Value PositiveUseGuard(const Block* use) const { + Value PositiveLaneWitness(const Block* use) const { if (use == nullptr || use->ImmPredecessors().size() != 1u) return {}; - Value condition; - bool positive = false; - return ConditionalEdge(use->ImmPredecessors()[0], use, condition, positive) && positive - ? condition : Value {}; + const auto edge = ConditionalEdge(use->ImmPredecessors()[0], use); + return edge && edge->positive ? edge->condition : Value {}; + } + + bool HasActiveLane(Value mask, const Block* use) const { + mask = mask.Resolve(); + const auto incoming_is_nonempty = [&](const Block* from, const Block* to, + Value incoming, const Block* header) { + for (size_t depth = 0; depth < m_program.blocks.size(); ++depth) { + const auto edge = ConditionalEdge(from, to); + if (edge && edge->positive && Implies(edge->condition, incoming)) return true; + if (from == header || from->ImmSuccessors().size() != 1u || + from->ImmPredecessors().size() != 1u) return false; + to = from; + from = from->ImmPredecessors()[0]; + } + return false; + }; + const auto* phi = mask.TryInstruction(); + if (phi != nullptr && phi->GetOpcode() == ValueOpcode::Phi && + phi->GetType() == Type::U1 && phi->NumArgs() != 0u) { + for (size_t arm = 0; arm < phi->NumArgs(); ++arm) { + if (!incoming_is_nonempty(phi->PhiBlock(arm), phi->Parent(), + phi->Arg(arm), phi->Parent())) return false; + } + return true; + } + return use != nullptr && use->ImmPredecessors().size() == 1u && + incoming_is_nonempty(use->ImmPredecessors()[0], use, mask, nullptr); } Value SimplifyGuard(Value guard) const { @@ -673,10 +1101,14 @@ private: EquivalentValue(m_program, inst->Arg(1), index)); } - bool ClearsFirstSetBit(Value value, Value mask) const { + bool MaskOnlyLosesBits(Value value, Value mask) const { const auto* update = value.Resolve().TryInstruction(); - if (update == nullptr || update->GetOpcode() != ValueOpcode::BitwiseXor32 || - update->NumArgs() != 2u) return false; + if (update == nullptr || update->NumArgs() != 2u) return false; + if (update->GetOpcode() == ValueOpcode::BitwiseAnd32) { + return EquivalentValue(m_program, update->Arg(0), mask) || + EquivalentValue(m_program, update->Arg(1), mask); + } + if (update->GetOpcode() != ValueOpcode::BitwiseXor32) return false; Value bit; if (EquivalentValue(m_program, update->Arg(0), mask)) bit = update->Arg(1); else if (EquivalentValue(m_program, update->Arg(1), mask)) bit = update->Arg(0); @@ -701,13 +1133,13 @@ private: first->NumArgs() == 1u && EquivalentValue(m_program, first->Arg(0), mask); } - Value InitialClearingMask(Value value, const Block* update_block) const { + Value InitialCandidateMask(Value value, const Block* update_block) const { const auto* phi = value.Resolve().TryInstruction(); if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi || phi->NumArgs() != 2u || phi->GetType() != Type::U32) return {}; for (uint32_t back = 0; back < 2u; ++back) { if (phi->PhiBlock(back) != update_block || - !ClearsFirstSetBit(phi->Arg(back), value)) continue; + !MaskOnlyLosesBits(phi->Arg(back), value)) continue; const auto initial = phi->Arg(back ^ 1u).Resolve(); return ValidateRuntimeValue(m_program, initial, RuntimeValueType::Integer) ? initial : Value {}; @@ -715,11 +1147,12 @@ private: return {}; } - bool EdgeOn(const Block* from, const Block* to, Value predicate, bool positive) const { - Value condition; - bool taken = false; - return ConditionalEdge(from, to, condition, taken) && taken == positive && - EquivalentValue(m_program, condition, predicate); + bool MaskEdge(const Block* from, const Block* to, Value predicate, bool positive) const { + const auto edge = ConditionalEdge(from, to); + // Continuation needs an active lane; an inactive exit must include every lane. + return edge && edge->positive == positive && + (positive || edge->lanes == LaneQuantifier::All) && + EquivalentValue(m_program, edge->condition, predicate); } Value BoundedSetBitMask(Value index, Value guard) const { @@ -728,7 +1161,7 @@ private: phi->NumArgs() != 3u || phi->GetType() != Type::U32 || !ImpliesIndexBelow32(guard, index)) return {}; for (uint32_t bit_arm = 0; bit_arm < 3u; ++bit_arm) { - const auto bit_guard = PositiveUseGuard(phi->PhiBlock(bit_arm)); + const auto bit_guard = PositiveLaneWitness(phi->PhiBlock(bit_arm)); const auto* selected = phi->Arg(bit_arm).Resolve().TryInstruction(); if (selected == nullptr || selected->GetOpcode() != ValueOpcode::SelectU32 || selected->NumArgs() != 3u || @@ -737,11 +1170,11 @@ private: if (first == nullptr || first->GetOpcode() != ValueOpcode::FindILsb32 || first->NumArgs() != 1u || !ImpliesNonzero(bit_guard, first->Arg(0))) continue; - const auto initial_mask = InitialClearingMask(first->Arg(0), phi->PhiBlock(bit_arm)); + const auto initial_mask = InitialCandidateMask(first->Arg(0), phi->PhiBlock(bit_arm)); if (initial_mask.IsEmpty()) continue; for (uint32_t sentinel_arm = 0; sentinel_arm < 3u; ++sentinel_arm) { if (sentinel_arm == bit_arm) continue; - const auto sentinel_guard = PositiveUseGuard(phi->PhiBlock(sentinel_arm)); + const auto sentinel_guard = PositiveLaneWitness(phi->PhiBlock(sentinel_arm)); const auto* sentinel = phi->Arg(sentinel_arm).Resolve().TryInstruction(); uint32_t bound = 0; if (sentinel == nullptr || sentinel->GetOpcode() != ValueOpcode::SelectU32 || @@ -752,16 +1185,27 @@ private: const auto* active_phi = loop_active.TryInstruction(); if (active_phi == nullptr || active_phi->GetOpcode() != ValueOpcode::Phi || active_phi->NumArgs() != 2u) continue; + const auto other_arm = 3u - bit_arm - sentinel_arm; + const auto* carried = phi->Arg(other_arm).Resolve().TryInstruction(); + const auto* bit_block = phi->PhiBlock(bit_arm); + if (carried == nullptr || carried->GetOpcode() != ValueOpcode::Phi || + carried->NumArgs() != 2u || carried->Parent() != active_phi->Parent() || + selected->Arg(2).Resolve() != phi->Arg(sentinel_arm).Resolve() || + sentinel->Arg(2).Resolve() != phi->Arg(other_arm).Resolve()) continue; + const uint32_t back = carried->PhiBlock(0) == bit_block ? 0u : 1u; + if (carried->PhiBlock(back) != bit_block || + carried->Arg(back).Resolve() != phi->Arg(bit_arm).Resolve()) continue; bool invariant = false; for (uint32_t initial = 0; initial < 2u; ++initial) { - invariant = Implies(guard, active_phi->Arg(initial)) && - EdgeOn(active_phi->PhiBlock(initial ^ 1u), active_phi->Parent(), - active_phi->Arg(initial ^ 1u), true); + invariant = active_phi->PhiBlock(initial) == carried->PhiBlock(back ^ 1u) && + active_phi->PhiBlock(initial ^ 1u) == bit_block && + Implies(guard, active_phi->Arg(initial)) && + MaskEdge(bit_block, active_phi->Parent(), + active_phi->Arg(initial ^ 1u), true); if (invariant) break; } if (!invariant) continue; - const auto other_arm = 3u - bit_arm - sentinel_arm; - if (EdgeOn(phi->PhiBlock(other_arm), phi->Parent(), loop_active, false)) + if (MaskEdge(phi->PhiBlock(other_arm), phi->Parent(), loop_active, false)) return initial_mask; } } @@ -770,10 +1214,32 @@ private: bool MatchUniformizedMaterialKey(Value key, const Inst& image, DescriptorSource::IndirectImage& indirect, - DescriptorSource& material_source, uint32_t pc) { - const auto guard = PositiveUseGuard(image.Parent()); - if (guard.IsEmpty()) return false; - const auto local = EqualLocalKey(guard, key); + DescriptorSource& material_source) { + Value guard; + Value local; + const auto* first = key.Resolve().TryInstruction(); + if (first != nullptr && first->GetOpcode() == ValueOpcode::ReadFirstLane && + first->NumArgs() == 2u) { + guard = first->Arg(1).Resolve(); + // Empty EXEC selects lane zero, which may not have loaded a material key. + if (!HasActiveLane(guard, first->Parent())) return false; + local = first->Arg(0).Resolve(); + const auto* phi = guard.TryInstruction(); + if (phi != nullptr && phi->GetOpcode() == ValueOpcode::Phi && + phi->GetType() == Type::U1 && phi->NumArgs() == 2u) { + for (uint32_t initial = 0; initial < 2u; ++initial) { + if (Implies(phi->Arg(initial ^ 1u), guard)) { + // The backedge only removes lanes from the initial mask. + guard = phi->Arg(initial).Resolve(); + break; + } + } + } + } else { + guard = PositiveLaneWitness(image.Parent()); + if (guard.IsEmpty()) return false; + local = EqualLocalKey(guard, key); + } const auto* selected = local.Resolve().TryInstruction(); if (selected == nullptr || selected->GetOpcode() != ValueOpcode::SelectU32 || selected->NumArgs() != 3u || !Implies(guard, selected->Arg(0))) return false; @@ -792,7 +1258,7 @@ private: const auto* material_handle = read->Arg(0).Resolve().TryInstruction(); if (material_handle == nullptr || material_handle->GetOpcode() != ValueOpcode::GetAddressResource || - !MakeRuntimeTableSource(*material_handle, pc, material_source)) return false; + !MakeRuntimeTableSource(*read, material_source)) return false; AffineOffset offset; if (!MatchAffineOffset(read->Arg(1), active, offset) || offset.index.IsEmpty() || offset.stride == 0u || offset.offset + memory.offset > UINT32_MAX || @@ -812,59 +1278,57 @@ private: const auto* phi = key.Resolve().TryInstruction(); if (m_shader_writes || phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi || phi->GetType() != Type::U32 || phi->NumArgs() != 2u || - m_program.blocks.size() != m_program.block_info.size()) { - return {}; - } - bool induction = false; + m_program.blocks.size() != m_program.block_info.size()) return {}; + const Block* increment_block = nullptr; for (uint32_t initial = 0; initial < 2u; ++initial) { const auto zero = phi->Arg(initial).Resolve(); const auto* step = phi->Arg(initial ^ 1u).Resolve().TryInstruction(); if (!zero.IsImmediate() || zero.GetType() != Type::U32 || zero.U32() != 0u || step == nullptr || step->GetOpcode() != ValueOpcode::IAdd32 || - step->Parent() != phi->PhiBlock(initial ^ 1u)) { - continue; - } + step->Parent() != phi->PhiBlock(initial ^ 1u)) continue; uint32_t increment = 0; - induction = (step->Arg(0).Resolve() == key && - ImmediateU32(step->Arg(1), increment) && increment == 1u) || - (step->Arg(1).Resolve() == key && - ImmediateU32(step->Arg(0), increment) && increment == 1u); - if (induction) break; + if ((step->Arg(0).Resolve() == key && + ImmediateU32(step->Arg(1), increment) && increment == 1u) || + (step->Arg(1).Resolve() == key && + ImmediateU32(step->Arg(0), increment) && increment == 1u)) { + increment_block = step->Parent(); + break; + } } - if (!induction) return {}; + if (increment_block == nullptr) return {}; - const auto contains = [&](auto&& self, Value value, const Inst* comparison) -> bool { - value = value.Resolve(); - if (value.TryInstruction() == comparison) return true; - const auto* inst = value.TryInstruction(); - return inst != nullptr && inst->GetOpcode() == ValueOpcode::LogicalAnd && - (self(self, inst->Arg(0), comparison) || self(self, inst->Arg(1), comparison)); + const auto guarded_on_entry = [&](const Block* block, const auto& accepts) { + std::vector pending {block}; + for (size_t i = 0; i < pending.size(); ++i) { + const auto* current = pending[i]; + if (current == phi->Parent() || current->ImmPredecessors().empty()) return false; + for (const auto* previous: current->ImmPredecessors()) { + const auto edge = ConditionalEdge(previous, current); + if (edge && accepts(*edge)) continue; + if (std::ranges::find(pending, previous) == pending.end()) + pending.push_back(previous); + } + } + return true; }; for (const auto& use_of_key: phi->Uses()) { const auto* compare = use_of_key.user; if (compare->GetOpcode() != ValueOpcode::SLessThan32 || use_of_key.operand != 0u || !ValidateRuntimeValue(m_program, compare->Arg(1))) continue; - for (uint32_t i = 0; i < m_program.block_info.size(); ++i) { - const auto& info = m_program.block_info[i]; - const auto* negated = info.condition.Resolve().TryInstruction(); - if (info.terminator.kind != CFG::TerminatorKind::ConditionalBranch || - compare->Parent() != m_program.blocks[i] || negated == nullptr || - negated->GetOpcode() != ValueOpcode::LogicalNot || - !contains(contains, negated->Arg(0), compare) || - use == m_program.blocks[i] || - CanReach(m_program.blocks.front(), use, m_program.blocks[i]) || - CanReach(phi->Parent(), use, m_program.blocks[i]) || - CanReach(FindBlock(info.terminator.true_block), use, phi->Parent()) || - !CanReach(FindBlock(info.terminator.false_block), use, phi->Parent())) { - continue; - } - return compare->Arg(1); - } + if (!guarded_on_entry(use, [&](const EdgePredicate& edge) { + return edge.positive && Implies(edge.condition, Value(use_of_key.user)); + })) continue; + LoopBoundProof proof(m_program, *phi, *compare); + // The image bound and the increment guard are separate obligations: a + // skipped image alone does not prevent signed induction wraparound. + if (guarded_on_entry(increment_block, [&](const EdgePredicate& edge) { + return proof.Excludes(edge.condition, edge.positive); + })) return compare->Arg(1); } return {}; } - bool TryMakeIndirectImage(Inst& handle, uint32_t pc, IndirectImagePlan& plan) { + bool TryMakeIndirectImage(Inst& handle, IndirectImagePlan& plan) { if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) { return false; } @@ -914,7 +1378,7 @@ private: } DescriptorSource table_source; - if (!MakeRuntimeTableSource(*table_handle, pc, table_source)) { + if (!MakeRuntimeTableSource(*plan.reads[0], table_source)) { return false; } DescriptorSource material_source; @@ -931,7 +1395,7 @@ private: indirect.key_count = BoundedLoopCount(key, handle.Parent()); } if (indirect.key_count.IsEmpty() && - !MatchUniformizedMaterialKey(key, handle, indirect, material_source, pc)) { + !MatchUniformizedMaterialKey(key, handle, indirect, material_source)) { return false; } if ((table_offset & 3u) != 0u || @@ -942,7 +1406,7 @@ private: const auto* memory = material_read != nullptr ? ScalarReadMemory(*material_read, material_memory_index) : nullptr; if (table_offset != 0u || memory == nullptr || memory->kind != ResourceKind::ScalarBuffer || - memory->offset != 0u || !MemoryIndexBelongsTo(material_memory_index, *material_read)) { + memory->offset > INT32_MAX || !MemoryIndexBelongsTo(material_memory_index, *material_read)) { return false; } Value selector; @@ -950,6 +1414,9 @@ private: indirect.selector_offset)) { return false; } + const auto step = std::gcd(indirect.selector_stride, uint64_t {1} << 32u); + indirect.selector_offset = + (static_cast(indirect.selector_offset % step) & ~3u) + (memory->offset & ~3u); const auto* shift = plan.reads[0]->Arg(1).Resolve().TryInstruction(); const std::array material_users {shift}; if (!UsesOnly(*material_read, material_users) || !UsesOnly(*shift, plan.reads)) { @@ -958,7 +1425,7 @@ private: const auto* material_handle = material_read->Arg(0).Resolve().TryInstruction(); if (material_handle == nullptr || material_handle->GetOpcode() != ValueOpcode::GetBufferResource || - !MakeRuntimeTableSource(*material_handle, pc, material_source)) { + !MakeRuntimeTableSource(*material_read, material_source)) { return false; } indirect.material_source = InternSource(material_source); @@ -1007,21 +1474,22 @@ private: continue; } IndirectImagePlan plan; - if (TryMakeIndirectImage(*handle, inst.Flags().pc, plan)) { + if (TryMakeIndirectImage(*handle, plan)) { m_indirect_images.push_back(std::move(plan)); } } } } - bool GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, Inst*& handle, - uint32_t& source, bool sampler = false, bool sample_adjust = false) { + bool GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, + uint32_t base_reg, Inst*& handle, uint32_t& source, bool sampler = false, + bool sample_adjust = false) { handle = value.Resolve().TryInstruction(); if (handle == nullptr || handle->GetOpcode() != expected) { Fail(pc, fmt::format("memory operation requires {}", ValueOpcodeName(expected))); } DescriptorSource descriptor; - MakeSource(*handle, width, sampler, sample_adjust, descriptor, pc); + MakeSource(*handle, width, sampler, sample_adjust, base_reg, descriptor, pc); uint32_t bad_dword = 0; if (!ValidateSource(descriptor, bad_dword)) { if (expected == ValueOpcode::GetBufferResource && @@ -1204,8 +1672,8 @@ private: uint32_t resource = 0; if (buffer != BufferAccess::None) { - if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle, - source)) { + if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, + memory.resource * 4u, handle, source)) { if (memory.kind != ResourceKind::Buffer || !memory.SupportsIndirectBufferLoad(op)) { Fail(flags.pc, "buffer descriptor is not a valid runtime value; GPU-selected access " @@ -1255,7 +1723,8 @@ private: if (indirect != nullptr) { source = indirect->source; } else { - GetHandle(inst.Arg(0), ValueOpcode::GetImageResource, 8, flags.pc, handle, source); + GetHandle(inst.Arg(0), ValueOpcode::GetImageResource, 8, flags.pc, + memory.resource * 4u, handle, source); } resource = AddImage(source, memory, op, flags.pc); if (resource == UINT32_MAX) { @@ -1271,8 +1740,8 @@ private: uint32_t sampler_source = 0; const bool sample_adjust = (memory.image_sample_flags & Decoder::ImageSampleFlagAdjust) != 0; - GetHandle(inst.Arg(1), ValueOpcode::GetSamplerResource, 4, flags.pc, sampler_handle, - sampler_source, true, sample_adjust); + GetHandle(inst.Arg(1), ValueOpcode::GetSamplerResource, 4, flags.pc, + memory.sampler * 4u, sampler_handle, sampler_source, true, sample_adjust); sampler = AddSampler(sampler_source, flags.pc); if (sampler == UINT32_MAX) { Fail(flags.pc, "sampler resource limit exceeded"); @@ -1313,6 +1782,13 @@ private: } Program& m_program; + const Decoder::Program& m_decoded; + const CFG::Graph& m_native_cfg; + std::vector m_scalar_writes; + std::vector m_resolved_handles; + std::vector m_srt_visiting; + std::vector m_srt_visited; + std::vector m_scalar_reads; ShaderInfo m_info; std::vector m_sources; std::vector m_handle_patches; @@ -1324,8 +1800,8 @@ private: } // namespace -void TrackResources(Program& program) { - Tracker(program).Run(); +void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg) { + Tracker(program, decoded, native_cfg).Run(); } } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.h b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.h index 78b00c3c7..03e530235 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.h +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.h @@ -5,9 +5,8 @@ namespace Libs::Graphics::ShaderRecompiler::IR { -// Collects immutable resource topology from typed SSA handles, interns their resolved dwords in -// descriptor_sources, then writes dense indices to handle flags and MemoryInfo. -void TrackResources(Program& program); +// Resolves native descriptor sources, plans their scalar reads, and assigns dense resource bindings. +void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 4349d0c7a..34fdc08b0 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include namespace Libs::Graphics::ShaderRecompiler::IR { @@ -23,21 +22,6 @@ namespace { constexpr uint64_t AddressMask = 0x0000ffffffffffffull; -const char* StageName(ShaderType stage) { - switch (stage) { - case ShaderType::Vertex: return "vertex"; - case ShaderType::Pixel: return "pixel"; - case ShaderType::Fetch: return "fetch"; - case ShaderType::Compute: return "compute"; - default: return "unknown"; - } -} - -std::string Diagnostic(const ResourcePlan& program, uint32_t pc, const std::string& message) { - return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash, - StageName(program.stage), pc, message); -} - bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) { if (base > AddressMask) { return false; @@ -89,6 +73,7 @@ bool IsRuntimeSelect(ValueOpcode op) { bool IsRuntimeUniformOp(ValueOpcode op) { switch (op) { + case ValueOpcode::ConditionRef: case ValueOpcode::BitCastU32F32: case ValueOpcode::BitCastF32U32: case ValueOpcode::ConvertU32F32: @@ -312,151 +297,6 @@ private: std::unordered_set m_validated_dependencies; }; -class PlanBuilder { -public: - explicit PlanBuilder(Program& program): m_program(program) {} - - void Run() { - m_program.srt_reads.clear(); - m_program.dynamic_reads.clear(); - for (auto* block: m_program.blocks) { - for (auto& inst: *block) { - const auto op = inst.GetOpcode(); - if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) { - const auto flags = inst.Flags(); - if (flags.index < m_program.memory_info.size()) { - const auto kind = m_program.memory_info[flags.index].kind; - const bool crosswired = (op == ValueOpcode::LoadAddressU32 && - kind == ResourceKind::ScalarBuffer) || - (op == ValueOpcode::ReadConstBuffer && - kind == ResourceKind::ScalarAddress); - if (crosswired) { - Fail(flags.pc, - fmt::format("{} has incompatible scalar memory metadata", - ValueOpcodeName(op))); - } - } - } - if (IsDescriptorHandle(inst.GetOpcode())) { - for (size_t index = 0; index < inst.NumArgs(); index++) { - Collect(inst.Arg(index), 0); - } - } - } - } - for (auto* block: m_program.blocks) { - for (auto& inst: *block) { - if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_program, inst) && - inst.Arg(1).Resolve().IsImmediate() && - ValidateRuntimeValue(m_program, Value(&inst))) { - Collect(Value(&inst), inst.Flags().pc); - } - } - } - PatchReads(); - } - -private: - struct Patch { - Inst* inst = nullptr; - uint32_t slot = 0; - bool keep = false; - }; - - [[noreturn]] void Fail(uint32_t pc, const std::string& message) const { - const auto diagnostic = Diagnostic(m_program, pc, message); - EXIT("shader SRT planning failed: %s", diagnostic.c_str()); - std::abort(); - } - - void Collect(Value value, uint32_t use_pc) { - value = value.Resolve(); - if (value.IsImmediate()) { - return; - } - auto* inst = value.TryInstruction(); - if (inst == nullptr) { - Fail(use_pc, "invalid typed planning value"); - } - const auto cycle = std::ranges::find(m_visiting, inst); - if (cycle != m_visiting.end()) { - const auto contains_phi = std::any_of(cycle, m_visiting.end(), [](const Inst* value) { - return value->GetOpcode() == ValueOpcode::Phi; - }); - if (contains_phi) { - return; - } - Fail(use_pc, fmt::format("cyclic typed planning value {} without a phi", - ValueOpcodeName(inst->GetOpcode()))); - } - if (std::ranges::find(m_visited, inst) != m_visited.end()) { - return; - } - m_visiting.push_back(inst); - for (size_t index = 0; index < inst->NumArgs(); index++) { - Collect(inst->Arg(index), use_pc); - } - m_visiting.pop_back(); - m_visited.push_back(inst); - if (!IsRawRead(m_program, *inst)) { - return; - } - const auto offset = inst->Arg(1).Resolve(); - if (!offset.IsImmediate() || offset.GetType() != Type::U32) { - if (std::ranges::find(m_program.dynamic_reads, value) == - m_program.dynamic_reads.end()) { - m_program.dynamic_reads.push_back(value); - } - return; - } - for (uint32_t slot = 0; slot < m_program.srt_reads.size(); slot++) { - if (EquivalentValue(m_program, value, m_program.srt_reads[slot].value)) { - m_patches.push_back({inst, slot, false}); - return; - } - } - const auto slot = static_cast(m_program.srt_reads.size()); - m_program.srt_reads.push_back({value, slot}); - m_patches.push_back({inst, slot, true}); - } - - void PatchReads() { - for (const auto& patch: m_patches) { - auto* block = patch.inst->Parent(); - auto& list = block->Instructions(); - auto where = - std::ranges::find_if(list, [&](const Inst& inst) { return &inst == patch.inst; }); - const auto resource = - Value(&*block->PrependNewInst(where, ValueOpcode::GetSrtResource)); - const auto flat = Value(&*block->PrependNewInst(where, ValueOpcode::ReadConst, - {resource, Value(patch.slot)})); - const auto uses = patch.inst->Uses(); - for (const auto& use: uses) { - use.user->SetArg(use.operand, flat); - } - for (auto& info: m_program.block_info) { - if (info.condition.Resolve() == Value(patch.inst)) { - info.condition = flat; - } - if (info.indirect_target.Resolve() == Value(patch.inst)) { - info.indirect_target = flat; - } - } - if (patch.keep) { - const auto memory = patch.inst->Flags().index; - if (memory < m_program.memory_info.size()) { - m_program.memory_info[memory].planning_only = true; - } - block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(patch.inst)}); - } - } - } - - Program& m_program; - std::vector m_visiting; - std::vector m_visited; - std::vector m_patches; -}; } // namespace @@ -616,16 +456,15 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) { return false; } const auto byte_offset = - static_cast(immediate) + static_cast(offset); - const auto aligned = byte_offset & ~uint64_t {3}; + (static_cast(immediate) & ~uint64_t {3}) + (static_cast(offset) & ~3u); const auto stride = (static_cast(high) >> 16u) & 0x3fffu; const auto size = stride == 0u ? static_cast(static_cast(records)) : static_cast(stride) * static_cast(records); - if (aligned > size || size - aligned < sizeof(uint32_t)) { + if (byte_offset > size || size - byte_offset < sizeof(uint32_t)) { return false; } - address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3}; + address = (base & ~uint64_t {3}) + byte_offset; } else { const auto relative = (immediate & ~int64_t {3}) + static_cast(static_cast(offset) & ~3u); @@ -780,14 +619,18 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { } return false; case ValueOpcode::FPOrdLessThanEqual32: - if (binary()) { - result = Float32(a) <= Float32(b); - return true; - } - return false; case ValueOpcode::FPOrdGreaterThanEqual32: if (binary()) { - result = Float32(a) >= Float32(b); + const auto operand = [&](uint64_t bits) { + if (inst.Flags().flush_input_denorms && + (bits & 0x7fffffffu) < 0x00800000u) { + bits &= 0x80000000u; + } + return Float32(bits); + }; + result = inst.GetOpcode() == ValueOpcode::FPOrdLessThanEqual32 + ? operand(a) <= operand(b) + : operand(a) >= operand(b); return true; } return false; @@ -970,6 +813,7 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { return true; } return false; + case ValueOpcode::ConditionRef: return Arg(inst, 0, result); case ValueOpcode::LogicalNot: if (Arg(inst, 0, a)) { result = a == 0u; @@ -1061,13 +905,5 @@ bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValue return RuntimeValidator(program, type).Run(value); } -void BuildSrtPlan(Program& program) { - if (program.resource_tracking_complete) { - EXIT("shader SRT planning failed: cannot rebuild SRT after resource tracking"); - } - program.srt_plan_complete = false; - PlanBuilder(program).Run(); - program.srt_plan_complete = true; -} } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index 4b9ead833..9e160f41b 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -21,9 +21,6 @@ struct SrtRuntime { enum class RuntimeValueType { Any, Integer }; -// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots; -// dynamic offsets remain explicit and are never assigned a fake slot. -void BuildSrtPlan(Program& program); bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValueType type = RuntimeValueType::Any); // Uses the strict reader for values that affect shader specialization. diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 4dd4d8af3..484b49fab 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -1,6 +1,7 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/BindingLayout.h" +#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" @@ -126,8 +127,28 @@ struct Fixture { } void PlanAndTrack() { - BuildSrtPlan(program); - TrackResources(program); + for (size_t index = 0; index < program.block_info.size(); ++index) { + const auto condition = program.block_info[index].condition; + if (!condition.IsEmpty()) + Emit(ValueOpcode::Reference, {condition}, 0, program.blocks[index]); + } + for (auto *target : program.blocks) { + for (auto &inst : *target) { + if (inst.HasUses() || inst.MayHaveSideEffects() || + (BufferAccessOf(inst.GetOpcode()) == BufferAccess::None && + AddressOpcodeInfoOf(inst.GetOpcode()).access == AddressAccess::None && + ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None)) + continue; + auto value = Value(&inst); + if (value.GetType() == Type::U32x4) + value = Emit(ValueOpcode::CompositeExtractU32x4, {value, Value(0u)}, 0, target); + if (value.GetType() == Type::U8) + value = Emit(ValueOpcode::ConvertU32U8, {value}, 0, target); + Check(value.GetType() == Type::U32, "unhandled memory result type in fixture"); + Emit(ValueOpcode::ReferenceU32, {value}, 0, target); + } + } + TrackResources(program, {}, {}); } }; @@ -185,8 +206,9 @@ bool ReadLinearTestMemory(void *userdata, uint64_t address, std::span } std::unique_ptr -MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0, - bool memory_backed_material = false) { +MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 4, + bool memory_backed_material = false, uint32_t member_offset = 0, + uint32_t material_stride = 224) { auto fixture = std::make_unique(); std::array material_words; std::array heap_words; @@ -228,9 +250,9 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0, const auto selector = fixture->Emit(ValueOpcode::ReadFirstLane, {invocation, Value(true)}); const auto record = - fixture->Emit(ValueOpcode::IMul32, {selector, Value(224u)}); - const auto member = fixture->Emit(ValueOpcode::IAdd32, {record, Value(4u)}); - fixture->Emit(ValueOpcode::ReferenceU32, {record}); + fixture->Emit(ValueOpcode::IMul32, {selector, Value(material_stride)}); + const auto member = member_offset == 0 ? record : + fixture->Emit(ValueOpcode::IAdd32, {record, Value(member_offset)}); fixture->Emit(ValueOpcode::ReferenceU32, {member}); MemoryInfo material_scalar; material_scalar.kind = ResourceKind::ScalarBuffer; @@ -277,7 +299,9 @@ void TestInvariantIndirectImageMaterialization() { Check(fixture->program.info.buffers.size() == 1 && fixture->program.info.images.size() == 1 && - fixture->program.dynamic_reads.size() == 1, + std::ranges::any_of(*fixture->block, [](const Inst &inst) { + return inst.GetOpcode() == ValueOpcode::ReadConstBuffer; + }), "indirect image key was not retained as a scalar-buffer read"); const auto source = fixture->program.info.images[0].source; Check(source < fixture->program.descriptor_sources.size() && @@ -474,7 +498,7 @@ void TestInvariantIndirectImageMaterialization() { rebound_specialization != collapsed_specialization, "larger indirect candidate topology reused the old specialization"); - auto memory_backed = MakeIndirectImageFixture(false, 0u, true); + auto memory_backed = MakeIndirectImageFixture(false, 4u, true); memory_backed->PlanAndTrack(); auto memory_backed_plan = ExtractResourcePlan(memory_backed->program); EliminateDeadCode(memory_backed->program.blocks); @@ -519,28 +543,50 @@ void TestInvariantIndirectImageMaterialization() { "a same-shape refresh discarded the runtime output storage"); auto malformed = MakeIndirectImageFixture(true); - BuildSrtPlan(malformed->program); - CheckFatal([&] { TrackResources(malformed->program); }, "not a valid runtime value", + + CheckFatal([&] { malformed->PlanAndTrack(); }, "not a valid runtime value", "malformed indirect image pattern was accepted"); Check(!malformed->program.resource_tracking_complete && malformed->program.info.images.empty() && malformed->program.descriptor_sources.empty(), "malformed indirect image pattern was partially accepted"); - auto wrapped_immediate = MakeIndirectImageFixture(false, 4u); - BuildSrtPlan(wrapped_immediate->program); - CheckFatal([&] { TrackResources(wrapped_immediate->program); }, + auto negative_immediate = MakeIndirectImageFixture(false, 0xfffffffcu); + CheckFatal([&] { negative_immediate->PlanAndTrack(); }, "not a valid runtime value", - "wrapped scalar immediate entered the invariant image proof"); - Check(!wrapped_immediate->program.resource_tracking_complete, - "wrapped scalar immediate entered the invariant image proof"); + "negative scalar immediate entered the indirect image proof"); + + for (const auto [immediate, member, first, stride] : + {std::array{0u, 4u, 4u, 224u}, std::array{4u, 4u, 8u, 224u}, + std::array{36u, 0u, 36u, 224u}, std::array{1u, 3u, 0u, 224u}, + std::array{1u, 3u, 0u, 1u}, std::array{1u, 3u, 0u, 2u}}) { + auto split_offset = MakeIndirectImageFixture(false, immediate, false, member, stride); + split_offset->PlanAndTrack(); + const auto split_plan = ExtractResourcePlan(split_offset->program); + LinearTestMemory split_memory; + std::copy(image_descriptor.begin(), image_descriptor.end(), + split_memory.words.begin() + 0x1000u / 4u); + split_memory.fail_address = first == 36u ? 0x1004u : UINT64_MAX; + split_memory.watched_address = split_memory.base + first; + user_data[1] = stride << 16u; + user_data[2] = stride < 4u ? 8u / stride : 1u; + SrtRuntime split_runtime{.user_data = user_data, + .userdata = &split_memory, + .read_specialization_memory = ReadLinearTestMemory}; + Check(MaterializeResources(split_plan, split_runtime, snapshot, specialization) && + snapshot.images.size() == 1 && + snapshot.images[0].dwords == image_descriptor && + split_memory.watched_reads == 1u, + "indirect scalar offsets did not align and bound their components independently"); + } } void TestGuardedDirectImageTable() { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; - enum class Guard { Nonzero, Zero, Unrelated, Bypass }; + enum class Guard { Nonzero, SccNonZero, Plain, Zero, Unrelated, Bypass, ExecZero, VccZero }; const auto make_plan = [](Guard guard) { Fixture fixture(ShaderType::Pixel); + fixture.program.wave_size = 64u; auto *entry = fixture.block; auto *middle = fixture.AddBlock(); auto *before_sample = fixture.AddBlock(); @@ -557,8 +603,19 @@ void TestGuardedDirectImageTable() { const auto nonzero = fixture.Emit( ValueOpcode::INotEqual32, {Value(0u), guard == Guard::Unrelated ? fixture.UserData(2) : mask}); - fixture.program.block_info[0].condition = - fixture.Emit(ValueOpcode::LogicalNot, {nonzero}); + const auto kind = guard == Guard::ExecZero ? CFG::BranchCondition::ExecZero + : guard == Guard::VccZero ? CFG::BranchCondition::VccZero + : CFG::BranchCondition::SccZero; + auto condition = nonzero; + if (guard == Guard::SccNonZero) { + condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, + CFG::BranchCondition::SccNonZero); + } + condition = fixture.Emit(ValueOpcode::LogicalNot, {condition}); + if (guard != Guard::Plain && guard != Guard::SccNonZero) { + condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, kind); + } + fixture.program.block_info[0].condition = condition; fixture.program.block_info[0].terminator = { .kind = CFG::TerminatorKind::ConditionalBranch, .true_block = guard == Guard::Zero ? 1u : 4u, @@ -622,7 +679,10 @@ void TestGuardedDirectImageTable() { }; auto plan = make_plan(Guard::Nonzero); - for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass}) { + make_plan(Guard::SccNonZero); + make_plan(Guard::Plain); + for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass, + Guard::ExecZero, Guard::VccZero}) { CheckFatal([&] { make_plan(guard); }, "not a valid runtime value", "direct table accepted a selector without a dominating nonzero guard"); } @@ -703,55 +763,183 @@ void TestGuardedDirectImageTable() { void TestBoundedComputeImageLoop() { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; enum class Variant { - Bounded, WrongGuard, EntryBypass, ExitBypass, GuardBlock, WrongStep + Bounded, Plain, Nonzero, TrueEdge, WrongGuard, EntryBypass, ExitBypass, GuardBlock, + WrongStep, DivergentBound, DivergentKey, Disjunction, WrongPolarity, + IncrementBypass, PreviousBound, Masked, MaskedWrongGuard, MaskResurrection, StatusOverwrite, + GuardedDiamond, GuardedDiamondBypass }; const auto make_plan = [](Variant variant) { - Fixture fixture; + const bool diamond = variant >= Variant::GuardedDiamond; + Fixture fixture(diamond ? ShaderType::Vertex : ShaderType::Compute); + fixture.program.wave_size = 64u; + const bool masked = variant >= Variant::Masked; auto *entry = fixture.block; auto *header = fixture.AddBlock(); auto *body = fixture.AddBlock(); auto *latch = fixture.AddBlock(); auto *exit = fixture.AddBlock(); + auto *compare = masked ? fixture.AddBlock() : header; + auto *guard = masked ? fixture.AddBlock() : header; + auto *increment = masked ? fixture.AddBlock() : latch; + auto *final_exit = variant == Variant::PreviousBound ? fixture.AddBlock() : exit; entry->AddBranch(header); - header->AddBranch(exit); - header->AddBranch(body); - body->AddBranch(latch); - latch->AddBranch(header); - if (variant == Variant::EntryBypass) entry->AddBranch(body); - if (variant == Variant::ExitBypass) exit->AddBranch(body); - fixture.program.block_info[0].terminator = { - .kind = variant == Variant::EntryBypass ? CFG::TerminatorKind::ConditionalBranch - : CFG::TerminatorKind::Branch, - .true_block = 1u, .false_block = 2u}; - fixture.program.block_info[1].terminator = { - .kind = CFG::TerminatorKind::ConditionalBranch, - .true_block = 4u, .false_block = 2u}; - fixture.program.block_info[2].terminator = { - .kind = CFG::TerminatorKind::Branch, .true_block = 3u}; - fixture.program.block_info[3].terminator = { - .kind = CFG::TerminatorKind::Branch, .true_block = 1u}; - fixture.program.block_info[4].terminator = { - .kind = variant == Variant::ExitBypass ? CFG::TerminatorKind::Branch - : CFG::TerminatorKind::Return, - .true_block = 2u}; + if (!masked) { + header->AddBranch(exit); + header->AddBranch(body); + body->AddBranch(latch); + latch->AddBranch(header); + if (variant == Variant::PreviousBound) latch->AddBranch(final_exit); + if (variant == Variant::EntryBypass) entry->AddBranch(body); + if (variant == Variant::ExitBypass) exit->AddBranch(body); + if (variant == Variant::IncrementBypass || variant == Variant::PreviousBound) + exit->AddBranch(latch); + fixture.program.block_info[0].terminator = { + .kind = variant == Variant::EntryBypass ? CFG::TerminatorKind::ConditionalBranch + : CFG::TerminatorKind::Branch, + .true_block = 1u, .false_block = 2u}; + fixture.program.block_info[1].terminator = { + .kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = variant == Variant::TrueEdge ? 2u : 4u, + .false_block = variant == Variant::TrueEdge ? 4u : 2u}; + fixture.program.block_info[2].terminator = { + .kind = CFG::TerminatorKind::Branch, .true_block = 3u}; + fixture.program.block_info[3].terminator = { + .kind = variant == Variant::PreviousBound ? CFG::TerminatorKind::ConditionalBranch + : CFG::TerminatorKind::Branch, + .true_block = 1u, .false_block = 5u}; + fixture.program.block_info[4].terminator = { + .kind = final_exit != exit || variant == Variant::ExitBypass || + variant == Variant::IncrementBypass + ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return, + .true_block = final_exit != exit || variant == Variant::IncrementBypass ? 3u : 2u}; + if (final_exit != exit) + fixture.program.block_info[5].terminator.kind = CFG::TerminatorKind::Return; + } auto &phi = header->AppendNewInst(ValueOpcode::Phi, {}, static_cast(Type::U32)); const auto key = Value(&phi); - const auto count = fixture.UserData(2); + auto *previous = variant == Variant::PreviousBound ? + &header->AppendNewInst(ValueOpcode::Phi, {}, static_cast(Type::U1)) : nullptr; + const auto local = fixture.Emit( + ValueOpcode::GetBuiltin, + {Value(static_cast(diamond ? StageInputKind::VertexIndex + : StageInputKind::LocalInvocationId)), Value(0u)}); + const auto count = variant == Variant::DivergentBound ? local : fixture.UserData(2); const auto in_range = fixture.Emit(ValueOpcode::SLessThan32, {variant == Variant::WrongGuard ? Value(0u) : key, - count}, 0, header); - const auto allowed = fixture.Emit(ValueOpcode::LogicalAnd, - {in_range, Value(true)}, 0, header); - fixture.program.block_info[1].condition = - fixture.Emit(ValueOpcode::LogicalNot, {allowed}, 0, header); + count}, 0, compare); + const auto initial_active = fixture.Emit(ValueOpcode::INotEqual32, + {local, Value(0u)}, 0, entry); + if (masked) { + auto *active_phi = diamond ? nullptr : &header->AppendNewInst( + ValueOpcode::Phi, {}, static_cast(Type::U1)); + auto &saved_phi = header->AppendNewInst(ValueOpcode::Phi, {}, + static_cast(Type::U1)); + auto &status_phi = header->AppendNewInst(ValueOpcode::Phi, {}, + static_cast(Type::U32)); + const auto saved = Value(&saved_phi); + const auto status = Value(&status_phi); + const auto active = diamond ? fixture.Emit(ValueOpcode::LogicalAnd, + {saved, fixture.Emit(ValueOpcode::UGreaterThanEqual32, + {Value(0u), status}, 0, header)}, 0, header) : Value(active_phi); + const auto mask = fixture.Emit(ValueOpcode::LogicalOr, + {fixture.Emit(ValueOpcode::LogicalAnd, {in_range, active}, 0, compare), + fixture.Emit(ValueOpcode::LogicalNot, {active}, 0, compare)}, 0, compare); + auto next_saved = fixture.Emit(ValueOpcode::LogicalAnd, + {variant == Variant::MaskResurrection ? Value(true) : saved, mask}, 0, compare); + const auto execute = fixture.Emit(ValueOpcode::LogicalAnd, {active, mask}, 0, compare); + const auto body_status = fixture.Emit(ValueOpcode::SelectU32, + {execute, local, status}, 0, guard); + const auto next_active = fixture.Emit(ValueOpcode::LogicalAnd, + {next_saved, fixture.Emit(ValueOpcode::UGreaterThanEqual32, + {Value(3u), body_status}, 0, latch)}, 0, latch); + auto next_status = fixture.Emit(ValueOpcode::SelectU32, + {variant == Variant::StatusOverwrite ? Value(true) : next_active, + Value(0u), body_status}, 0, + variant == Variant::GuardedDiamondBypass ? latch : increment); + if (variant == Variant::GuardedDiamondBypass) { + auto &merged_saved = increment->AppendNewInst(ValueOpcode::Phi, {}, + static_cast(Type::U1)); + merged_saved.AddPhiOperand(compare, saved); + merged_saved.AddPhiOperand(latch, next_saved); + next_saved = Value(&merged_saved); + auto &merged_status = increment->AppendNewInst(ValueOpcode::Phi, {}, + static_cast(Type::U32)); + merged_status.AddPhiOperand(compare, status); + merged_status.AddPhiOperand(latch, next_status); + next_status = Value(&merged_status); + } + if (active_phi != nullptr) { + active_phi->AddPhiOperand(entry, initial_active); + active_phi->AddPhiOperand(increment, next_active); + } + saved_phi.AddPhiOperand(entry, initial_active); + saved_phi.AddPhiOperand(increment, next_saved); + status_phi.AddPhiOperand(entry, diamond ? Value(0u) : local); + status_phi.AddPhiOperand(increment, next_status); + const auto branch = [&](uint32_t index, uint32_t yes, uint32_t no, + Value predicate, CFG::BranchCondition kind) { + auto *block = fixture.program.blocks[index]; + block->AddBranch(fixture.program.blocks[yes]); + block->AddBranch(fixture.program.blocks[no]); + const auto inverse = fixture.Emit(ValueOpcode::LogicalNot, {predicate}, 0, block); + fixture.program.block_info[index].condition = + fixture.Emit(ValueOpcode::ConditionRef, {inverse}, kind, block); + fixture.program.block_info[index].terminator = { + .kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = yes, .false_block = no}; + }; + fixture.program.block_info[0].terminator = { + .kind = CFG::TerminatorKind::Branch, .true_block = 1u}; + branch(1u, 4u, 5u, active, CFG::BranchCondition::ExecZero); + branch(5u, variant == Variant::GuardedDiamondBypass ? 7u : 4u, 6u, + diamond ? execute : mask, + diamond ? CFG::BranchCondition::ExecZero : CFG::BranchCondition::VccZero); + const auto image_guard = variant == Variant::MaskedWrongGuard ? active : + fixture.Emit(ValueOpcode::LogicalAnd, + {fixture.Emit(ValueOpcode::LogicalAnd, {execute, in_range}, 0, guard), + fixture.Emit(ValueOpcode::IEqual32, {local, Value(1u)}, 0, guard)}, 0, guard); + branch(6u, 3u, 2u, image_guard, CFG::BranchCondition::ExecZero); + body->AddBranch(latch); + fixture.program.block_info[2].terminator = { + .kind = CFG::TerminatorKind::Branch, .true_block = 3u}; + branch(3u, 4u, 7u, next_active, CFG::BranchCondition::ExecZero); + increment->AddBranch(header); + fixture.program.block_info[7].terminator = { + .kind = CFG::TerminatorKind::Branch, .true_block = 1u}; + fixture.program.block_info[4].terminator.kind = CFG::TerminatorKind::Return; + } else { + const auto active = initial_active; + const auto allowed = fixture.Emit(variant == Variant::Disjunction + ? ValueOpcode::LogicalOr : ValueOpcode::LogicalAnd, + {in_range, active}, 0, header); + const bool nonzero = variant == Variant::Nonzero || variant == Variant::TrueEdge; + auto condition = allowed; + if (!nonzero) + condition = fixture.Emit(ValueOpcode::LogicalNot, {condition}, 0, header); + if (variant != Variant::Plain) + condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, + nonzero ? CFG::BranchCondition::ExecNonZero + : CFG::BranchCondition::ExecZero, header); + if (variant == Variant::Nonzero || variant == Variant::WrongPolarity) + condition = fixture.Emit(ValueOpcode::LogicalNot, {condition}, 0, header); + fixture.program.block_info[1].condition = condition; + if (variant == Variant::PreviousBound) { + previous->AddPhiOperand(entry, Value(false)); + previous->AddPhiOperand(latch, in_range); + const auto continuing = fixture.Emit(ValueOpcode::LogicalOr, + {in_range, Value(previous)}, 0, latch); + fixture.program.block_info[3].condition = fixture.Emit(ValueOpcode::ConditionRef, + {continuing}, CFG::BranchCondition::SccNonZero, latch); + } + } const auto step = fixture.Emit(ValueOpcode::IAdd32, {key, Value(variant == Variant::WrongStep ? 2u : 1u)}, - 0, latch); - phi.AddPhiOperand(entry, Value(0u)); - phi.AddPhiOperand(latch, step); + 0, increment); + phi.AddPhiOperand(entry, variant == Variant::DivergentKey ? local : Value(0u)); + phi.AddPhiOperand(increment, step); fixture.block = variant == Variant::GuardBlock ? header : body; const auto table = fixture.Address(fixture.UserData(0), fixture.UserData(1)); @@ -787,6 +975,18 @@ void TestBoundedComputeImageLoop() { }; auto plan = make_plan(Variant::Bounded); + make_plan(Variant::Plain); + make_plan(Variant::Nonzero); + make_plan(Variant::TrueEdge); + make_plan(Variant::Masked); + make_plan(Variant::GuardedDiamond); + for (const auto variant : {Variant::IncrementBypass, Variant::PreviousBound, + Variant::MaskedWrongGuard, + Variant::MaskResurrection, Variant::StatusOverwrite, + Variant::GuardedDiamondBypass}) { + CheckFatal([&] { make_plan(variant); }, "not a valid runtime value", + "compute image loop allowed an unbounded induction or mask resurrection"); + } CheckFatal([&] { make_plan(Variant::WrongGuard); }, "not a valid runtime value", "compute image loop accepted an unrelated guard"); @@ -802,6 +1002,11 @@ void TestBoundedComputeImageLoop() { CheckFatal([&] { make_plan(Variant::WrongStep); }, "not a valid runtime value", "compute image loop accepted a two-step induction"); + for (const auto variant : {Variant::DivergentBound, Variant::DivergentKey, + Variant::Disjunction, Variant::WrongPolarity}) { + CheckFatal([&] { make_plan(variant); }, "not a valid runtime value", + "compute image loop accepted a guard without a uniform bound"); + } LinearTestMemory memory; const auto table = 0x1800u + 0x6b0u; @@ -849,8 +1054,21 @@ void TestBoundedComputeImageLoop() { void TestUniformizedMaterialImageKeys() { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; - const auto make_plan = [](bool wrong_update, bool wrong_equality) { + enum class Variant { + Valid, Plain, WrongUpdate, WrongEquality, WrongExit, WrongCarry, WrongBackedge, + AndNot, Subset, FirstLane, FirstLaneDirect, FirstLaneAndNot, EmptyFirstEntry, WrongFirstBackedge, + WideningFirstEntry, WideningFirstBackedge, WrongFirstLoadMask + }; + const auto make_plan = [](Variant variant) { Fixture fixture; + fixture.program.wave_size = 64u; + const bool first_lane = variant >= Variant::FirstLane; + const bool first_lane_loop = first_lane && variant != Variant::FirstLaneDirect; + const uint32_t sample_entry_id = first_lane_loop ? 10u : 9u; + const auto branch = [&](Value predicate, CFG::BranchCondition kind, Block *block) { + return variant == Variant::Plain ? predicate : fixture.Emit( + ValueOpcode::ConditionRef, {predicate}, kind, block); + }; auto *entry = fixture.block; auto *header = fixture.AddBlock(); auto *inactive = fixture.AddBlock(); @@ -860,6 +1078,8 @@ void TestUniformizedMaterialImageKeys() { auto *choose = fixture.AddBlock(); auto *sample = fixture.AddBlock(); auto *done = fixture.AddBlock(); + auto *sample_header = first_lane_loop ? fixture.AddBlock() : nullptr; + auto *sample_entry = first_lane ? fixture.AddBlock() : nullptr; entry->AddBranch(header); header->AddBranch(inactive); inactive->AddBranch(merge); @@ -869,23 +1089,44 @@ void TestUniformizedMaterialImageKeys() { bit->AddBranch(header); bit->AddBranch(merge); merge->AddBranch(choose); - choose->AddBranch(sample); + choose->AddBranch(first_lane ? sample_entry : sample); choose->AddBranch(done); + if (first_lane) { + sample_entry->AddBranch(first_lane_loop ? sample_header : sample); + fixture.program.block_info[sample_entry_id].terminator = { + .kind = CFG::TerminatorKind::Branch, .true_block = first_lane_loop ? 9u : 7u}; + } + if (first_lane_loop) { + sample_header->AddBranch(sample); + sample->AddBranch(sample_header); + fixture.program.block_info[9].terminator = { + .kind = CFG::TerminatorKind::Branch, .true_block = 7u, + .merge_block = 8u, .continue_block = 7u, .loop_header = true}; + } sample->AddBranch(done); fixture.program.block_info[0].terminator = { .kind = CFG::TerminatorKind::Branch, .true_block = 1u}; fixture.program.block_info[1].terminator = { .kind = CFG::TerminatorKind::Branch, .true_block = 2u}; - const auto active_on_entry = fixture.Emit( + const auto enabled = fixture.Emit( ValueOpcode::INotEqual32, {fixture.UserData(5u), Value(0u)}, 0, entry); + const auto local = fixture.Emit(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::LocalInvocationId)), Value(0u)}, 0, entry); + const auto active_on_entry = fixture.Emit(ValueOpcode::LogicalAnd, + {enabled, fixture.Emit(ValueOpcode::INotEqual32, {local, Value(0u)}, 0, entry)}, + 0, entry); auto &active_phi = header->AppendNewInst( ValueOpcode::Phi, {}, static_cast(Type::U1)); auto &mask_phi = header->AppendNewInst( ValueOpcode::Phi, {}, static_cast(Type::U32)); + auto &carry_phi = header->AppendNewInst( + ValueOpcode::Phi, {}, static_cast(Type::U32)); const auto mask = Value(&mask_phi); const auto active = Value(&active_phi); - fixture.program.block_info[2].condition = fixture.Emit( - ValueOpcode::LogicalNot, {active}, 0, inactive); + fixture.program.block_info[2].condition = branch( + fixture.Emit(ValueOpcode::LogicalNot, {active}, 0, inactive), + variant == Variant::WrongExit ? CFG::BranchCondition::ExecNonZero + : CFG::BranchCondition::ExecZero, inactive); fixture.program.block_info[2].terminator = { .kind = CFG::TerminatorKind::ConditionalBranch, .true_block = 5u, .false_block = 3u}; @@ -893,8 +1134,9 @@ void TestUniformizedMaterialImageKeys() { ValueOpcode::INotEqual32, {Value(0u), mask}, 0, sentinel); const auto bit_guard = fixture.Emit( ValueOpcode::LogicalAnd, {active, nonzero}, 0, sentinel); - fixture.program.block_info[3].condition = fixture.Emit( - ValueOpcode::LogicalNot, {bit_guard}, 0, sentinel); + fixture.program.block_info[3].condition = branch( + fixture.Emit(ValueOpcode::LogicalNot, {bit_guard}, 0, sentinel), + CFG::BranchCondition::ExecZero, sentinel); fixture.program.block_info[3].terminator = { .kind = CFG::TerminatorKind::ConditionalBranch, .true_block = 5u, .false_block = 4u}; @@ -903,12 +1145,19 @@ void TestUniformizedMaterialImageKeys() { ValueOpcode::BitwiseAnd32, {first, Value(31u)}, 0, bit); const auto one_bit = fixture.Emit( ValueOpcode::ShiftLeftLogical32, {Value(1u), position}, 0, bit); + const bool and_not = variant == Variant::AndNot || variant == Variant::FirstLaneAndNot; + const auto removed = and_not + ? fixture.Emit(ValueOpcode::BitwiseNot32, {one_bit}, 0, bit) + : variant == Variant::Subset ? fixture.UserData(7u) : one_bit; const auto cleared = fixture.Emit( - wrong_update ? ValueOpcode::BitwiseOr32 : ValueOpcode::BitwiseXor32, - {mask, one_bit}, 0, bit); + variant == Variant::WrongUpdate ? ValueOpcode::BitwiseOr32 + : and_not || variant == Variant::Subset ? ValueOpcode::BitwiseAnd32 + : ValueOpcode::BitwiseXor32, + {mask, removed}, 0, bit); const auto continuation = fixture.Emit( ValueOpcode::LogicalAnd, {bit_guard, active_on_entry}, 0, bit); - fixture.program.block_info[4].condition = continuation; + fixture.program.block_info[4].condition = branch( + continuation, CFG::BranchCondition::ExecNonZero, bit); fixture.program.block_info[4].terminator = { .kind = CFG::TerminatorKind::ConditionalBranch, .true_block = 1u, .false_block = 5u}; @@ -917,13 +1166,18 @@ void TestUniformizedMaterialImageKeys() { mask_phi.AddPhiOperand(entry, fixture.UserData(4u)); mask_phi.AddPhiOperand(bit, cleared); const auto arbitrary = fixture.UserData(6u); + const auto carry = Value(&carry_phi); const auto sentinel_index = fixture.Emit( - ValueOpcode::SelectU32, {active, Value(32u), arbitrary}, 0, sentinel); + ValueOpcode::SelectU32, + {active, Value(32u), variant == Variant::WrongCarry ? arbitrary : carry}, + 0, sentinel); const auto bit_index = fixture.Emit( ValueOpcode::SelectU32, {bit_guard, first, sentinel_index}, 0, bit); + carry_phi.AddPhiOperand(entry, arbitrary); + carry_phi.AddPhiOperand(bit, variant == Variant::WrongBackedge ? arbitrary : bit_index); auto &index_phi = merge->AppendNewInst( ValueOpcode::Phi, {}, static_cast(Type::U32)); - index_phi.AddPhiOperand(inactive, arbitrary); + index_phi.AddPhiOperand(inactive, carry); index_phi.AddPhiOperand(sentinel, sentinel_index); index_phi.AddPhiOperand(bit, bit_index); const auto index = Value(&index_phi); @@ -950,24 +1204,54 @@ void TestUniformizedMaterialImageKeys() { const auto base = fixture.Address(fixture.UserData(0u), fixture.UserData(1u)); MemoryInfo material_memory; material_memory.kind = ResourceKind::Global; + const auto load_mask = variant == Variant::WrongFirstLoadMask + ? fixture.Emit(ValueOpcode::LogicalNot, {material_guard}, 0, choose) + : material_guard; const auto loaded = fixture.Emit( ValueOpcode::LoadAddressU32, - {base, material_offset, Value(0u), material_guard}, + {base, material_offset, Value(0u), load_mask}, fixture.AddMemory(material_memory, 0x1a88u), choose); - const auto local = fixture.Emit( + const auto local_key = fixture.Emit( ValueOpcode::SelectU32, {material_guard, loaded, arbitrary}, 0, choose); - const auto key = fixture.Emit( - ValueOpcode::ReadLane, {local, Value(0u)}, 0, choose); + Inst *sample_active_phi = nullptr; + auto sample_active = material_guard; + if (first_lane) { + const auto initial = variant == Variant::EmptyFirstEntry ? Value(false) + : variant == Variant::WideningFirstEntry ? enabled + : fixture.Emit(ValueOpcode::LogicalAnd, + {material_guard, fixture.Emit(ValueOpcode::SLessThanEqual32, + {Value(0u), local_key}, 0, choose)}, 0, choose); + sample_active = initial; + if (first_lane_loop) { + sample_active_phi = &sample_header->AppendNewInst( + ValueOpcode::Phi, {}, static_cast(Type::U1)); + sample_active_phi->AddPhiOperand(sample_entry, initial); + sample_active = Value(sample_active_phi); + } + fixture.program.block_info[6].condition = branch( + variant == Variant::EmptyFirstEntry ? enabled : initial, + CFG::BranchCondition::ExecNonZero, choose); + fixture.program.block_info[6].terminator = { + .kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = sample_entry_id, .false_block = 8u}; + } + auto *key_block = first_lane ? sample : choose; + const auto key = first_lane + ? fixture.Emit(ValueOpcode::ReadFirstLane, {local_key, sample_active}, 0, key_block) + : fixture.Emit(ValueOpcode::ReadLane, {local_key, Value(0u)}, 0, key_block); const auto compared = fixture.Emit( ValueOpcode::IEqual32, - {key, wrong_equality ? arbitrary : local}, 0, choose); + {key, variant == Variant::WrongEquality ? arbitrary : local_key}, 0, key_block); const auto sample_guard = fixture.Emit( - ValueOpcode::LogicalAnd, {material_guard, compared}, 0, choose); - fixture.program.block_info[6].condition = fixture.Emit( - ValueOpcode::LogicalNot, {sample_guard}, 0, choose); - fixture.program.block_info[6].terminator = { - .kind = CFG::TerminatorKind::ConditionalBranch, - .true_block = 8u, .false_block = 7u}; + ValueOpcode::LogicalAnd, {sample_active, compared}, 0, key_block); + if (!first_lane) { + fixture.program.block_info[6].condition = branch( + fixture.Emit(ValueOpcode::LogicalNot, {sample_guard}, 0, choose), + CFG::BranchCondition::ExecZero, choose); + fixture.program.block_info[6].terminator = { + .kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = 8u, .false_block = 7u}; + } const auto table_offset = fixture.Emit( ValueOpcode::IAdd32, {fixture.Emit(ValueOpcode::ShiftLeftLogical32, @@ -997,10 +1281,33 @@ void TestUniformizedMaterialImageKeys() { MemoryInfo image_memory; image_memory.kind = ResourceKind::Image; image_memory.image_dimension = Decoder::ImageDimension::Dim2D; - fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, address}, - fixture.AddMemory(image_memory, 0x1aecu), sample); - fixture.program.block_info[7].terminator = { - .kind = CFG::TerminatorKind::Branch, .true_block = 8u}; + const auto sampled = fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, address}, + fixture.AddMemory(image_memory, 0x1aecu), sample); + if (first_lane) { + const auto component = fixture.Emit( + ValueOpcode::CompositeExtractU32x4, {sampled, Value(0u)}, 0, sample); + fixture.Emit(ValueOpcode::ReferenceU32, + {fixture.Emit(ValueOpcode::SelectU32, + {variant == Variant::EmptyFirstEntry ? material_guard : sample_guard, + component, Value(0u)}, 0, sample)}, 0, sample); + } + if (first_lane_loop) { + auto remaining = fixture.Emit(ValueOpcode::LogicalAnd, + {sample_active, fixture.Emit(ValueOpcode::LogicalNot, + {sample_guard}, 0, sample)}, 0, sample); + if (variant == Variant::WideningFirstBackedge) + remaining = fixture.Emit(ValueOpcode::LogicalOr, {remaining, enabled}, 0, sample); + sample_active_phi->AddPhiOperand(sample, remaining); + fixture.program.block_info[7].condition = branch( + variant == Variant::WrongFirstBackedge ? sample_guard : remaining, + CFG::BranchCondition::ExecNonZero, sample); + fixture.program.block_info[7].terminator = { + .kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = 9u, .false_block = 8u}; + } else { + fixture.program.block_info[7].terminator = { + .kind = CFG::TerminatorKind::Branch, .true_block = 8u}; + } fixture.program.block_info[8].terminator.kind = CFG::TerminatorKind::Return; const auto output = fixture.Emit( ValueOpcode::GetBufferResource, @@ -1024,7 +1331,13 @@ void TestUniformizedMaterialImageKeys() { "uniformized image key lost its finite material range"); return ExtractResourcePlan(fixture.program); }; - auto plan = make_plan(false, false); + auto plan = make_plan(Variant::Valid); + make_plan(Variant::Plain); + make_plan(Variant::AndNot); + make_plan(Variant::Subset); + make_plan(Variant::FirstLane); + make_plan(Variant::FirstLaneDirect); + plan = make_plan(Variant::FirstLaneAndNot); Check(plan.requires_specialization_memory && plan.descriptor_sources[plan.info.images[0].source] .indirect_image->selector_mask.Resolve().TryInstruction() != nullptr, @@ -1078,10 +1391,22 @@ void TestUniformizedMaterialImageKeys() { user_data[8] = first_table; Check(!MaterializeResources(plan, runtime, snapshot, specialization), "written buffer alias with an image record was accepted"); - CheckFatal([&] { make_plan(true, false); }, "not a valid runtime value", + CheckFatal([&] { make_plan(Variant::WrongUpdate); }, "not a valid runtime value", "non-clearing material mask was accepted"); - CheckFatal([&] { make_plan(false, true); }, "not a valid runtime value", + CheckFatal([&] { make_plan(Variant::WrongEquality); }, "not a valid runtime value", "unrelated ReadLane key was accepted"); + CheckFatal([&] { make_plan(Variant::WrongExit); }, "not a valid runtime value", + "one inactive lane bypassed material index initialization for active lanes"); + for (const auto variant : {Variant::WrongCarry, Variant::WrongBackedge}) { + CheckFatal([&] { make_plan(variant); }, "not a valid runtime value", + "inactive material lane did not preserve its selected index across iterations"); + } + for (const auto variant : {Variant::EmptyFirstEntry, Variant::WrongFirstBackedge, + Variant::WideningFirstEntry, Variant::WideningFirstBackedge, + Variant::WrongFirstLoadMask}) { + CheckFatal([&] { make_plan(variant); }, "not a valid runtime value", + "first-lane material key escaped its active load mask"); + } } void TestImageDescriptorFields() { @@ -1405,7 +1730,7 @@ void TestDenseBufferTracking() { fixture.program.memory_info[other_flags.index].resource == 1, "typed memory metadata was not patched to dense indices"); - CheckFatal([&] { TrackResources(fixture.program); }, "already tracked", + CheckFatal([&] { fixture.PlanAndTrack(); }, "already tracked", "resource tracking allowed a second mutation pass"); } @@ -1601,8 +1926,8 @@ void TestSampleAdjustSamplerScratch() { rejected.Emit(ValueOpcode::ImageSampleRaw, {rejected_image, rejected_sampler, rejected.ImageAddress()}, rejected.AddMemory(rejected_memory, 0x200)); - BuildSrtPlan(rejected.program); - CheckFatal([&] { TrackResources(rejected.program); }, + + CheckFatal([&] { rejected.PlanAndTrack(); }, "not a valid runtime value", message); }; CheckRejected(0u, 12u, @@ -1936,7 +2261,8 @@ void TestDynamicSrtReadRemainsExplicit() { fixture.PlanAndTrack(); Check(fixture.program.srt_reads.empty() && - fixture.program.dynamic_reads.size() == 1 && + read.ResolveInstruction()->GetOpcode() == ValueOpcode::LoadAddressU32 && + !fixture.program.memory_info[0].planning_only && fixture.program.info.uses_dma, "dynamic scalar read was incorrectly flattened or lost"); std::array user_data{0x1000u, 0u, 4u}; @@ -1990,8 +2316,7 @@ void TestPhiValidation() { {handle, Value(0u), Value(0u), Value(0u), Value(true)}, fixture.AddMemory(memory, 20), merge); - BuildSrtPlan(fixture.program); - CheckFatal([&] { TrackResources(fixture.program); }, "not a valid runtime value", + CheckFatal([&] { fixture.PlanAndTrack(); }, "not a valid runtime value", "control-dependent descriptor phi was accepted"); Check(!fixture.program.resource_tracking_complete && fixture.program.info.buffers.empty() && @@ -2092,7 +2417,9 @@ ResourcePlan ConditionalSamplerPlan(bool diamond, bool reverse, bool reverse_phi {result, Value(0u)})}); } fixture.PlanAndTrack(); - Check(fixture.program.value_storage.size() == 4, + Check(std::ranges::count_if(fixture.program.value_storage, [](const Inst &inst) { + return inst.GetOpcode() == ValueOpcode::SelectU32; + }) == 4, "repeated sampler uses retained duplicate planning selections"); Check(sampler_words[0].ResolveInstruction()->GetOpcode() == ValueOpcode::Phi, "host descriptor selection changed the GPU Phi"); @@ -2155,90 +2482,6 @@ void TestConditionalSamplerPhi() { } } -void TestGuardedSamplerPhi() { - namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; - const auto make_plan = [](bool take_true, bool reverse_phi, bool bypass, - bool mismatched, bool ambiguous) { - Fixture fixture(ShaderType::Pixel); - auto *entry = fixture.block; - auto *loaded = fixture.AddBlock(); - auto *merge = fixture.AddBlock(); - auto *sample = fixture.AddBlock(); - auto *exit = fixture.AddBlock(); - entry->AddBranch(loaded); - entry->AddBranch(merge); - loaded->AddBranch(merge); - merge->AddBranch(sample); - merge->AddBranch(exit); - sample->AddBranch(exit); - if (bypass) exit->AddBranch(sample); - const auto lane = fixture.Emit(ValueOpcode::LaneId); - const auto predicate = fixture.Emit(ValueOpcode::IEqual32, {lane, Value(0u)}); - fixture.program.block_info[0].condition = predicate; - fixture.program.block_info[0].terminator = { - .kind = CFG::TerminatorKind::ConditionalBranch, - .true_block = 1u, .false_block = 2u}; - fixture.program.block_info[1].terminator = { - .kind = CFG::TerminatorKind::Branch, .true_block = 2u}; - auto &guard = merge->AppendNewInst(ValueOpcode::Phi, {}, - static_cast(Type::U1)); - guard.AddPhiOperand(entry, ambiguous ? predicate : Value(!take_true)); - guard.AddPhiOperand(mismatched ? sample : loaded, predicate); - fixture.program.block_info[2].condition = Value(&guard); - fixture.program.block_info[2].terminator = { - .kind = CFG::TerminatorKind::ConditionalBranch, - .true_block = take_true ? 3u : 4u, - .false_block = take_true ? 4u : 3u}; - fixture.program.block_info[3].terminator = { - .kind = CFG::TerminatorKind::Branch, .true_block = 4u}; - fixture.program.block_info[4].terminator = { - .kind = bypass ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return, - .true_block = 3u}; - std::array words; - for (uint32_t word = 0; word < words.size(); ++word) { - // An arbitrary excluded value proves this is control-flow reasoning, not null filtering. - const auto first = Value(0xbad000u + word); - const auto second = fixture.UserData(word); - auto &phi = merge->AppendNewInst(ValueOpcode::Phi, {}, - static_cast(Type::U32)); - phi.AddPhiOperand(reverse_phi ? loaded : entry, reverse_phi ? second : first); - phi.AddPhiOperand(reverse_phi ? entry : loaded, reverse_phi ? first : second); - words[word] = Value(&phi); - } - fixture.block = sample; - const auto image = fixture.Image({Value(0u), Value(0u), Value(0u), Value(0u), - Value(0u), Value(0u), Value(0u), Value(0u)}); - const auto sampler = fixture.Sampler(words); - MemoryInfo memory; - memory.kind = ResourceKind::Image; - memory.image_dimension = Decoder::ImageDimension::Dim2D; - fixture.Emit(ValueOpcode::ImageSampleRaw, - {image, sampler, fixture.ImageAddress()}, - fixture.AddMemory(memory, 0x2a0)); - fixture.PlanAndTrack(); - Check(words[0].ResolveInstruction()->GetOpcode() == ValueOpcode::Phi, - "guarded host descriptor selection rewrote the GPU Phi"); - return ExtractResourcePlan(fixture.program); - }; - for (const bool take_true : {false, true}) { - for (const bool reverse_phi : {false, true}) { - auto plan = make_plan(take_true, reverse_phi, false, false, false); - const std::array user_data{0x444u, 0x555u, 0x666u, 0x777u}; - DescriptorValue selected; - Check(SrtWalker(plan, {.user_data = user_data}).EvaluateDescriptor( - plan.info.samplers[0].source, selected) && - std::equal(user_data.begin(), user_data.end(), selected.dwords.begin()), - "guarded sampler did not match descriptor and predicate predecessors"); - } - } - CheckFatal([&] { make_plan(true, false, true, false, false); }, - "not a valid runtime value", "sampler guard accepted an unguarded path"); - CheckFatal([&] { make_plan(true, false, false, true, false); }, - "not a valid runtime value", "sampler guard ignored predecessor identity"); - CheckFatal([&] { make_plan(true, false, false, false, true); }, - "not a valid runtime value", "sampler guard discarded a reachable alternative"); -} - void TestLoopCycleEnteredThroughRuntimeValue() { Fixture fixture; auto *entry = fixture.block; @@ -2252,11 +2495,24 @@ void TestLoopCycleEnteredThroughRuntimeValue() { {Value(&phi), Value(0xffffffffu)}, 0, loop); phi.AddPhiOperand(entry, initial); phi.AddPhiOperand(loop, carried); - fixture.Emit(ValueOpcode::GetBufferResource, + const auto handle = fixture.Emit(ValueOpcode::GetBufferResource, {carried, Value(0u), Value(0u), Value(0u)}, MemoryFlags{0, 12}, loop); - - BuildSrtPlan(fixture.program); + MemoryInfo memory; + memory.kind = ResourceKind::Buffer; + fixture.Emit(ValueOpcode::LoadBufferU32, + {handle, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(memory, 16), loop); + ConstantPropagationPass(fixture.program.blocks); + RemoveIdentities(fixture.program.blocks); + fixture.PlanAndTrack(); + const std::array user_data{0x4000u}; + DescriptorValue descriptor; + Check(fixture.program.info.buffers.size() == 1u && + SrtWalker(fixture.program, {.user_data = user_data}).EvaluateDescriptor( + fixture.program.info.buffers[0].source, descriptor) && + descriptor.dwords[0] == user_data[0], + "runtime-rooted invariant loop lost its buffer source"); } void TestInvariantLoopPhi() { @@ -2600,14 +2856,15 @@ void TestShaderInfoAndBindingLayout() { fixture.Emit(ValueOpcode::LoadBufferU32, {handle, Value(0u), Value(0u), Value(0u), Value(true)}, fixture.AddMemory(buffer, 4)); - fixture.Emit( + const auto invocation = fixture.Emit( ValueOpcode::GetBuiltin, {Value(static_cast(StageInputKind::GlobalInvocationId)), Value(2u)}); - fixture.Emit(ValueOpcode::BitwiseXor32, {Value(1u), Value(2u)}); + const auto value = fixture.Emit(ValueOpcode::BitwiseXor32, + {invocation, Value(2u)}); MemoryInfo gds; gds.kind = ResourceKind::Gds; - fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), Value(1u), Value(true)}, + fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), value, Value(true)}, fixture.AddMemory(gds, 8)); fixture.PlanAndTrack(); @@ -2851,8 +3108,8 @@ void TestResourceLimitIsTransactional() { {handle, Value(0u), Value(0u), Value(0u), Value(true)}, fixture.AddMemory(memory, index * 4u)); } - BuildSrtPlan(fixture.program); - CheckFatal([&] { TrackResources(fixture.program); }, + + CheckFatal([&] { fixture.PlanAndTrack(); }, "buffer resource limit exceeded", "resource-limit failure was not reported"); Check(!fixture.program.resource_tracking_complete && @@ -2870,9 +3127,9 @@ void TestMalformedMemoryKindsRejected() { fixture.Emit(ValueOpcode::StoreAddressU32, {address, Value(0u), Value(0u), Value(1u), Value(true)}, fixture.AddMemory(memory, 4)); - BuildSrtPlan(fixture.program); + CheckFatal( - [&] { TrackResources(fixture.program); }, + [&] { fixture.PlanAndTrack(); }, "address operation has invalid resource kind", "resource tracking accepted an address opcode with buffer metadata"); } @@ -2887,9 +3144,9 @@ void TestMalformedMemoryKindsRejected() { fixture.Emit(ValueOpcode::ImageRead, {image, fixture.ImageAddress(), Value(true)}, fixture.AddMemory(memory, 8)); - BuildSrtPlan(fixture.program); + CheckFatal( - [&] { TrackResources(fixture.program); }, + [&] { fixture.PlanAndTrack(); }, "image operation has invalid resource kind", "resource tracking accepted an image opcode with address metadata"); } @@ -2924,7 +3181,6 @@ int main() { Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit); Run("phi validation", TestPhiValidation); Run("conditional sampler phi", TestConditionalSamplerPhi); - Run("guarded sampler phi", TestGuardedSamplerPhi); Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue); Run("invariant loop phi", TestInvariantLoopPhi); Run("DMA address materialization", TestDmaAddressMaterialization); @@ -2981,3 +3237,4 @@ void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); } #include "graphics/shader/recompiler/ir/Value.cpp" #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp" +#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp" diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index 3dc1e3bf4..0144fdb50 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -2,6 +2,7 @@ #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" +#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include @@ -71,7 +72,7 @@ struct Fixture { return static_cast(program.memory_info.size() - 1u); } - void Plan() { BuildSrtPlan(program); } + void Plan() { TrackResources(program, {}, {}); } }; struct TestMemory { @@ -102,6 +103,16 @@ Value RawRead(Fixture &fixture, Value address, Value offset, uint32_t memory, 0x80, block); } +void LoadBuffer(Fixture &fixture, std::array words) { + const auto handle = fixture.Emit(ValueOpcode::GetBufferResource, + {words[0], words[1], words[2], words[3]}); + const auto loaded = fixture.EmitMemory( + ValueOpcode::LoadBufferU32, + {handle, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(ResourceKind::Buffer)); + fixture.Emit(ValueOpcode::ReferenceU32, {loaded}); +} + void TestImmediateFlatteningAndGvn() { Fixture fixture; const auto memory = fixture.AddMemory(ResourceKind::ScalarAddress, 0x20); @@ -109,14 +120,11 @@ void TestImmediateFlatteningAndGvn() { fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory); const auto second = RawRead( fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory); - fixture.Emit(ValueOpcode::GetBufferResource, - {first, second, Value(16u), Value(0u)}); + LoadBuffer(fixture, {first, second, Value(16u), Value(0u)}); fixture.Plan(); Check(fixture.program.srt_reads.size() == 1, "equivalent typed scalar reads were not coalesced"); - Check(fixture.program.dynamic_reads.empty(), - "immediate scalar read was classified as dynamic"); Check(fixture.program.memory_info[memory].planning_only, "flattened raw read was not kept as a planning-only root"); @@ -133,8 +141,7 @@ void TestRawScalarComponentAlignment() { const auto memory = fixture.AddMemory(ResourceKind::ScalarAddress, 2); const auto read = RawRead( fixture, Address(fixture, Value(0x1003u), Value(0u)), Value(2u), memory); - fixture.Emit(ValueOpcode::GetBufferResource, - {read, Value(0u), Value(16u), Value(0u)}); + LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)}); fixture.Plan(); TestMemory memory_image{{{0x1000u, 0x12345678u}}}; @@ -150,7 +157,7 @@ void TestScalarMemoryDomainMismatchFails() { Fixture raw; const auto raw_memory = raw.AddMemory(ResourceKind::ScalarBuffer); RawRead(raw, Address(raw, Value(0x1000u), Value(0u)), Value(0u), raw_memory); - CheckFatal([&] { BuildSrtPlan(raw.program); }, + CheckFatal([&] { TrackResources(raw.program, {}, {}); }, "incompatible scalar memory metadata", "raw scalar load accepted descriptor-buffer metadata"); @@ -161,7 +168,7 @@ void TestScalarMemoryDomainMismatchFails() { {Value(0x1000u), Value(0u), Value(16u), Value(0u)}); buffer.EmitMemory(ValueOpcode::ReadConstBuffer, {resource, Value(0u)}, buffer_memory); - CheckFatal([&] { BuildSrtPlan(buffer.program); }, + CheckFatal([&] { TrackResources(buffer.program, {}, {}); }, "incompatible scalar memory metadata", "descriptor scalar load accepted raw-address metadata"); } @@ -173,12 +180,12 @@ void TestDynamicReadRemainsTyped() { {Value(static_cast(2))}); const auto read = RawRead( fixture, Address(fixture, Value(0x1000u), Value(0u)), offset, memory); - fixture.Emit(ValueOpcode::GetBufferResource, - {read, Value(0u), Value(16u), Value(0u)}); + LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)}); fixture.Plan(); Check(fixture.program.srt_reads.empty() && - fixture.program.dynamic_reads == std::vector{read}, + read.ResolveInstruction()->GetOpcode() == ValueOpcode::LoadAddressU32 && + !fixture.program.memory_info[memory].planning_only, "dynamic scalar read received a fake flattened slot"); } @@ -189,8 +196,7 @@ void TestNestedSrtWalk() { fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory); const auto value = RawRead(fixture, Address(fixture, pointer, Value(0u)), Value(0u), memory); - fixture.Emit(ValueOpcode::GetBufferResource, - {value, Value(0u), Value(16u), Value(0u)}); + LoadBuffer(fixture, {value, Value(0u), Value(16u), Value(0u)}); fixture.Plan(); TestMemory memory_image{{{0x1000u, 0x2000u}, {0x2000u, 0xabcdef01u}}}; @@ -211,7 +217,7 @@ void TestShaderBaseAndUserData() { const auto user = fixture.Emit(ValueOpcode::GetUserData, {Value(static_cast(2))}); const auto sum = fixture.Emit(ValueOpcode::IAdd32, {user, Value(4u)}); - fixture.Plan(); + fixture.program.descriptor_sources.push_back( {.dwords = {low, high, sum}, .dword_count = 3}); @@ -239,7 +245,7 @@ void TestCarryAndBitFields() { {Value(0u), Value(0x89abcdefu), Value(0u), Value(32u)}); const auto sign = fixture.Emit(ValueOpcode::BitFieldSExtract, {Value(0x000000f0u), Value(4u), Value(4u)}); - fixture.Plan(); + fixture.program.descriptor_sources.push_back( {.dwords = {low, high, inserted, sign}, .dword_count = 4}); @@ -261,7 +267,7 @@ void TestInvariantAndDivergentPhi() { divergent.SetFlags(Type::U32); divergent.AddPhiOperand(&fixture.BlockAt(0), Value(7u)); divergent.AddPhiOperand(&fixture.BlockAt(1), Value(9u)); - fixture.Plan(); + fixture.program.descriptor_sources.push_back( {.dwords = {Value(&invariant)}, .dword_count = 1}); fixture.program.descriptor_sources.push_back( @@ -285,6 +291,7 @@ void TestControlDependentStandaloneLoadStaysTyped() { const auto read = RawRead(fixture, Address(fixture, Value(&base), Value(0u), 2), Value(0u), memory, 2); + fixture.Emit(ValueOpcode::ReferenceU32, {read}, 0, 2); fixture.Plan(); Check(fixture.program.srt_reads.empty() && read.ResolveInstruction()->GetOpcode() == @@ -307,7 +314,7 @@ void TestRuntime64BitDescriptorOps() { fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(0u)}); const auto high = fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(1u)}); - fixture.Plan(); + fixture.program.descriptor_sources.push_back( {.dwords = {low, high}, .dword_count = 2}); @@ -374,7 +381,6 @@ void TestUniformFirstLaneSamplerLod() { {packed, active}); const auto divergent = fixture.Emit( ValueOpcode::ReadFirstLane, {stale, active}); - fixture.Plan(); Check(ValidateRuntimeValue(fixture.program, first), "uniform sampler LOD construction was rejected"); @@ -396,6 +402,49 @@ void TestUniformFirstLaneSamplerLod() { "uniform sampler LOD clamp evaluated incorrectly"); } +void TestFloatComparisonDescriptorInputs() { + struct Input { + uint32_t bits; + uint32_t less_equal; + uint32_t greater_equal; + }; + constexpr std::array inputs = { + Input{0x00000001u, 0, 1}, Input{0x80000001u, 1, 0}, + Input{0x007fffffu, 0, 1}, Input{0x807fffffu, 1, 0}, + Input{0x00800000u, 0, 1}, Input{0x80800000u, 1, 0}, + Input{0x00000000u, 1, 1}, Input{0x80000000u, 1, 1}, + Input{0x7fc00000u, 0, 0}}; + for (const bool flush : {false, true}) { + Fixture fixture; + const auto user = fixture.Emit( + ValueOpcode::GetUserData, {Value(static_cast(2))}); + const auto value = fixture.Emit(ValueOpcode::BitCastF32U32, {user}); + const auto less_equal = fixture.Emit(ValueOpcode::FPOrdLessThanEqual32, + {value, Value::F32(0.0f)}); + const auto greater_equal = fixture.Emit(ValueOpcode::FPOrdGreaterThanEqual32, + {value, Value::F32(0.0f)}); + less_equal.Instruction()->SetFlags(FPCompareFlags{flush}); + greater_equal.Instruction()->SetFlags(FPCompareFlags{flush}); + const auto low = fixture.Emit(ValueOpcode::SelectU32, + {less_equal, Value(1u), Value(0u)}); + const auto high = fixture.Emit(ValueOpcode::SelectU32, + {greater_equal, Value(1u), Value(0u)}); + fixture.program.descriptor_sources.push_back( + {.dwords = {low, high}, .dword_count = 2}); + + for (size_t index = 0; index < inputs.size(); index++) { + const auto &input = inputs[index]; + const std::array user_data{input.bits}; + DescriptorValue result; + Check(SrtWalker(fixture.program, {.user_data = user_data}) + .EvaluateDescriptor(0, result) && + result.dwords[0] == (flush && index < 4 ? 1 : input.less_equal) && + result.dwords[1] == (flush && index < 4 ? 1 : input.greater_equal), + "descriptor comparison disagrees with native FP32 input mode"); + } + } +} + void TestSharedIntegerRuntimeDependencies() { Fixture fixture; const auto lane = fixture.Emit(ValueOpcode::LaneId); @@ -439,37 +488,34 @@ void TestSharedIntegerRuntimeDependencies() { } void TestConstantBufferBounds() { - Fixture fixture; - const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer); - const auto buffer = - fixture.Emit(ValueOpcode::GetBufferResource, - {Value(0x3000u), Value(0u), Value(16u), Value(0u)}); - const auto read = fixture.EmitMemory(ValueOpcode::ReadConstBuffer, - {buffer, Value(12u)}, memory); - fixture.Emit(ValueOpcode::GetBufferResource, - {read, Value(0u), Value(16u), Value(0u)}); - fixture.Plan(); + struct Case { + uint32_t offset; + uint32_t immediate; + bool valid; + uint32_t expected; + }; + for (const auto &test : {Case{12u, 0u, true, 0xa5a5a5a5u}, + Case{3u, 1u, true, 0x12345678u}, + Case{0xfffffffcu, 4u, false, 0u}, + Case{16u, 0u, false, 0u}}) { + Fixture fixture; + const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer, test.immediate); + const auto buffer = + fixture.Emit(ValueOpcode::GetBufferResource, + {Value(0x3000u), Value(0u), Value(16u), Value(0u)}); + const auto read = fixture.EmitMemory(ValueOpcode::ReadConstBuffer, + {buffer, Value(test.offset)}, memory); + LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)}); + fixture.Plan(); - TestMemory memory_image{{{0x300cu, 0xa5a5a5a5u}}}; - SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; - std::vector flat; - Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat) && - flat == std::vector{0xa5a5a5a5u}, - "constant-buffer SRT walk failed"); - - Fixture overflow; - const auto overflow_memory = overflow.AddMemory(ResourceKind::ScalarBuffer); - const auto overflow_buffer = - overflow.Emit(ValueOpcode::GetBufferResource, - {Value(0x3000u), Value(0u), Value(16u), Value(0u)}); - const auto overflow_read = - overflow.EmitMemory(ValueOpcode::ReadConstBuffer, - {overflow_buffer, Value(16u)}, overflow_memory); - overflow.Emit(ValueOpcode::GetBufferResource, - {overflow_read, Value(0u), Value(16u), Value(0u)}); - overflow.Plan(); - Check(!SrtWalker(overflow.program, runtime).RefreshFlatBuffer(flat), - "out-of-bounds constant-buffer walk was accepted"); + TestMemory memory_image{{{0x3000u, 0x12345678u}, {0x300cu, 0xa5a5a5a5u}}}; + SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; + std::vector flat; + Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat) == test.valid, + "constant-buffer walk misaligned or wrapped its offset components"); + Check(test.valid ? flat == std::vector{test.expected} : memory_image.reads == 0, + "constant-buffer walk read the wrong word or accessed an out-of-bounds address"); + } } void TestReadLaneElimination() { @@ -552,7 +598,7 @@ void TestControlFlowValueSurvivesReadLaneFolding() { void TestUndefinedRuntimeValueFails() { Fixture fixture; const auto undef = fixture.Emit(ValueOpcode::UndefU32); - fixture.Plan(); + fixture.program.descriptor_sources.push_back( {.dwords = {undef}, .dword_count = 1}); DescriptorValue result; @@ -591,6 +637,7 @@ int main() { TestControlDependentStandaloneLoadStaysTyped(); TestRuntime64BitDescriptorOps(); TestUniformFirstLaneSamplerLod(); + TestFloatComparisonDescriptorInputs(); TestSharedIntegerRuntimeDependencies(); TestConstantBufferBounds(); TestReadLaneElimination(); @@ -611,6 +658,7 @@ int main() { #include "../src/graphics/shader/recompiler/ir/Program.cpp" #include "../src/graphics/shader/recompiler/ir/Type.cpp" #include "../src/graphics/shader/recompiler/ir/Value.cpp" +#include "../src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp" #include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" #include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp" #include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp" diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index fd7d85a2f..b2f64353a 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -9584,6 +9584,83 @@ public: std::printf("[gpu] %-32s ok\n", name); } + void CheckSamplerBorderColors() { + constexpr const char *name = "SamplerBorderColors"; + auto &context = RuntimeRenderer(); + constexpr std::array, 5> coordinates{{ + {{0.5f, -1.f}}, {{-1.f, 0.5f}}, {{2.f, 0.5f}}, + {{0.5f, 2.f}}, {{0.5f, 0.5f}}}}; + std::array float_samplers{}; + for (const bool integer : {false, true}) { + auto image = CreateImage2D(name, 2, 2, + integer ? vk::Format::eR8G8B8A8Uint : vk::Format::eR8G8B8A8Unorm, + vk::ImageUsageFlagBits::eSampled, std::vector(4, 0xffffffffu), 1, + vk::ImageLayout::eShaderReadOnlyOptimal); + for (const auto border : {Prospero::SamplerBorderColor::kOpaqueWhite, + Prospero::SamplerBorderColor::kOpaqueBlack, + Prospero::SamplerBorderColor::kTransBlack}) { + ShaderSamplerResource sampler_descriptor{}; + const auto clamp = static_cast(Prospero::SamplerClampMode::kClampBorder); + sampler_descriptor.fields[0] = clamp | (clamp << 3u); + sampler_descriptor.fields[2] = 1u << 24u; + sampler_descriptor.fields[3] = static_cast(border) << 30u; + TestCase test; + test.name = name; + test.has_user_data = true; + test.user_data = MakeSampledTextureData(integer ? Prospero::BufferFormat::k8_8_8_8UInt + : Prospero::BufferFormat::k8_8_8_8UNorm); + test.user_data[2] = 1u | (1u << 14u); + std::copy_n(sampler_descriptor.fields, 4, test.user_data.begin() + 8); + test.user_data[50] = coordinates.size() * 4u * sizeof(u32); + test.user_data[51] = 3u << 28u; + test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE, + ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM}; + for (u32 sample = 0; sample < coordinates.size(); ++sample) { + AppendVMovLiteral(&test.code, 20, std::bit_cast(coordinates[sample][0])); + AppendVMovLiteral(&test.code, 21, std::bit_cast(coordinates[sample][1])); + test.code.push_back(EncodeMimg0(0x20, 0xf)); + test.code.push_back(EncodeMimg1(0, 20, 0, 2)); + for (u32 channel = 0; channel < 4; ++channel) { + AppendStoreVgpr(&test.code, channel, sample * 4u + channel); + const bool inside = sample == coordinates.size() - 1u; + const bool one = inside || border == Prospero::SamplerBorderColor::kOpaqueWhite || + (border == Prospero::SamplerBorderColor::kOpaqueBlack && channel == 3u); + test.expected.push_back(one ? (integer ? (inside ? 255u : 1u) : 0x3f800000u) : 0u); + } + } + AppendEnd(&test.code); + const auto compiled = CompileCase(test, SubgroupSize()); + const auto &sampler_info = compiled.program.info.samplers[0]; + Require(name, "sampler specialization", sampler_info.integer_border == integer, + "sampled image numeric type did not select its sampler border class"); + auto output = CreateStorageBuffer(name, {}, test.expected.size()); + const auto sampler = context.GetSamplerCache().GetSampler( + sampler_descriptor, sampler_info.integer_border); + Require(name, "sampler reuse", + sampler == context.GetSamplerCache().GetSampler(sampler_descriptor, integer), + "identical typed sampler descriptors did not reuse the cached sampler"); + if (integer) { + Require(name, "typed border cache key", sampler != float_samplers[static_cast(border)], + "integer image reused the floating-point border sampler"); + } else { + float_samplers[static_cast(border)] = sampler; + } + Dispatch(test, compiled, output, nullptr, &image, nullptr, nullptr, sampler); + const auto actual = ReadBuffer(name, output, test.expected.size()); + for (u32 i = 0; i < actual.size(); ++i) { + if (actual[i] != test.expected[i]) { + Fail(name, "native border sample", "integer=" + std::to_string(integer) + + " border=" + std::to_string(static_cast(border)) + " word=" + std::to_string(i) + + " expected=" + std::to_string(test.expected[i]) + " actual=" + std::to_string(actual[i])); + } + } + DestroyBuffer(&output); + } + DestroyImage(&image); + } + std::printf("[gpu] %-32s ok\n", name); + } + void CheckComparisonDepthTexture() { constexpr const char *name = "ComparisonDepthTexture"; constexpr uintptr_t base = 0x0000000204200000ull; @@ -9702,7 +9779,8 @@ public: scheduler.Finish(); auto output = CreateStorageBuffer(name, {}, test.expected.size()); - const auto sampler = context.GetSamplerCache().GetSampler(sampler_descriptor); + const auto sampler = context.GetSamplerCache().GetSampler( + sampler_descriptor, compiled.program.info.samplers[0].integer_border); Image sampled; sampled.view = view; sampled.layout = image.backing.state.layout; @@ -10713,7 +10791,8 @@ public: texture_cache, scheduler.Current(), lod_binding.image_id, {vk::ImageAspectFlagBits::eColor, mip, 1, 0, 1}, clear); } - const auto sampler = context.GetSamplerCache().GetSampler(lod_sampler); + const auto sampler = context.GetSamplerCache().GetSampler( + lod_sampler, lod_program.program.info.samplers[0].integer_border); auto output = CreateStorageBuffer(lod_test.name, {}, 1); struct LodCase { uint32_t base_level; @@ -14373,6 +14452,40 @@ public: "a stale color target clipped stencil coverage or the NULL-export shader was skipped"); } + // A depth-only draw retains the previous PS address and write masks, but + // SPI_SHADER_COL_FORMAT disables its exports. Compiling this stale PS + // with the new zero-input metadata would fail before the depth draw. + static const auto stale_pixel = [] { + std::vector code{EncodeVintrp(0, 0, 0, 0, 0), + EncodeVintrp(1, 0, 0, 0, 1), + EncodeExp0(0, 0xf), EncodeExp1(0, 0, 0, 0)}; + AppendEnd(&code); + return code; + }(); + const auto stale_address = reinterpret_cast(stale_pixel.data()); + ShaderMapUserData(stale_address, + {.type = Prospero::ShaderBinaryType::kPs, + .code_size_bytes = static_cast(stale_pixel.size() * sizeof(u32))}); + shaders.SetPsShaderBase(stale_address); + registers.SetPsInControl(0); + registers.SetDepthShaderControl({}); + registers.SetDepthControl({.z_enable = true, .z_write_enable = true, + .zfunc = static_cast(vk::CompareOp::eAlways)}); + for (const auto slot : {0u, 3u}) { + registers.SetTargetOutputMode(slot, 0); + } + RenderExecutorTestAccess::DrawAuto( + executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1}); + const auto depth_only_pixels = ReadCachedTexel( + name, context, depth.image_id, {}, {extent, extent, 1}); + Require(name, "disabled stale PS preserves fixed-function depth", + std::ranges::all_of(depth_only_pixels, [](u32 v) { return v == 0; }), + "disabled color exports invoked the stale PS or clipped fixed-function depth"); + registers.SetPsInControl(0x8000); + for (const auto slot : {0u, 3u}) { + registers.SetTargetOutputMode(slot, 4); + } + // With MRT0 still bound, an MRT3-only export must retain location3 in // rendering attachments, pipeline formats, blend masks and dynamic write enables. static const auto sparse_pixel = [] { @@ -22148,6 +22261,87 @@ TestCase VectorSinCosMaxFiniteSpecialCases() { O::S_ENDPGM}}; } +TestCase VectorCompareF32ExposureGuard() { + using O = ShaderOpcode; + TestCase test; + test.name = "VectorCompareF32ExposureGuard"; + test.initial = {0x00000001u, 0x80000001u, 0x007fffffu, 0x807fffffu, + 0x00000000u, 0x80000000u, 0x3f800000u, 0x40000000u}; + test.expected = test.initial; + AppendVMovLiteral(&test.code, 1, 0x3f800000u); + for (u32 i = 0; i < test.initial.size(); i++) { + AppendVMovU32(&test.code, 31, i * sizeof(u32)); + AppendBufferLoadDword(&test.code, 0, 31); + // Captured exposure guard: replace zero with 1.0 before taking its reciprocal. + test.code.push_back(EncodeVopc(0x02, InlineU32(0), 0)); + test.code.push_back(EncodeVop2(0x01, 2, Vgpr(0), 1)); + test.code.push_back(EncodeVop1(0x2a, 3, Vgpr(2))); + AppendStoreVgpr(&test.code, 2, static_cast(test.expected.size())); + AppendStoreVgpr(&test.code, 3, static_cast(test.expected.size() + 1)); + test.expected.push_back(i == 7 ? 0x40000000u : 0x3f800000u); + test.expected.push_back(i == 7 ? 0x3f000000u : 0x3f800000u); + } + test.initial.resize(test.expected.size()); + AppendEnd(&test.code); + test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F32, O::V_CNDMASK_B32, + O::V_RCP_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + +TestCase VectorCompareF32DenormalModes(u32 mode) { + using O = ShaderOpcode; + enum Relation : u32 { Less, Equal, Greater, Unordered }; + struct Pair { + u32 lhs; + u32 rhs; + Relation relation; + }; + constexpr Pair pairs[] = { + {0x00000001u, 0x00000000u, Equal}, + {0x80000001u, 0x80000000u, Equal}, + {0x00000000u, 0x00000001u, Equal}, + {0x80000000u, 0x80000001u, Equal}, + {0x007fffffu, 0x80000001u, Equal}, + {0x00800000u, 0x007fffffu, Greater}, + {0xbf800000u, 0x80000001u, Less}, + {0xff800000u, 0xbf800000u, Less}, + {0x7f800000u, 0x7f800000u, Equal}, + {0x7fc12345u, 0x00000000u, Unordered}, + {0x00000000u, 0x7fa12345u, Unordered}, + {0x00000000u, 0x80000000u, Equal}, + }; + TestCase test; + constexpr const char* names[] = {"VectorCompareF32DenormalModeC0", + "VectorCompareF32DenormalModeE0"}; + test.name = names[(mode >> 5u) & 1u]; + test.compute_info.float_mode = static_cast(mode); + test.has_compute_info = true; + for (const auto& pair : pairs) { + test.initial.insert(test.initial.end(), {pair.lhs, pair.rhs}); + } + test.expected = test.initial; + AppendVMovU32(&test.code, 1, 1); + for (u32 i = 0; i < std::size(pairs); i++) { + AppendVMovU32(&test.code, 31, i * 2u * sizeof(u32)); + AppendBufferLoadDword(&test.code, 2, 31); + AppendVMovU32(&test.code, 31, (i * 2u + 1u) * sizeof(u32)); + AppendBufferLoadDword(&test.code, 3, 31); + // The sixteen architectural predicates select subsets of LT/EQ/GT/unordered. + for (u32 predicate = 0; predicate < 16; predicate++) { + test.code.push_back(EncodeVopc(predicate, Vgpr(2), 3)); + test.code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 1)); + AppendStoreVgpr(&test.code, 4, static_cast(test.expected.size())); + test.expected.push_back((predicate >> pairs[i].relation) & 1u); + } + } + test.initial.resize(test.expected.size()); + AppendEnd(&test.code); + test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F32, O::V_CMP_LE_F32, + O::V_CMP_GE_F32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; + return test; +} + TestCase VectorCompareOps() { using O = ShaderOpcode; @@ -22498,6 +22692,8 @@ TestCase VectorCompareClassF32() { append_class(0x40000000u, 1u << 8u, true); append_class(0x7f800000u, 1u << 9u, true); append_class(0x40000000u, 1u << 3u, false); + append_class(0x00000001u, 1u << 7u, true); + append_class(0x80000001u, 1u << 4u, true); AppendVMovLiteral(&code, 0, 0x3f800000u); AppendVMovU32(&code, 2, 1u << 3u); @@ -23390,6 +23586,73 @@ TestCase SharedReturnKeepsSelectedValues() { return test; } +TestCase SiblingSharedExitKeepsCapturedConditions() { + using O = ShaderOpcode; + std::vector code = { + EncodeVop1(0x01, 5, 4), + EncodeVop2(0x1a, 5, InlineU32(6), 5), + EncodeVop2(0x25, 5, Vgpr(0), 5), // group * 64 + lane + EncodeSop2(0x0e, 6, 4, InlineU32(1)), + 0, // choose left or right + EncodeSMovB32(8, InlineU32(20)), + EncodeSop2(0x0e, 6, 4, InlineU32(2)), + 0, // left: skip nested exit + EncodeSop2(0x0e, 6, 4, InlineU32(4)), + 0, // nested left: shared exit + 0, // left -> normal return + EncodeSMovB32(8, InlineU32(30)), + EncodeSop2(0x0e, 6, 4, InlineU32(8)), + 0, // first right shared exit + EncodeSMovB32(8, InlineU32(40)), + EncodeSop2(0x0e, 6, 4, InlineU32(16)), + 0, // second right shared exit + EncodeSMovB32(8, InlineU32(50)), + EncodeSopc(0x06, InlineU32(0), InlineU32(1)), // overwrite SCC + }; + AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0); + AppendEnd(&code); + const auto shared_exit = static_cast(code.size()); + code.push_back(EncodeSop2(0x00, 8, 8, InlineU32(1))); // overwrite SCC again + AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0); + AppendEnd(&code); + code[4] = EncodeSopp(0x04, 11u - 5u); + code[7] = EncodeSopp(0x04, 10u - 8u); + code[9] = EncodeSopp(0x04, shared_exit - 10u); + code[10] = EncodeSopp(0x02, 17u - 11u); + code[13] = EncodeSopp(0x04, shared_exit - 14u); + code[16] = EncodeSopp(0x04, shared_exit - 17u); + + TestCase test; + test.name = "SiblingSharedExitKeepsCapturedConditions"; + test.code = std::move(code); + for (u32 group = 0; group < 32; ++group) { + u32 result = 50; + if (group & 1u) { + if ((group & 2u) && !(group & 4u)) result = 21; + } else if (!(group & 8u)) { + result = 31; + } else if (!(group & 16u)) { + result = 41; + } + test.expected.insert(test.expected.end(), 64, result); + } + test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, + O::S_AND_B32, O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_ADD_U32, + O::S_CBRANCH_SCC0, O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpSelectionMerge"}; + test.forbidden_spirv = {"OpSwitch"}; + test.compute_info.wave_size = 64; + test.compute_info.threads_num[0] = 64; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 4; + test.has_compute_info = true; + test.dispatch_x = 32; + return test; +} + TestCase BranchVccnzUsesWaveMask() { using O = ShaderOpcode; @@ -23509,6 +23772,35 @@ TestCase ScalarMemoryLoadVariants() { O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase ScalarBufferOffsetAlignmentAndCarry() { + using O = ShaderOpcode; + std::vector code; + AppendSMovLiteral(&code, 12, 0xfffffffcu); + code.push_back(EncodeSop2(0x00, 12, 12, 4)); + code.push_back(EncodeSmem0(0x08, 20)); + code.push_back(EncodeSmem1(4, 12)); + AppendSMovLiteral(&code, 12, 3u); + code.push_back(EncodeSop2(0x00, 12, 12, 4)); + code.push_back(EncodeSmem0(0x08, 21)); + code.push_back(EncodeSmem1(1, 12)); + AppendStoreSgpr(&code, 20, 0); + AppendStoreSgpr(&code, 21, 1); + AppendEnd(&code); + + TestCase test{"ScalarBufferOffsetAlignmentAndCarry", code, + {0x11111111u, 0x22222222u}, {0u, 0x11111111u}, + {O::S_MOV_B32, O::S_ADD_U32, O::S_BUFFER_LOAD_DWORD, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + test.forbidden_spirv = {"flattened_srt"}; + test.compute_info.threads_num[0] = 1; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.workgroup_register = 4; + test.has_compute_info = true; + return test; +} + TestCase ScalarLoadSignedImmediateOffsetAddsSoffset() { using O = ShaderOpcode; @@ -25536,6 +25828,52 @@ TestCase ScalarLoadAlignsComponentsAndMasksAddress() { {O::S_MOV_B32, O::S_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase ScalarLoadAlignsDynamicBase() { + using O = ShaderOpcode; + constexpr uint64_t GuestBase = 0x10000; + std::vector code; + AppendSMovLiteral(&code, 8, GuestBase); + AppendSMovLiteral(&code, 9, 0); + code.push_back(EncodeSop2(0x00, 8, 8, 4)); // group ID supplies each base low-bit pattern + AppendSMovLiteral(&code, 10, 7); + code.push_back(EncodeSmem0(0x01, 12, 4)); + code.push_back(EncodeSmem1(7, 10)); // all three address components align independently + code.push_back(EncodeVop1(0x01, 0, 12)); + code.push_back(EncodeVop1(0x01, 1, 13)); + code.push_back(EncodeVop1(0x01, 2, 4)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(3), 2)); + AppendBufferStoreDword(&code, 0, 3); + code.push_back(EncodeVop2(0x25, 3, InlineU32(4), 3)); + AppendBufferStoreDword(&code, 1, 3); + AppendEnd(&code); + + TestCase test; + test.name = "ScalarLoadAlignsDynamicBase"; + test.code = std::move(code); + test.initial.resize(20); + test.initial[18] = 0x12345678; + test.initial[19] = 0x9abcdef0; + test.expected = test.initial; + for (u32 group = 0; group < 4; ++group) { + test.expected[group * 2] = test.initial[18]; + test.expected[group * 2 + 1] = test.initial[19]; + } + test.bda_mappings = {{GuestBase, 64}}; + test.required_spirv = {"get_bda_pointer"}; + test.forbidden_spirv = {"flattened_srt"}; + test.opcodes = {O::S_MOV_B32, O::S_ADD_U32, O::S_LOAD_DWORDX2, O::V_MOV_B32, + O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.compute_info.wave_size = 32; + test.compute_info.threads_num[0] = 1; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.workgroup_register = 4; + test.has_compute_info = true; + test.dispatch_x = 4; + return test; +} + TestCase FlatVirtualAddressRebasesGuestAllocation() { using O = ShaderOpcode; @@ -27166,17 +27504,17 @@ TestCase DsAppendConsumeUsesEncodedLdsSelector() { using O = ShaderOpcode; std::vector code; - AppendSMovLiteral(&code, 124, 0x0000ffffu); + AppendSMovLiteral(&code, 124, 0xffff0000u); AppendVMovU32(&code, 1, 0); AppendVMovU32(&code, 2, 10); code.push_back(EncodeDs0(0x0d, 0)); code.push_back(EncodeDs1(0, 2, 1)); - code.push_back(EncodeDs0(0x3e, 0)); + code.push_back(EncodeDs0(0x3e, 1)); code.push_back(EncodeDs1(3, 0, 0)); - code.push_back(EncodeDs0(0x3d, 0)); + code.push_back(EncodeDs0(0x3d, 2)); code.push_back(EncodeDs1(4, 0, 0)); AppendSMovLiteral(&code, 124, 0); - code.push_back(EncodeDs0(0x3e, 0)); + code.push_back(EncodeDs0(0x3e, 3)); code.push_back(EncodeDs1(5, 0, 0)); code.push_back(EncodeDs0(0x36, 0)); code.push_back(EncodeDs1(6, 0, 1)); @@ -27189,11 +27527,64 @@ TestCase DsAppendConsumeUsesEncodedLdsSelector() { return {"DsAppendConsumeLdsSelector", code, {}, - {10, 11, 0, 10}, + {10, 11, 10, 11}, {O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_B32, O::DS_APPEND, O::DS_CONSUME, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase DsAppendAllocatesAcrossWaves(u32 wave_size) { + using O = ShaderOpcode; + TestCase test; + test.name = wave_size == 64 ? "DsAppendAllocatesAcrossWaves64" + : "DsAppendAllocatesAcrossWaves32"; + auto &code = test.code; + AppendVMovU32(&code, 2, 0); + AppendVMovU32(&code, 5, 1); + // Only workgroup lane zero initializes the shared allocation counter. + code.push_back(EncodeSop1(0x04, 10, 126)); + code.push_back(EncodeVopc(0xc2, InlineU32(0), 0)); + code.push_back(EncodeSop1(0x04, 126, 106)); + code.push_back(EncodeDs0(0x0d, 0)); + code.push_back(EncodeDs1(0, 2, 2)); + code.push_back(EncodeSop1(0x04, 126, 10)); + code.push_back(EncodeSopp(0x0a, 0)); + for (u32 batch = 0; batch < 2; ++batch) { + // Captured CS ea0aceac sets M0=0 and uses OFFSET=1. LDS ignores M0 + // and aligns OFFSET before allocating once for every active wave. + AppendSMovLiteral(&code, 124, batch == 0 ? 0 : 0xffff0000u); + code.push_back(EncodeDs0(0x3e, batch == 0 ? 1 : 3)); + code.push_back(EncodeDs1(3, 0, 0)); + code.push_back(EncodeVop2(0x24, 6, 127, 3)); + code.push_back(EncodeVop2(0x23, 6, 126, 6)); + code.push_back(EncodeVop2(0x1a, 7, InlineU32(2), 6)); + // A histogram detects duplicate allocations without assuming wave order. + AppendBufferStoreOpcode(&code, 0x32, 5, 7); + } + code.push_back(EncodeSopp(0x0a, 0)); + code.push_back(EncodeDs0(0x36, 0)); + code.push_back(EncodeDs1(8, 0, 2)); + AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, 512); + AppendEnd(&code); + test.expected.assign(512, 1); + test.expected.resize(768, 512); + test.initial.resize(test.expected.size()); + test.compute_info.threads_num[0] = 256; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.compute_info.lds_size_dwords = 1; + test.has_compute_info = true; + test.forbidden_spirv = {"\"gds\""}; + test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_BARRIER, + O::V_MOV_B32, O::V_CMP_EQ_U32, O::DS_WRITE_B32, + O::DS_APPEND, O::V_MBCNT_HI_U32_B32, + O::V_MBCNT_LO_U32_B32, O::V_LSHLREV_B32, + O::BUFFER_ATOMIC_ADD, O::DS_READ_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + TestCase DsAppendUsesEncodedGdsSelector() { using O = ShaderOpcode; @@ -27231,6 +27622,46 @@ TestCase DsAppendUsesEncodedGdsSelector() { return test; } +TestCase DsAppendConsumeGdsRegionBounds() { + using O = ShaderOpcode; + struct Access { + u32 m0, offset; + bool valid; + }; + constexpr Access accesses[] = { + {0x00000004u, 3, true}, + {0x00000004u, 4, false}, + {0x00000003u, 0, false}, + {0xbffc0004u, 3, true}, + {0xbffc0008u, 0, false}, + {0xc0000004u, 0, false}, + {0x0000c000u, 0xbfff, true}, + {0x0000c000u, 0xc000, false}, + }; + TestCase test; + test.name = "DsAppendConsumeGdsRegionBounds"; + auto &code = test.code; + for (const auto &access : accesses) { + AppendSMovLiteral(&code, 124, access.m0); + code.push_back(EncodeDs0(0x3e, access.offset, true)); + code.push_back(EncodeDs1(0, 0, 0)); + code.push_back(EncodeDs0(0x3d, access.offset, true)); + code.push_back(EncodeDs1(1, 0, 0)); + AppendStoreVgpr(&code, 0, static_cast(test.expected.size())); + test.expected.push_back(access.valid ? 10 : 0); + AppendStoreVgpr(&code, 1, static_cast(test.expected.size())); + test.expected.push_back(access.valid ? 11 : 0); + } + AppendEnd(&code); + test.initial.resize(test.expected.size()); + // Backing beyond the guest partition must not make an invalid region valid. + test.gds_initial.assign(0x10000u / sizeof(u32), 10); + test.expected_gds = test.gds_initial; + test.opcodes = {O::S_MOV_B32, O::DS_APPEND, O::DS_CONSUME, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + TestCase DsGdsSubdwordAndAtomicWrites() { using O = ShaderOpcode; @@ -29446,6 +29877,10 @@ std::vector MakeCases() { AddCase(VectorF64CapturedScreenSpaceShadows); AddCase(VectorF64ModesModifiersAndExec); AddCase(VectorSinCosMaxFiniteSpecialCases); + AddCase(VectorCompareF32ExposureGuard); + for (const auto mode : {0xc0u, 0xe0u}) { + cases.push_back(VectorCompareF32DenormalModes(mode)); + } AddCase(VectorCompareOps); AddCase(VectorVop3CompareEqI64OnGpu); AddCase(VectorVop3CompareEqU64OnGpu); @@ -29477,12 +29912,15 @@ std::vector MakeCases() { AddCase(BranchSelect); AddCase(SimpleLoop); AddCase(SharedReturnKeepsSelectedValues); + AddCase(SiblingSharedExitKeepsCapturedConditions); AddCase(BranchVccnzUsesWaveMask); AddCase(BranchVccnzUsesCarryProducedWaveMask); AddCase(ScalarMemRealtimeCapturedPlaceholder); AddCase(ScalarMemoryLoadVariants); + AddCase(ScalarBufferOffsetAlignmentAndCarry); AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset); AddCase(ScalarLoadAlignsComponentsAndMasksAddress); + AddCase(ScalarLoadAlignsDynamicBase); AddCase(BufferLoadStore); AddCase(BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset); AddCase(BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset); @@ -29571,7 +30009,10 @@ std::vector MakeCases() { AddCase(DsReadU16D16CapturedPreservesHighHalf); AddCase(DsReadU16D16HiCapturedPreservesLowHalf); AddCase(DsAppendConsumeUsesEncodedLdsSelector); + cases.push_back(DsAppendAllocatesAcrossWaves(32)); + cases.push_back(DsAppendAllocatesAcrossWaves(64)); AddCase(DsAppendUsesEncodedGdsSelector); + AddCase(DsAppendConsumeGdsRegionBounds); AddCase(DsGdsSubdwordAndAtomicWrites); AddCase(DsReadWrite2Variants); AddCase(DsWideReadSnapshotsOverlappingAddress); @@ -31912,10 +32353,10 @@ void CheckImageSamplerSpecialization() { sampled_image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; sampled_image.read = true; mixed_sampler_program.info.images = {sampled_image, sampled_image, - sampled_image}; + sampled_image, sampled_image}; mixed_sampler_program.info.samplers.push_back({0u, 4u}); mixed_sampler_program.info.sampled_pairs = { - {0u, 0u, 8u}, {1u, 0u, 12u}, {2u, 0u, 16u}}; + {0u, 0u, 8u}, {1u, 0u, 12u}, {2u, 0u, 16u}, {3u, 0u, 20u}}; ShaderRecompiler::IR::MemoryInfo signed_memory; signed_memory.kind = ShaderRecompiler::IR::ResourceKind::Image; signed_memory.resource = 2u; @@ -31965,11 +32406,16 @@ void CheckImageSamplerSpecialization() { signed_image_descriptor.dwords[0] = 0x3000u; signed_image_descriptor.dwords[1] = static_cast(Prospero::BufferFormat::k32SInt) << 20u; + auto unsigned_image_descriptor = native_image_descriptor; + unsigned_image_descriptor.dwords[0] = 0x4000u; + unsigned_image_descriptor.dwords[1] = + static_cast(Prospero::BufferFormat::k32UInt) << 20u; ShaderRecompiler::IR::DescriptorValue sampler_descriptor{}; sampler_descriptor.dword_count = 4; const std::array descriptors{native_image_descriptor, packed_image_descriptor, - signed_image_descriptor, sampler_descriptor}; + signed_image_descriptor, unsigned_image_descriptor, + sampler_descriptor}; mixed_sampler_program.descriptor_sources.resize(descriptors.size()); for (u32 source = 0; source < descriptors.size(); source++) { auto &destination = mixed_sampler_program.descriptor_sources[source]; @@ -31981,7 +32427,7 @@ void CheckImageSamplerSpecialization() { for (u32 image = 0; image < mixed_sampler_program.info.images.size(); image++) { mixed_sampler_program.info.images[image].source = image; } - mixed_sampler_program.info.samplers[0].source = 3; + mixed_sampler_program.info.samplers[0].source = 4; mixed_sampler_program.srt_plan_complete = true; auto mixed_sampler_plan = ShaderRecompiler::IR::ExtractResourcePlan(mixed_sampler_program); @@ -31995,20 +32441,80 @@ void CheckImageSamplerSpecialization() { ShaderRecompiler::IR::ApplyResourceSpecialization( mixed_sampler_program, mixed_sampler_specialization); Require("ImageSamplerSpecialization", "mixed sampler variant", - mixed_sampler_program.info.samplers.size() == 2u && + mixed_sampler_program.info.samplers.size() == 3u && !mixed_sampler_program.info.samplers[0].force_point_filtering && - mixed_sampler_program.info.samplers[1].force_point_filtering && + !mixed_sampler_program.info.samplers[0].integer_border && + !mixed_sampler_program.info.samplers[1].force_point_filtering && + mixed_sampler_program.info.samplers[1].integer_border && + mixed_sampler_program.info.samplers[2].force_point_filtering && + mixed_sampler_program.info.samplers[2].integer_border && mixed_sampler_program.info.sampled_pairs[0].sampler == 0u && - mixed_sampler_program.info.sampled_pairs[1].sampler == 1u && - mixed_sampler_program.info.sampled_pairs[2].sampler == 1u && - mixed_sampler_program.memory_info[0].sampler == 1u && - mixed_sampler_snapshot.samplers.size() == 2u, - "a shared float/integer sampler was not split into point and native " - "variants or the signed instruction retained the native sampler"); + mixed_sampler_program.info.sampled_pairs[1].sampler == 2u && + mixed_sampler_program.info.sampled_pairs[2].sampler == 2u && + mixed_sampler_program.info.sampled_pairs[3].sampler == 1u && + mixed_sampler_program.memory_info[0].sampler == 2u && + mixed_sampler_snapshot.samplers.size() == 3u, + "a shared float/unsigned/signed sampler lost its border or filtering " + "class in the compiled bindings, instruction or descriptor snapshot"); std::printf("[host] %-32s ok\n", "ImageSpecializationPipelineId"); } +void CheckResourcePlanHandoff() { + constexpr const char *name = "ResourcePlanHandoff"; + using namespace ShaderRecompiler::IR; + for (const bool numeric_read : {false, true}) { + std::vector code{EncodeSmem0(0x02, 8), EncodeSmem1(0, 125), + EncodeSmem0(0x08, 12, 4), EncodeSmem1(0, 125)}; + if (numeric_read) { + code.insert(code.end(), {EncodeSmem0(0x00, 13), EncodeSmem1(16, 125)}); + AppendVop3(&code, 0x125u, 0, 12, 13); // v_add_nc_u32 v0, s12, s13 + } else { + code.push_back(EncodeVop1(0x01, 0, 12)); + } + code.insert(code.end(), {EncodeMubuf0(0x1c, 0, false, false), + EncodeMubuf1(0, 2, 0)}); + AppendEnd(&code); + + std::array user_data{}; + std::vector memory{0x1000u, 0u, 16u, 0x30000000u | DstSel(4, 5, 6, 7), 7u}; + ShaderComputeInputInfo compute{}; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.user_data = user_data; + options.input_info.compute = &compute; + auto translated = ShaderRecompiler::TranslateProgram(code, options); + auto plan = ExtractResourcePlan(translated.program); + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + const SrtRuntime runtime{.user_data = user_data, + .read_memory = ReadTestMemory, + .userdata = &memory}; + Require(name, "initial descriptors", + MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.buffers.size() == 1 && snapshot.buffers[0].dwords[0] == 0x1000u, + "SRT buffer descriptor was not materialized"); + auto compiled = ShaderRecompiler::CompileProgram(std::move(translated), options, + specialization); + ValidateSpirv(name, compiled.spirv); + const auto &bindings = compiled.program.bindings; + Require(name, "GPU data requirements", + bindings.user_data_registers.empty() && bindings.memory_offset_count == 1 && + (FindBinding(bindings, DescriptorBindingKind::FlattenedSrt) != nullptr) == + numeric_read, + "descriptor-only shader retained SRT uploads or live scalar data was removed"); + + memory[0] = 0x2000u; + memory[4] = 13u; + Require(name, "independent host plan", + MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.buffers[0].dwords[0] == 0x2000u && + (!numeric_read || snapshot.flattened_srt.back() == 13u), + "compiled shader cleanup invalidated or froze the host resource plan"); + } + std::printf("[host] %-32s ok\n", name); +} + void CheckNativeImageDescriptorTypes() { using ShaderRecompiler::IR::DescriptorBindingKind; using ShaderRecompiler::IR::FirstImageBinding; @@ -32981,7 +33487,7 @@ void CheckPm4Predication(RenderContext &renderer) { const auto predicated_address = reinterpret_cast(&predicated); const auto unconditional_address = reinterpret_cast(&unconditional); return std::array{ - // Captured AGC form: c0022000 00010100 . + // Captured packet form: c0022000 00010100 . 0xc0022000u, (op << 16u) | (wait << 12u) | (condition << 8u), static_cast(address), static_cast(address >> 32u), KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0) | 1u, 0, @@ -33217,7 +33723,7 @@ void CheckPm4NativeTargetGeometryRegisters(RenderContext &renderer) { std::printf("[host] %-32s ok\n", "Pm4NativeTargetGeometry"); } -void CheckPm4PrivateAgcShaderRegisters(RenderContext &renderer) { +void CheckPm4IndirectShaderRegisters(RenderContext &renderer) { GraphicsInitJmpTables(); CommandProcessor processor(renderer, 0); std::array registers{ @@ -33245,13 +33751,13 @@ void CheckPm4PrivateAgcShaderRegisters(RenderContext &renderer) { g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_GS] != nullptr && g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_CHKSUM_HS] != nullptr && g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_HS] != nullptr; - Require("Pm4PrivateAgcShaderRegisters", "private shader register stream", + Require("Pm4IndirectShaderRegisters", "indirect shader register stream", handlers_present && processor.Process(execution, command) == Pm4ProcessResult::Complete, - "private AGC shader registers were rejected by SET_SH_REG_INDIRECT"); + "shader registers were rejected by SET_SH_REG_INDIRECT"); const auto &vs = processor.GetShCtx().GetVs(); const auto &ps = processor.GetShCtx().GetPs().ps_regs; - Require("Pm4PrivateAgcShaderRegisters", "native state around front metadata", + Require("Pm4IndirectShaderRegisters", "native state around front metadata", vs.es_regs.data_addr == 0x50025e500ull && vs.gs_regs.rsrc1.vgprs == 2u && vs.gs_regs.rsrc1.gs_vgpr_component_count == 3u && @@ -33260,10 +33766,10 @@ void CheckPm4PrivateAgcShaderRegisters(RenderContext &renderer) { ps.data_addr == 0x500010400ull && ps.rsrc1.vgprs == 13u && ps.rsrc2.user_sgpr == 30u, "GS-front metadata changed native GS state or prevented later PS writes"); - std::printf("[host] %-32s ok\n", "Pm4PrivateAgcShaderRegisters"); + std::printf("[host] %-32s ok\n", "Pm4IndirectShaderRegisters"); } -void CheckPm4PrivateAgcUconfigRegisters(RenderContext &renderer) { +void CheckPm4IndirectUconfigRegisters(RenderContext &renderer) { GraphicsInitJmpTables(); CommandProcessor processor(renderer, 0); std::array registers{ @@ -33278,11 +33784,11 @@ void CheckPm4PrivateAgcUconfigRegisters(RenderContext &renderer) { Pm4Execution execution; const bool handler_present = g_hw_uc_indirect_func[Pm4::UC_PARAMETER_OVERSUBSCRIPTION] != nullptr; - Require("Pm4PrivateAgcUconfigRegisters", "private UCONFIG register stream", + Require("Pm4IndirectUconfigRegisters", "indirect UCONFIG register stream", handler_present && processor.Process(execution, command) == Pm4ProcessResult::Complete, - "private AGC UCONFIG register was rejected by SET_UCONFIG_REG_INDIRECT"); - std::printf("[host] %-32s ok\n", "Pm4PrivateAgcUconfigRegisters"); + "UCONFIG register was rejected by SET_UCONFIG_REG_INDIRECT"); + std::printf("[host] %-32s ok\n", "Pm4IndirectUconfigRegisters"); } void CheckPm4DirectShaderRegisterFallback(RenderContext &renderer) { @@ -33538,12 +34044,12 @@ void CheckPm4DepthRenderOverride(RenderContext &renderer) { std::printf("[host] %-32s ok\n", name); } -void CheckAgcShaderFusion() { +void CheckShaderFusion() { struct Case { bool gs; uint32_t front1, front2, back1, back2, fused1, fused2, older2; }; - // Register results from both exports in the supplied libSceAgc.sprx. + // Expected register results for both shader-fusion variants. constexpr Case cases[]{ {true, 0x600c0007, 0x00030010, 0x600c0007, 0x008b0000, 0x600c0007, 0x008b0010, 0x008b0010}, // Captured volume ES + GS. @@ -33585,7 +34091,7 @@ void CheckAgcShaderFusion() { const auto fuse = older ? Gen5::AgcUnknownNApJjpKNBl4 : Gen5::AgcUnknownFuseShaderHalves; const auto result = fuse(&fused, &front, &back, in_place ? nullptr : scratch.data()); const auto *regs = fused.sh_registers; - Require("AgcShaderFusion", "native fused registers", + Require("ShaderFusion", "native fused registers", result == 0 && regs == (in_place ? registers.data() : scratch.data()) && regs[0].value == 0x1234 && regs[1].value == 0x5678 && regs[2].value == c.fused1 && regs[3].value == (older ? c.older2 : c.fused2) && @@ -33594,14 +34100,14 @@ void CheckAgcShaderFusion() { : Prospero::ShaderBinaryType::kHs) && fused.user_data == (older ? &user_data : nullptr) && (in_place || std::memcmp(registers.data(), back_regs.data(), sizeof(registers)) == 0), - "AGC lost front resources or changed the export's allocation/metadata behavior"); + "shader fusion lost front resources or changed the export's allocation/metadata behavior"); } } } - std::printf("[host] %-32s ok\n", "AgcShaderFusion"); + std::printf("[host] %-32s ok\n", "ShaderFusion"); } -struct AgcCommandBufferLayout { +struct CommandBufferLayout { using Callback = KYTY_SYSV_ABI bool (*)(Gen5::CommandBuffer *, uint32_t, void *); @@ -33613,10 +34119,10 @@ struct AgcCommandBufferLayout { void *user_data; uint32_t reserved_dw; }; -static_assert(offsetof(AgcCommandBufferLayout, cursor_up) == 0x10); -static_assert(offsetof(AgcCommandBufferLayout, reserved_dw) == 0x30); +static_assert(offsetof(CommandBufferLayout, cursor_up) == 0x10); +static_assert(offsetof(CommandBufferLayout, reserved_dw) == 0x30); -void CheckAgcWaitPackets(RenderContext &renderer) { +void CheckPm4WaitPackets(RenderContext &renderer) { GraphicsInitJmpTables(); CommandProcessor processor(renderer, 0); constexpr auto packet_mismatch = static_cast(0x8a6c000cu); @@ -33628,18 +34134,18 @@ void CheckAgcWaitPackets(RenderContext &renderer) { alignas(8) uint32_t label32 = 0x11223344u; std::array packet32{}; - AgcCommandBufferLayout dcb32{packet32.data(), - packet32.data() + packet32.size(), - packet32.data(), - packet32.data() + packet32.size(), - nullptr, - nullptr, - 0}; + CommandBufferLayout dcb32{packet32.data(), + packet32.data() + packet32.size(), + packet32.data(), + packet32.data() + packet32.size(), + nullptr, + nullptr, + 0}; const auto address32 = reinterpret_cast(&label32); auto *emitted32 = Gen5::AgcDcbWaitRegMem(reinterpret_cast(&dcb32), 0, 3, 0, 0, &label32, label32, UINT32_MAX, 400); - Require("AgcWaitPackets", "native 32-bit packet", + Require("Pm4WaitPackets", "native 32-bit packet", Gen5::AgcDcbWaitOnAddressGetSize(0) == 56 && emitted32 == packet32.data() && dcb32.cursor_up == packet32.data() + 14 && @@ -33656,14 +34162,14 @@ void CheckAgcWaitPackets(RenderContext &renderer) { packet32[10] == 25u && packet32[11] == 0xc0017904u && packet32[12] == 0x342u && packet32[13] == 0xc8000000u && execute(packet32.data(), 14), - "32-bit waitOnAddress stream differs from native AGC"); + "32-bit wait packet differs from the expected stream"); alignas(8) uint32_t patched_label32 = 0xaabbccddu; const auto patched_address32 = reinterpret_cast(&patched_label32); packet32[6] |= 0x2u; packet32[7] |= 0xa5a40000u; Require( - "AgcWaitPackets", "native 32-bit patch", + "Pm4WaitPackets", "native 32-bit patch", Gen5::AgcWaitRegMemPatchAddress(packet32.data(), &patched_label32) == 0 && Gen5::AgcWaitRegMemPatchReference(packet32.data(), patched_label32) == 0 && @@ -33681,18 +34187,18 @@ void CheckAgcWaitPackets(RenderContext &renderer) { alignas(8) uint64_t label64 = 0x1122334455667788ull; std::array packet64{}; - AgcCommandBufferLayout dcb64{packet64.data(), - packet64.data() + packet64.size(), - packet64.data(), - packet64.data() + packet64.size(), - nullptr, - nullptr, - 0}; + CommandBufferLayout dcb64{packet64.data(), + packet64.data() + packet64.size(), + packet64.data(), + packet64.data() + packet64.size(), + nullptr, + nullptr, + 0}; const auto address64 = reinterpret_cast(&label64); auto *emitted64 = Gen5::AgcDcbWaitRegMem(reinterpret_cast(&dcb64), 1, 3, 0, 0, &label64, label64, UINT64_MAX, 400); - Require("AgcWaitPackets", "native 64-bit packet", + Require("Pm4WaitPackets", "native 64-bit packet", Gen5::AgcDcbWaitOnAddressGetSize(1) == 64 && Gen5::AgcDcbWaitOnAddressGetSize(2) == 0 && emitted64 == packet64.data() && @@ -33712,14 +34218,14 @@ void CheckAgcWaitPackets(RenderContext &renderer) { packet64[12] == 25u && packet64[13] == 0xc0017904u && packet64[14] == 0x342u && packet64[15] == 0xc8000000u && execute(packet64.data(), packet64.size()), - "64-bit waitOnAddress stream differs from native AGC"); + "64-bit wait packet differs from the expected stream"); alignas(8) uint64_t patched_label64 = 0x11223344aabbccddull; const auto patched_address64 = reinterpret_cast(&patched_label64); packet64[6] |= 0x5u; packet64[7] |= 0x5a580000u; Require( - "AgcWaitPackets", "native 64-bit patch", + "Pm4WaitPackets", "native 64-bit patch", Gen5::AgcWaitRegMemPatchAddress(packet64.data(), &patched_label64) == 0 && Gen5::AgcWaitRegMemPatchReference(packet64.data(), 0xdeadbeefaabbccddull) == 0 && @@ -33740,18 +34246,18 @@ void CheckAgcWaitPackets(RenderContext &renderer) { std::array invalid{}; const uint32_t short_payload = 0x342u; Require( - "AgcWaitPackets", "invalid packet", + "Pm4WaitPackets", "invalid packet", Gen5::AgcWaitRegMemPatchAddress(invalid.data(), &label32) == packet_mismatch && Gen5::AgcWaitRegMemPatchReference(invalid.data(), label32) == packet_mismatch && !Gen5::AgcIsInternalDataPacket( KYTY_PM4(2, Pm4::IT_SET_UCONFIG_REG, 1), &short_payload), - "native wait patch accepted a packet without the AGC metadata prefix"); - std::printf("[host] %-32s ok\n", "AgcWaitPackets"); + "wait patch accepted a packet without the metadata prefix"); + std::printf("[host] %-32s ok\n", "Pm4WaitPackets"); } -void CheckAgcDrawIndirectMultiPacket(RenderContext &renderer) { +void CheckPm4DrawIndirectMultiPacket(RenderContext &renderer) { GraphicsInitJmpTables(); CommandProcessor processor(renderer, 0); const auto execute = [&](uint32_t *packet, uint32_t size_dw) { @@ -33771,19 +34277,19 @@ void CheckAgcDrawIndirectMultiPacket(RenderContext &renderer) { 0x00000022u, 0xc0017904u, 0x00000342u, 0xc6000000u}; std::array packet{}; - AgcCommandBufferLayout dcb{packet.data(), - packet.data() + packet.size(), - packet.data(), - packet.data() + packet.size(), - nullptr, - nullptr, - 0}; + CommandBufferLayout dcb{packet.data(), + packet.data() + packet.size(), + packet.data(), + packet.data() + packet.size(), + nullptr, + nullptr, + 0}; auto *emitted = Gen5::AgcDcbDrawIndirectMulti( reinterpret_cast(&dcb), 0x11223344u, 1u, 0x55667788u, count_address, 0xaabbccddu, modifier); const uint32_t invalid_payload[]{0x342u, 0xc6000010u}; - Require("AgcDrawIndirectMulti", "native packet", + Require("Pm4DrawIndirectMulti", "native packet", emitted == packet.data() && dcb.cursor_up == packet.data() + packet.size() && packet == expected && execute(packet.data(), 3u) && @@ -33793,8 +34299,8 @@ void CheckAgcDrawIndirectMultiPacket(RenderContext &renderer) { Gen5::AgcWaitRegMemPatchReference(packet.data(), 0x12345678u) == packet_mismatch && !Gen5::AgcIsInternalDataPacket(0xc0017904u, invalid_payload), - "drawIndirectMulti stream differs from native AGC"); - std::printf("[host] %-32s ok\n", "AgcDrawIndirectMulti"); + "indirect multi-draw packet differs from the expected stream"); + std::printf("[host] %-32s ok\n", "Pm4DrawIndirectMulti"); } void CheckPm4ContextStateOperations(RenderContext &renderer) { @@ -33814,13 +34320,13 @@ void CheckPm4ContextStateOperations(RenderContext &renderer) { const auto invoke = [&](ContextStateOperation operation) { std::array packet{}; std::array segment_sizes{}; - AgcCommandBufferLayout dcb{packet.data(), - packet.data() + packet.size(), - packet.data(), - packet.data() + packet.size(), - nullptr, - nullptr, - 0}; + CommandBufferLayout dcb{packet.data(), + packet.data() + packet.size(), + packet.data(), + packet.data() + packet.size(), + nullptr, + nullptr, + 0}; const auto operation_value = static_cast(operation); size_t segment_count = 0; switch (operation) { @@ -33923,7 +34429,7 @@ void CheckPm4ContextStateOperations(RenderContext &renderer) { processor.GetCtx().GetRenderTargetMask() == 0x0badc0de, "CLEAR_STATE discarded the pushed Cx state"); - // AGC CLEAR_STATE restores the viewport, scissor, and guard-band register defaults. + // CLEAR_STATE restores the viewport, scissor, and guard-band register defaults. processor.GetCtx().SetScreenScissor(1, 2, 3, 4); processor.GetCtx().SetGuardBands(2.0f, 3.0f, 4.0f, 5.0f); Pm4Execution scissor_reset; @@ -33982,7 +34488,7 @@ void CheckPm4ContextStateOperations(RenderContext &renderer) { Gen5::AgcDcbContextStateOpGetSize(2) == 108 && Gen5::AgcDcbContextStateOpGetSize(3) == 128 && Gen5::AgcDcbContextStateOpGetSize(4) == 0, - "context-state HLE sizes do not match libSceAgc"); + "context-state HLE sizes do not match the expected packet layout"); std::printf("[host] %-32s ok\n", "Pm4ContextState"); } @@ -34254,6 +34760,16 @@ int main(int argc, char **argv) { RunCase(&vulkan, VectorVopcCmpxOrderedCapturedExecMask()); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--fp32-compare-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, VectorCompareF32ExposureGuard()); + for (const auto mode : {0xc0u, 0xe0u}) { + RunCase(&vulkan, VectorCompareF32DenormalModes(mode)); + } + RunCase(&vulkan, VectorCompareClassF32()); + RunCase(&vulkan, VectorCompareF16Ops()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--buffer-format-store-only") == 0) { VulkanHarness vulkan; RunCase(&vulkan, BufferStoreFormatXyzwPackedUnormSkinningVectors()); @@ -34395,6 +34911,7 @@ int main(int argc, char **argv) { RunCase(&vulkan, ImageSampleAndGather()); RunCase(&vulkan, ImageGatherLodApproximatesLevelZero()); RunCase(&vulkan, SharedReturnKeepsSelectedValues()); + RunCase(&vulkan, SiblingSharedExitKeepsCapturedConditions()); return 0; } if (argc == 2 && std::strcmp(argv[1], "--position-w-only") == 0) { @@ -34456,6 +34973,7 @@ int main(int argc, char **argv) { return 0; } if (argc == 2 && std::strcmp(argv[1], "--shader-data-storage-only") == 0) { + CheckResourcePlanHandoff(); VulkanHarness vulkan; RunCase(&vulkan, BufferOffsetsUsePackedLaneAndStorageFallback()); return 0; @@ -34593,10 +35111,10 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--context-state-only") == 0) { VulkanHarness vulkan; - CheckAgcShaderFusion(); + CheckShaderFusion(); CheckPm4NativeTargetGeometryRegisters(vulkan.RuntimeRenderer()); - CheckPm4PrivateAgcShaderRegisters(vulkan.RuntimeRenderer()); - CheckPm4PrivateAgcUconfigRegisters(vulkan.RuntimeRenderer()); + CheckPm4IndirectShaderRegisters(vulkan.RuntimeRenderer()); + CheckPm4IndirectUconfigRegisters(vulkan.RuntimeRenderer()); CheckPm4BlendColorRegisterRanges(vulkan.RuntimeRenderer()); CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer()); CheckPm4DepthControlHighBits(vulkan.RuntimeRenderer()); @@ -34604,15 +35122,15 @@ int main(int argc, char **argv) { CheckPm4ContextStateOperations(vulkan.RuntimeRenderer()); return 0; } - if (argc == 2 && std::strcmp(argv[1], "--agc-wait-only") == 0) { + if (argc == 2 && std::strcmp(argv[1], "--pm4-wait-only") == 0) { VulkanHarness vulkan; - CheckAgcWaitPackets(vulkan.RuntimeRenderer()); + CheckPm4WaitPackets(vulkan.RuntimeRenderer()); CheckPm4WaitResume(vulkan.RuntimeRenderer()); return 0; } - if (argc == 2 && std::strcmp(argv[1], "--agc-draw-multi-only") == 0) { + if (argc == 2 && std::strcmp(argv[1], "--pm4-draw-multi-only") == 0) { VulkanHarness vulkan; - CheckAgcDrawIndirectMultiPacket(vulkan.RuntimeRenderer()); + CheckPm4DrawIndirectMultiPacket(vulkan.RuntimeRenderer()); return 0; } if (argc == 2 && std::strcmp(argv[1], "--rewind-only") == 0) { @@ -34703,6 +35221,12 @@ int main(int argc, char **argv) { vulkan.CheckBufferCacheDirtyGarbageCollection(); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--sampler-border-only") == 0) { + CheckImageSamplerSpecialization(); + VulkanHarness vulkan; + vulkan.CheckSamplerBorderColors(); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--sampled-depth-resource-only") == 0) { VulkanHarness vulkan; CheckSampledDepthResource(); @@ -34793,6 +35317,7 @@ int main(int argc, char **argv) { CheckImageTransitionState(vulkan.RuntimeRenderer()); CheckSampledDepthResource(); CheckDepthTextureEncoding(); + vulkan.CheckSamplerBorderColors(); vulkan.CheckComparisonDepthTexture(); vulkan.CheckRasterization(true); CheckBasicStorageTextureDescriptor(); @@ -34820,6 +35345,7 @@ int main(int argc, char **argv) { VulkanHarness vulkan; #endif CheckImageSamplerSpecialization(); + CheckResourcePlanHandoff(); CheckNativeImageDescriptorTypes(); CheckClipControlDepthClipState(); CheckReferenceClockScale(); @@ -34830,17 +35356,17 @@ int main(int argc, char **argv) { CheckPm4Predication(vulkan.RuntimeRenderer()); CheckPm4StencilInfoValueLane(vulkan.RuntimeRenderer()); CheckPm4NativeTargetGeometryRegisters(vulkan.RuntimeRenderer()); - CheckPm4PrivateAgcShaderRegisters(vulkan.RuntimeRenderer()); - CheckPm4PrivateAgcUconfigRegisters(vulkan.RuntimeRenderer()); + CheckPm4IndirectShaderRegisters(vulkan.RuntimeRenderer()); + CheckPm4IndirectUconfigRegisters(vulkan.RuntimeRenderer()); CheckPm4DirectShaderRegisterFallback(vulkan.RuntimeRenderer()); CheckPm4GuardBandRegisterRanges(vulkan.RuntimeRenderer()); CheckPm4BlendColorRegisterRanges(vulkan.RuntimeRenderer()); CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer()); CheckPm4DepthControlHighBits(vulkan.RuntimeRenderer()); CheckPm4DepthRenderOverride(vulkan.RuntimeRenderer()); - CheckAgcShaderFusion(); - CheckAgcWaitPackets(vulkan.RuntimeRenderer()); - CheckAgcDrawIndirectMultiPacket(vulkan.RuntimeRenderer()); + CheckShaderFusion(); + CheckPm4WaitPackets(vulkan.RuntimeRenderer()); + CheckPm4DrawIndirectMultiPacket(vulkan.RuntimeRenderer()); CheckPm4ContextStateOperations(vulkan.RuntimeRenderer()); CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer()); CheckPm4WaitResume(vulkan.RuntimeRenderer()); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 084bfbc14..258c7cd8d 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -1818,12 +1818,12 @@ void TestSopkCompareImmediateExtension() { TranslateProgram(shader, MakeCompileOptions(ShaderType::Vertex)); const auto branch = std::ranges::find_if( translated.program.block_info, [](const auto &block) { - return block.terminator.condition == CFG::BranchCondition::SccZero; + return block.terminator.kind == CFG::TerminatorKind::ConditionalBranch; }); Check(branch != translated.program.block_info.end(), "captured LUT parameter export branch was lost"); const auto condition = branch->condition.Resolve(); - Check(condition.IsImmediate() && condition.U1() == (counts == 6u), + Check(condition.IsImmediate() && condition.U1() == (counts != 6u), "captured LUT unsigned compare suppressed a live parameter export"); } } @@ -3923,6 +3923,35 @@ void TestNewShaderRecompilerScalarB64LaneTranslation() { CheckSpirvBinaryValidates(result.spirv); } +void TestFloatComparisonInputModes() { + using namespace ShaderRecompiler; + const uint32_t shader[] = { + EncodeVopc(0x02, 256, 1), // v_cmp_eq_f32 v0, v1 + EncodeVopc(0xca, 256, 1), // v_cmp_eq_f16 v0, v1 + EncodeSopp(0x01), + }; + Decoder::Program decoded; + Decoder::DecodeProgram(shader, decoded); + for (const uint8_t mode : {0xc0, 0xd0, 0xe0, 0xf0}) { + ShaderComputeInputInfo compute{}; + compute.float_mode = mode; + Frontend::TranslateOptions options{.stage = ShaderType::Compute}; + options.input_info.compute = &compute; + const auto program = Frontend::TranslateProgram(decoded, CFG::BuildGraph(decoded), options); + uint32_t comparisons = 0; + for (const auto* block : program.blocks) { + for (const auto& inst : *block) { + if (inst.GetOpcode() != IR::ValueOpcode::FPOrdEqual32) continue; + Check(inst.Flags().flush_input_denorms == + (comparisons == 0 && (mode & 0x10u) == 0), + "compute input mode leaked into preserved or promoted F16 comparison"); + comparisons++; + } + } + Check(comparisons == 2, "comparison input-mode fixture did not translate both precisions"); + } +} + void TestNewShaderRecompilerSignedCompareAlu() { const uint32_t shader[] = { EncodeSMovB32(0, 193), // s0 = -1 @@ -4842,11 +4871,12 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { }; const uint32_t raw_shader[] = { - EncodeSmem0(0x00, 12, 4), - 2u, // s_load_dword s12, s[8:9], s0 offset:2 + EncodeSmem0(0x01, 12, 4), + 2u, // s_load_dwordx2 s[12:13], s[8:9], s0 offset:2 EncodeVop1(0x01, 0, 12), - EncodeExp0(0x00, 0x1), - EncodeExp1(0, 0, 0, 0), + EncodeVop1(0x01, 1, 13), + EncodeExp0(0x00, 0x3), + EncodeExp1(0, 1, 0, 0), EncodeSopp(0x01), }; std::array raw_user_data{}; @@ -4879,8 +4909,10 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { "raw scalar load did not use only the DMA domain"); Check(count_live_memory_ops( raw.program, ShaderRecompiler::IR::ValueOpcode::LoadAddressU32, - ShaderRecompiler::IR::ResourceKind::ScalarAddress) == 1u, + ShaderRecompiler::IR::ResourceKind::ScalarAddress) == 2u, "raw scalar load did not remain a live typed address operation"); + Check(SpirvInstructionOpcodeCount(raw.spirv, 57) == 2u, + "aligned scalar DWORDs must each use one BDA lookup"); Check(SpirvContainsOpcode(raw.spirv, 199), "raw scalar SOFFSET alignment was not emitted"); CheckSpirvBinaryValidates(raw.spirv); @@ -7746,8 +7778,8 @@ void TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeStructured() { const auto original_block_count = graph.blocks.size(); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(graph.natural_loops.size() == 1u, "DS loop was not preserved"); Check(graph.blocks.size() == original_block_count + 1u, "DS loop structurization did not add exactly one empty header"); @@ -7882,17 +7914,14 @@ void TestNewShaderRecompilerCfgLoopExitSharedWithSelection() { const auto coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); Check(block_count == 8u && graph.natural_loops.size() == 1u, "shared loop-exit fixture has the wrong native CFG"); - Check(!ShaderRecompiler::CFG::Structurize(graph) && - graph.unsupported_reason.find("duplicate structured merge block") != - std::string::npos, - "shared loop exit did not terminate at its structured merge conflict"); - Check(graph.blocks.size() == block_count && - CfgInstructionCoverage(graph, decoded.instructions.size()) == coverage, - "shared loop-exit fallback changed semantic instruction coverage"); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == coverage, + "shared loop-exit routing changed semantic instruction coverage"); auto result = RecompileForTest(shader, MakeCompileOptions(ShaderType::Compute)); - Check(result.program.dispatcher_fallback && SpirvContainsOpcode(result.spirv, 251), - "shared loop exit did not emit its dispatcher fallback"); + Check(!result.program.dispatcher_fallback && !SpirvContainsOpcode(result.spirv, 251), + "shared loop exit did not retain structured control flow"); CheckSpirvBinaryValidates(result.spirv); } @@ -7922,7 +7951,7 @@ void TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher() { +void TestNewShaderRecompilerCfgNestedLoopNonlocalExitStructured() { const uint32_t shader[] = { EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 EncodeSopp(0x04, 9), // outer exit -> end @@ -7942,10 +7971,10 @@ void TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher() { options.dump_ir = true; auto result = RecompileForTest(shader, options); - Check((result.ir_dump.find("mode=dispatcher") != std::string::npos), - "nested-loop nonlocal exit did not select dispatcher fallback"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, - "nested-loop nonlocal exit dispatcher SPIR-V lacks OpSwitch"); + Check((result.ir_dump.find("mode=structured") != std::string::npos), + "nested-loop nonlocal exit did not retain structured control flow"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "nested-loop nonlocal exit unexpectedly selected dispatcher OpSwitch"); CheckSpirvBinaryValidates(result.spirv); } @@ -8001,32 +8030,28 @@ void TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit() { ShaderRecompiler::CFG::Graph graph; graph = ShaderRecompiler::CFG::BuildGraph(program); const auto original_block_count = graph.blocks.size(); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(graph.blocks.size() > original_block_count, "nested loop exit tails did not create a private inner merge"); - const auto *outer_header = graph.FindBlockByPc(0); - const auto *inner_header = graph.FindBlockByPc(8); - Check(outer_header != nullptr && inner_header != nullptr && - outer_header->terminator.loop_header && - inner_header->terminator.loop_header, - "nested loop exit-tail fixture did not retain both loop headers"); - Check(inner_header->terminator.merge_block != - outer_header->terminator.continue_block, - "inner loop merge still aliases the outer continue target"); - const auto *inner_merge = - graph.FindBlock(inner_header->terminator.merge_block); - Check(inner_merge != nullptr && - inner_merge->inst_begin == inner_merge->inst_end && - inner_merge->terminator.kind == - ShaderRecompiler::CFG::TerminatorKind::Branch && - inner_merge->terminator.true_block == - outer_header->terminator.continue_block, - "private inner merge does not forward to the outer continue target"); + std::vector headers; + for (const auto &block : graph.blocks) + if (block.terminator.loop_header) headers.push_back(&block); + Check(headers.size() == 2u, "nested loop exit-tail fixture lost a loop header"); + if (!graph.Dominates(headers[0]->id, headers[1]->id)) std::swap(headers[0], headers[1]); + const auto *outer_header = headers[0]; + const auto *inner_header = headers[1]; + Check(inner_header->terminator.merge_block != outer_header->terminator.continue_block && + graph.Dominates(outer_header->id, inner_header->terminator.merge_block), + "inner loop merge escaped the enclosing loop body"); + auto result = RecompileForTest(shader, MakeCompileOptions(ShaderType::Compute)); + Check(!result.program.dispatcher_fallback, + "nested loop exit tails selected the dispatcher"); + CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher() { +void TestNewShaderRecompilerCfgMixedContinueNonmergeExitStructured() { const uint32_t shader[] = { EncodeSopc(0x06, 7, 7), // entry branch bypasses loop -> exit X EncodeSopp(0x05, 5), // entry -> X @@ -8046,10 +8071,10 @@ void TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher() { options.dump_ir = true; auto result = RecompileForTest(shader, options); - Check((result.ir_dump.find("mode=dispatcher") != std::string::npos), - "mixed continue/nonmerge exit did not select dispatcher fallback"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, - "mixed continue/nonmerge exit dispatcher SPIR-V lacks OpSwitch"); + Check((result.ir_dump.find("mode=structured") != std::string::npos), + "mixed continue/nonmerge exit did not retain structured control flow"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "mixed continue/nonmerge exit unexpectedly selected dispatcher OpSwitch"); CheckSpirvBinaryValidates(result.spirv); } @@ -8100,7 +8125,7 @@ void TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() { +void TestNewShaderRecompilerCfgLoopEarlyContinuesStructured() { const uint32_t shader[] = { EncodeSMovB32(0, 128), // s0 = 0 EncodeSopc(0x0a, 0, 130), // loop: s_cmp_lt_u32 s0, 2 @@ -8125,8 +8150,8 @@ void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() { "loop early continues should stay on structured path"); Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, "loop early continues SPIR-V lacks OpLoopMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, - "loop early continues SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 250) <= 4u, + "loop early continues introduced duplicate conditional branches"); Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, "loop early continues unexpectedly used dispatcher OpSwitch"); CheckSpirvBinaryValidates(result.spirv); @@ -8153,8 +8178,8 @@ void TestNewShaderRecompilerCfgLoopGatewaySelection() { graph = ShaderRecompiler::CFG::BuildGraph(decoded); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "loop gateway structurization duplicated semantic instructions"); @@ -8192,8 +8217,8 @@ void TestNewShaderRecompilerCfgConditionalLoopHeaderSelection() { ShaderRecompiler::CFG::Graph graph; graph = ShaderRecompiler::CFG::BuildGraph(decoded); const auto original_block_count = graph.blocks.size(); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(graph.blocks.size() > original_block_count, "conditional guest loop header did not create a synthetic header"); @@ -8242,29 +8267,30 @@ void TestNewShaderRecompilerCfgMultipleLoopLatches() { ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded); ShaderRecompiler::CFG::Graph graph; graph = ShaderRecompiler::CFG::BuildGraph(decoded); - const auto original_block_count = graph.blocks.size(); + const auto coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); Check(graph.back_edges.size() == 2u, "multiple-latch fixture lacks two native backedges"); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); - Check(graph.blocks.size() == original_block_count + 2u, - "multiple native latches did not create one synthetic continue and one " - "empty header"); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == coverage, + "shared backedge construction duplicated guest instructions"); Check(graph.back_edges.size() == 1u && graph.natural_loops.size() == 1u, "multiple native latches were not coalesced to one SPIR-V backedge"); const auto &loop = graph.natural_loops.front(); - const auto *continue_block = graph.FindBlock(loop.continue_block); + const auto *header = graph.FindBlock(loop.header); + Check(header != nullptr, "natural loop header is missing"); + const auto *continue_block = graph.FindBlock(header->terminator.continue_block); Check(continue_block != nullptr && - continue_block->inst_begin == continue_block->inst_end && - continue_block->predecessors.size() == 2u, - "canonical continue does not join both native latches"); + continue_block->terminator.true_block == loop.header && + graph.Dominates(loop.header, continue_block->id), + "canonical continue does not return to its natural loop header"); auto options = MakeCompileOptions(ShaderType::Compute); auto result = RecompileForTest(shader, options); Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, "multiple-latch SPIR-V has the wrong loop-merge count"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0u, - "multiple-latch SPIR-V unexpectedly used a selection merge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 250) <= 2u, + "multiple-latch routing duplicated a native conditional branch"); Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, "multiple-latch SPIR-V unexpectedly used dispatcher OpSwitch"); CheckSpirvBinaryValidates(result.spirv); @@ -8319,36 +8345,20 @@ void TestNewShaderRecompilerCfgNestedEarlyExitLoopForwarders() { const auto original_block_count = graph.blocks.size(); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - const auto original_empty_blocks = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.inst_begin == block.inst_end; - }); Check(original_block_count == 7u && graph.natural_loops.size() == 1u, "nested early-exit fixture has the wrong native CFG"); - const bool structured = ShaderRecompiler::CFG::Structurize(graph); - Check(structured, graph.unsupported_reason.c_str()); - Check(graph.blocks.size() == original_block_count + 4u && - CfgInstructionCoverage(graph, decoded.instructions.size()) == - original_coverage && - std::ranges::count_if(graph.blocks, - [](const auto &block) { - return block.inst_begin == block.inst_end; - }) == original_empty_blocks + 4, + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "nested early-exit structurization changed semantic coverage"); - const auto *preheader = graph.FindBlockByPc(0x00u); - const auto *outer = graph.FindBlockByPc(0x08u); - const auto *inner = graph.FindBlockByPc(0x10u); - const auto *loop = graph.FindBlockByPc(0x20u); - Check( - preheader != nullptr && outer != nullptr && inner != nullptr && - loop != nullptr && loop->terminator.loop_header && - preheader->terminator.merge_block != UINT32_MAX && - outer->terminator.merge_block != UINT32_MAX && - inner->terminator.merge_block != UINT32_MAX && - preheader->terminator.merge_block != outer->terminator.merge_block && - preheader->terminator.merge_block != inner->terminator.merge_block && - outer->terminator.merge_block != inner->terminator.merge_block, - "nested early-exit constructs do not have distinct structured merges"); + std::unordered_set merges; + for (const auto &block : graph.blocks) { + if (block.terminator.merge_block != UINT32_MAX) + Check(merges.insert(block.terminator.merge_block).second, + "nested early-exit constructs share a structured merge"); + } + Check(graph.natural_loops.size() == 1u, + "nested early-exit routing changed the natural loop count"); auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; @@ -8365,7 +8375,7 @@ void TestNewShaderRecompilerCfgNestedEarlyExitLoopForwarders() { void TestNewShaderRecompilerCfgExecSccSharedArm() { const uint32_t shader[] = { - EncodeSop2(0x15, 126, 4, 126), // s_andn2_b64 exec, s4, exec + EncodeSop2(0x0f, 126, 4, 126), // s_and_b64 exec, s4, exec (runtime mask) EncodeSopp(0x08, 2), // execz -> shared arm EncodeSop2(0x15, 30, 30, 126), // s_andn2_b64 s30, s30, exec EncodeSopp(0x04, 2), // scc0 -> other arm, else shared arm @@ -8384,17 +8394,8 @@ void TestNewShaderRecompilerCfgExecSccSharedArm() { Check(std::ranges::all_of(original_coverage, [](uint32_t uses) { return uses == 1u; }), "shared-arm fixture already duplicated a semantic instruction"); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); - uint32_t route_selects = 0; - uint32_t route_sets = 0; - for (const auto &block : graph.blocks) { - route_selects += block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - route_sets += block.terminator.goto_value >= 0; - } - Check(graph.blocks.size() == 5u && route_selects == 0u && route_sets == 0u, - "shared return arm introduced synthetic routing state"); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "EXEC/SCC shared-arm structurization changed semantic coverage"); @@ -8433,11 +8434,11 @@ void TestSharedReturnPreservesDescriptorDominance() { auto graph = ShaderRecompiler::CFG::BuildGraph(decoded); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); const auto *overwrite = graph.FindBlockByPc(0x10u); const auto *body = graph.FindBlockByPc(0x20u); - Check(graph.blocks.size() == 5u && overwrite != nullptr && body != nullptr && + Check(overwrite != nullptr && body != nullptr && graph.Dominates(overwrite->id, body->id) && std::ranges::count_if(graph.blocks, [](const auto &block) { return block.terminator.kind == @@ -8466,6 +8467,339 @@ void TestSharedReturnPreservesDescriptorDominance() { CheckSpirvBinaryValidates(result.spirv); } +void TestNestedSelectionPreservesDescriptorSources() { + constexpr uint32_t nested_arm = 12, direct_arm = 18, join = 21, end = 23; + const uint32_t shader[] = { + EncodeSopc(0x07, 9, 128), + EncodeSopp(0x04, end - 1 - 1), + EncodeSopc(0x07, 8, 130), + EncodeSopp(0x04, direct_arm - 3 - 1), + EncodeSmem0(0x02, 16, 10), 125u << 25u, + EncodeSopc(0x07, 8, 129), + EncodeSopp(0x04, nested_arm - 7 - 1), + EncodeSmem0(0x08, 24, 8), 125u << 25u, + EncodeVop1(0x01, 0, 24), + EncodeSopp(0x02, join - 11 - 1), + EncodeSmem0(0x02, 16, 10), (125u << 25u) | 16u, + EncodeSmem0(0x08, 24, 8), 125u << 25u, + EncodeVop1(0x01, 0, 24), + EncodeSopp(0x02, join - 17 - 1), + EncodeSmem0(0x08, 24, 8), 125u << 25u, + EncodeVop1(0x01, 0, 24), + EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(0, 3, 0), + EncodeSopp(0x01), + }; + ShaderRecompiler::Decoder::Program decoded; + ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded); + auto graph = ShaderRecompiler::CFG::BuildGraph(decoded); + const auto coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); + Check(graph.blocks.size() == 8u && + std::ranges::all_of(coverage, [](uint32_t uses) { return uses == 1u; }), + "nested descriptor fixture has the wrong native topology"); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == coverage, + "nested descriptor selection changed semantic instruction coverage"); + + const std::array table{ + 0x3000u, 0u, 4u, 3u << 28u, 0x5000u, 0u, 4u, 3u << 28u}; + std::array user_data{}; + user_data[1] = 1u; + user_data[4] = 0x4000u; + user_data[6] = 4u; + user_data[7] = 3u << 28u; + user_data[10] = 4u; + user_data[11] = 3u << 28u; + const auto address = reinterpret_cast(table.data()); + user_data[12] = static_cast(address); + user_data[13] = static_cast(address >> 32u); + for (uint32_t mode = 0; mode != 3; ++mode) { + user_data[0] = mode; + user_data[8] = 0x2000u + mode * 0x10000u; + auto options = MakeCompileOptions(ShaderType::Compute); + options.user_data_base = 8; + options.user_data = user_data; + const auto result = RecompileForTest(shader, options, ReadHostTestMemory); + Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 4u, + "nested descriptor selection lost its structured resource bindings"); + for (size_t i = 0; i < result.program.info.buffers.size(); ++i) { + const auto pc = result.program.info.buffers[i].first_use_pc; + const uint32_t expected = pc == 8u * 4u ? table[0] + : pc == 14u * 4u ? table[4] + : pc == direct_arm * 4u ? user_data[8] + : pc == join * 4u ? user_data[4] : 0u; + Check(expected != 0u && result.resources.buffers[i].dwords[0] == expected, + "nested descriptor selection chose another arm's descriptor"); + } + CheckSpirvBinaryValidates(result.spirv); + } +} + +void TestSharedExitPreservesNativeDescriptorSources() { + constexpr uint32_t selected = 8, later = 18, normal = 26, killed = 29; + const uint32_t shader[] = { + EncodeSopc(0x07, 0, 128), + EncodeSopp(0x04, normal - 1 - 1), + EncodeSmem0(0x02, 8, 6), (125u << 25u) | 32u, + EncodeSopc(0x07, 1, 128), + EncodeSopp(0x04, selected - 5 - 1), + EncodeSmem0(0x02, 8, 6), (125u << 25u) | 16u, + EncodeSopc(0x07, 2, 128), + EncodeSopp(0x04, later - 9 - 1), + EncodeSmem0(0x02, 8, 6), (125u << 25u) | 32u, + EncodeSopc(0x07, 3, 128), + EncodeSopp(0x04, killed - 13 - 1), + EncodeSmem0(0x08, 16, 4), 125u << 25u, + EncodeVop1(0x01, 0, 16), + EncodeSopp(0x02, normal - 17 - 1), + EncodeSmem0(0x02, 8, 6), (125u << 25u) | 48u, + EncodeSopc(0x07, 4, 128), + EncodeSopp(0x04, killed - 21 - 1), + EncodeSmem0(0x08, 16, 4), 125u << 25u, + EncodeVop1(0x01, 0, 16), + EncodeSopp(0x02, normal - 25 - 1), + EncodeExp0(0x00, 0x1), EncodeExp1(0, 0, 0, 0), + EncodeSopp(0x01), + EncodeSopp(0x01), + }; + ShaderRecompiler::Decoder::Program decoded; + ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded); + auto graph = ShaderRecompiler::CFG::BuildGraph(decoded); + const auto coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); + Check(graph.blocks.size() == 10u && + std::ranges::all_of(coverage, [](uint32_t uses) { return uses == 1u; }), + "shared-exit descriptor fixture has the wrong native topology"); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == coverage, + "shared-exit descriptor selection changed semantic instruction coverage"); + + const std::array table{ + 0x2000u, 0u, 4u, 3u << 28u, 0x3000u, 0u, 4u, 3u << 28u, + 0x4000u, 0u, 4u, 3u << 28u, 0x5000u, 0u, 4u, 3u << 28u}; + std::array user_data{}; + std::fill_n(user_data.begin(), 5, 1u); + user_data[8] = 0x1000u; + user_data[10] = 4u; + user_data[11] = 3u << 28u; + const auto address = reinterpret_cast(table.data()); + user_data[12] = static_cast(address); + user_data[13] = static_cast(address >> 32u); + for (uint32_t arm = 0; arm != 2; ++arm) { + user_data[2] = arm; + auto options = MakeCompileOptions(ShaderType::Pixel); + options.user_data = user_data; + const auto result = RecompileForTest(shader, options, ReadHostTestMemory); + Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 2u, + "shared-exit descriptor selection lost its structured resource bindings"); + for (size_t i = 0; i < result.program.info.buffers.size(); ++i) { + const auto pc = result.program.info.buffers[i].first_use_pc; + const uint32_t expected = pc == 14u * 4u ? table[8] + : pc == 22u * 4u ? table[12] : 0u; + Check(expected != 0u && result.resources.buffers[i].dwords[0] == expected, + "shared-exit descriptor selection retained an overwritten native source"); + } + CheckSpirvBinaryValidates(result.spirv); + } +} + +void TestNativeScalarReadDescriptorPlanning() { + constexpr uint32_t nested = 10, read = 14, join = 18, end = 20; + const uint32_t shader[] = { + EncodeSopc(0x07, 9, 128), + EncodeSopp(0x04, end - 1 - 1), + EncodeSopc(0x07, 8, 130), + EncodeSopp(0x04, read - 3 - 1), + EncodeSop1(0x24, 0, 126), EncodeSop1(0x04, 2, 126), + EncodeSopc(0x07, 8, 129), + EncodeSopp(0x04, nested - 7 - 1), + EncodeVop1(0x01, 0, 128), + EncodeSopp(0x02, join - 9 - 1), + EncodeSop1(0x04, 0, 126), EncodeSop1(0x04, 2, 126), + EncodeVop1(0x01, 0, 129), + EncodeSopp(0x02, join - 13 - 1), + EncodeSmem0(0x0a, 16, 0), 125u << 25u, + EncodeMubuf0(0x0c, 0, false), EncodeMubuf1(0, 4, 1), + EncodeExp0(0x00, 0x1), EncodeExp1(0, 0, 0, 0), + EncodeSopp(0x01), + }; + const std::array table{0x5000u, 0u, 4u, 3u << 28u}; + const auto address = reinterpret_cast(table.data()); + std::array user_data{}; + user_data[0] = static_cast(address); + user_data[1] = static_cast(address >> 32u); + user_data[2] = sizeof(table); + user_data[3] = 3u << 28u; + user_data[9] = 1u; + for (const uint32_t mode : {2u, 0u, 1u}) { + user_data[8] = mode; + auto options = MakeCompileOptions(ShaderType::Pixel); + options.user_data = user_data; + const auto result = RecompileForTest(shader, options, ReadHostTestMemory); + Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 1u && + std::ranges::equal(result.resources.flattened_srt, table), + "scalar descriptor planning retained an unrelated native execution mask"); + const auto expected = mode == 2u ? table : std::array{}; + Check(std::equal(expected.begin(), expected.end(), result.resources.buffers[0].dwords.begin()), + "scalar descriptor planning selected the wrong native resource"); + CheckSpirvBinaryValidates(result.spirv); + } +} + +void TestNativeGuardedSamplerSource() { + constexpr uint32_t nested = 12, sample = 18, join = 20, end = 22; + const uint32_t shader[] = { + EncodeSopc(0x07, 21, 128), + EncodeSopp(0x04, end - 1 - 1), + EncodeSopc(0x07, 20, 130), + EncodeSopp(0x04, sample - 3 - 1), + EncodeSMovB32(16, 133), EncodeSMovB32(17, 134), + EncodeSMovB32(18, 135), EncodeSMovB32(19, 136), + EncodeSopc(0x07, 20, 129), + EncodeSopp(0x04, nested - 9 - 1), + EncodeVop1(0x01, 0, 128), + EncodeSopp(0x02, join - 11 - 1), + EncodeSMovB32(16, 137), EncodeSMovB32(17, 138), + EncodeSMovB32(18, 139), EncodeSMovB32(19, 140), + EncodeVop1(0x01, 0, 129), + EncodeSopp(0x02, join - 17 - 1), + EncodeMimg0(0x20, 0x1), EncodeMimg1(0, 2, 4, 2), + EncodeExp0(0x00, 0x1), EncodeExp1(0, 0, 0, 0), + EncodeSopp(0x01), + }; + auto user_data = ImageTestUserData(); + const std::array sampler{0u, 0u, 0x09500000u, 0u}; + std::ranges::copy(sampler, user_data.begin() + 8); + user_data[13] = 1u; + for (uint32_t mode = 0; mode != 3; ++mode) { + user_data[12] = mode; + auto options = MakeCompileOptions(ShaderType::Pixel); + options.user_data_base = 8; + options.user_data = user_data; + const auto result = RecompileForTest(shader, options); + const auto expected = mode == 2u ? sampler : std::array{}; + Check(!result.program.dispatcher_fallback && result.resources.samplers.size() == 1u && + result.resources.samplers[0].dword_count == expected.size() && + std::equal(expected.begin(), expected.end(), result.resources.samplers[0].dwords.begin()), + "guarded native sample selected an unrelated arm's sampler constants"); + CheckSpirvBinaryValidates(result.spirv); + } +} + +void TestNativeDescriptorProvenanceRejectsGpuSelection() { +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS + const std::array selected{ + EncodeSmem0(0x02, 8, 6), 125u << 25u, + EncodeVop1(0x01, 1, 128), + EncodeVopc(0xc4, 256, 1), + EncodeSopp(0x06, 2), + EncodeSmem0(0x02, 8, 6), (125u << 25u) | 16u, + EncodeSmem0(0x08, 16, 4), 125u << 25u, + EncodeVop1(0x01, 2, 16), + EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(2, 0, 0), + EncodeSopp(0x01), + }; + auto partial = selected; + partial[5] = EncodeVop1(0x02, 11, 256); + partial[6] = EncodeSopp(0x00); + const std::array loop{ + EncodeSmem0(0x02, 8, 6), 125u << 25u, + EncodeSmem0(0x08, 16, 4), 125u << 25u, + EncodeVop1(0x01, 2, 16), + EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(2, 0, 0), + EncodeSmem0(0x02, 8, 6), (125u << 25u) | 16u, + EncodeVop1(0x01, 1, 128), + EncodeVopc(0xc4, 256, 1), + EncodeSopp(0x07, 0xfff6u), + EncodeSopp(0x01), + }; + std::array user_data{}; + ShaderComputeInputInfo input_info{}; + input_info.thread_ids_num = 1; + input_info.threads_num[0] = 64; + input_info.threads_num[1] = input_info.threads_num[2] = 1; + auto options = MakeCompileOptions(ShaderType::Compute); + options.input_info.compute = &input_info; + options.user_data = user_data; + for (const auto &shader : {selected, partial, loop}) { + ExpectFatal([&] { ShaderRecompiler::TranslateProgram(shader, options); }, + "native descriptor provenance accepted a GPU-selected or loop-carried value"); + } +#endif +} + +void TestCfgSiblingSharedExit() { + // f7030726b9470dd8: a nested exit on one side and two exits on the + // other side share an epilogue, separate from the normal return. + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 128), EncodeSopp(0x04, 5), // right or left + EncodeSopc(0x06, 1, 128), EncodeSopp(0x04, 2), // join or nested exit + EncodeSopc(0x06, 2, 128), EncodeSopp(0x04, 7), // shared exit + EncodeSopp(0x02, 4), // left -> main + EncodeSopc(0x06, 3, 128), EncodeSopp(0x04, 4), // right -> shared exit + EncodeSopc(0x06, 4, 128), EncodeSopp(0x04, 2), // second shared exit + EncodeSMovB32(5, 129), EncodeSopp(0x01), // main and normal return + EncodeSMovB32(5, 130), EncodeSopp(0x01), // shared exit epilogue + }; + using namespace ShaderRecompiler; + Decoder::Program decoded; + Decoder::DecodeProgram(std::span{shader}, decoded); + const auto original = CFG::BuildGraph(decoded); + auto structured = CFG::Structurize(original); + Check(!structured.unsupported, structured.unsupported_reason.c_str()); + Check(CfgInstructionCoverage(structured, decoded.instructions.size()) == + CfgInstructionCoverage(original, decoded.instructions.size()), + "shared sibling exit duplicated or dropped guest instructions"); + const auto trace = [&](const CFG::Graph &graph, uint32_t decisions) { + std::vector executed; + std::vector variables(original.blocks.size() * 2); + const auto expression = [&](auto &&self, uint32_t index) -> bool { + const auto &value = graph.expressions.at(index); + switch (value.op) { + case CFG::ConditionExpression::Op::Constant: return value.lhs != 0; + case CFG::ConditionExpression::Op::Variable: return variables.at(value.lhs); + case CFG::ConditionExpression::Op::Native: std::abort(); + case CFG::ConditionExpression::Op::Not: return !self(self, value.lhs); + case CFG::ConditionExpression::Op::Or: + return self(self, value.lhs) || self(self, value.rhs); + } + std::abort(); + }; + auto current = graph.entry_block; + for (unsigned steps = 0; steps < 100; ++steps) { + const auto &block = graph.blocks.at(current); + for (auto inst = block.inst_begin; inst < block.inst_end; ++inst) + executed.push_back(inst); + for (const auto &assignment : block.assignments) { + variables.at(assignment.variable) = + graph.expressions[assignment.expression].op == CFG::ConditionExpression::Op::Native + ? (decisions & (1u << (assignment.variable - original.blocks.size()))) != 0 + : expression(expression, assignment.expression); + } + const auto &term = block.terminator; + if (term.kind == CFG::TerminatorKind::Return) return executed; + Check(term.kind == CFG::TerminatorKind::Branch || + term.kind == CFG::TerminatorKind::ConditionalBranch, + "shared exit trace has an unsupported terminator"); + const bool condition = term.expression != UINT32_MAX + ? expression(expression, term.expression) + : (decisions & (1u << block.id)) != 0; + current = term.kind == CFG::TerminatorKind::Branch || condition + ? term.true_block : term.false_block; + } + Check(false, "shared exit trace did not terminate"); + return executed; + }; + for (uint32_t decisions = 0; decisions < (1u << original.blocks.size()); ++decisions) + Check(trace(original, decisions) == trace(structured, decisions), + "goto elimination changed the executed shared-exit path"); + auto result = RecompileForTest(shader, MakeCompileOptions(ShaderType::Compute)); + Check(!result.program.dispatcher_fallback && + SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "shared sibling exit selected dispatcher lowering"); + CheckSpirvBinaryValidates(result.spirv); +} + void TestNewShaderRecompilerCfgNestedTailEarlyExit() { const uint32_t shader[] = { EncodeSopc(0x06, 0, 0), // preceding outer condition @@ -8494,22 +8828,11 @@ void TestNewShaderRecompilerCfgNestedTailEarlyExit() { graph = ShaderRecompiler::CFG::BuildGraph(decoded); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "nested-tail routing changed semantic instruction coverage"); - Check(std::ranges::count_if( - graph.blocks, - [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }) == 1u && - std::ranges::count_if(graph.blocks, - [](const auto &block) { - return block.terminator.goto_value >= 0; - }) == 3u, - "nested-tail early exit did not use typed route state"); auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; @@ -8544,22 +8867,11 @@ void TestNewShaderRecompilerCfgSharedReturnAfterNestedSelections() { graph = ShaderRecompiler::CFG::BuildGraph(decoded); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "shared-exit route ordering changed semantic instruction coverage"); - const auto route_selects = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }); - const auto route_sets = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.goto_value >= 0; - }); - Check(route_selects == 0u && route_sets == 0u, - "nested selections introduced routing state for a shared return"); auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; @@ -8593,7 +8905,8 @@ void TestNewShaderRecompilerCfgAlternatingSharedReturns() { auto graph = ShaderRecompiler::CFG::BuildGraph(decoded); const auto coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); Check(graph.blocks.size() == 5u, "alternating returns fixture has the wrong CFG"); - Check(ShaderRecompiler::CFG::Structurize(graph), graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == coverage, "alternating returns duplicated a terminal epilogue"); @@ -8628,31 +8941,20 @@ void TestNewShaderRecompilerCfgLoopSharedRegion() { ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded); ShaderRecompiler::CFG::Graph graph; graph = ShaderRecompiler::CFG::BuildGraph(decoded); - const auto original_block_count = graph.blocks.size(); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); Check(graph.natural_loops.size() == 1u, "loop shared-region fixture has the wrong native CFG"); - const bool structured = ShaderRecompiler::CFG::Structurize(graph); - Check(structured, graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); const auto *loop_header = graph.FindBlockByPc(0x00u); Check(graph.natural_loops.size() == 1u && graph.back_edges.size() == 1u && loop_header != nullptr && loop_header->terminator.loop_header && loop_header->terminator.continue_block != UINT32_MAX, "loop shared-region routing did not preserve the natural loop"); - uint32_t route_selects = 0; - uint32_t route_sets = 0; - for (const auto &block : graph.blocks) { - route_selects += block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - route_sets += block.terminator.goto_value >= 0; - } - Check(route_selects == 1u && route_sets == 3u && - graph.blocks.size() >= original_block_count + 5u && - CfgInstructionCoverage(graph, decoded.instructions.size()) == - original_coverage, - "loop shared region was not routed without semantic duplication"); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, + "loop shared region changed guest instruction coverage"); auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; @@ -8697,15 +8999,10 @@ void TestNewShaderRecompilerCfgSharedRegionBeforeEarlyBreakLoop() { CfgInstructionCoverage(graph, decoded.instructions.size()); Check(graph.blocks.size() == 9u && graph.natural_loops.size() == 1u, "shared-region/early-break fixture has the wrong native CFG"); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(graph.natural_loops.size() == 1u && graph.back_edges.size() == 1u, "selection routing introduced a cycle around a structured loop exit"); - Check(std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }) == 1u, - "shared-region routing rewrote unrelated loop-control branches"); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "shared-region/early-break routing changed semantic coverage"); @@ -8763,24 +9060,13 @@ void TestNewShaderRecompilerCfgOverlappingEarlyExitLadder() { const auto original_block_count = graph.blocks.size(); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - const bool structured = ShaderRecompiler::CFG::Structurize(graph); - Check(structured, graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "early-exit ladder routing changed semantic instruction coverage"); Check(graph.blocks.size() > original_block_count, "early-exit ladder routing did not add forwarding blocks"); - const auto route_selects = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }); - const auto route_sets = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.goto_value >= 0; - }); - Check(route_selects != 0u && route_sets >= 3u, - "early-exit ladder lacks explicit typed route state"); auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; @@ -8813,22 +9099,11 @@ void TestNewShaderRecompilerCfgNestedEarlyExitSharedTerminal() { graph = ShaderRecompiler::CFG::BuildGraph(decoded); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - const bool structured = ShaderRecompiler::CFG::Structurize(graph); - Check(structured, graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "nested early-exit structurization changed semantic coverage"); - const auto route_selects = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.condition == - ShaderRecompiler::CFG::BranchCondition::GotoVariable; - }); - const auto route_sets = - std::ranges::count_if(graph.blocks, [](const auto &block) { - return block.terminator.goto_value >= 0; - }); - Check(route_selects == 1u && route_sets == 3u, - "nested early exit lacks complete typed route state"); auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; @@ -8867,18 +9142,12 @@ void TestNewShaderRecompilerCfgEarlyReturnSharedLoopContinuation() { auto graph = ShaderRecompiler::CFG::BuildGraph(decoded); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage && graph.natural_loops.size() == 2u, "shared-continuation gateway changed semantic blocks or the loops"); - Check(std::ranges::none_of(graph.blocks, - [](const auto &block) { - return block.terminator.goto_variable != - UINT32_MAX; - }), - "private early return introduced unnecessary routing state"); auto options = MakeCompileOptions(ShaderType::Compute); auto result = RecompileForTest(shader, options); @@ -8910,8 +9179,8 @@ void TestNewShaderRecompilerCfgSharedTerminalEarlyExit() { graph = ShaderRecompiler::CFG::BuildGraph(decoded); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check( CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, @@ -8946,8 +9215,8 @@ void TestNewShaderRecompilerCfgPrunesUnreachableSelectionEntry() { CfgInstructionCoverage(graph, decoded.instructions.size()); Check(original_coverage[1] == 0u, "CFG retained an unreachable external selection entry"); - Check(ShaderRecompiler::CFG::Structurize(graph), - graph.unsupported_reason.c_str()); + graph = ShaderRecompiler::CFG::Structurize(graph); + Check(!graph.unsupported, graph.unsupported_reason.c_str()); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, "selection structurization changed reachable semantic code"); @@ -8975,28 +9244,24 @@ void TestNewShaderRecompilerCfgFailedStructurizationPreservesGraph() { const auto original = graph; Check(graph.natural_loops.size() == 1u && !graph.irreducible, "failed structurization fixture must have one reducible loop"); - Check(!CFG::Structurize(graph) && graph.unsupported && - graph.failure_kind == CFG::FailureKind::StructuredControlFlow && - !graph.unsupported_reason.empty(), + const auto failed = CFG::Structurize(graph); + Check(failed.unsupported && + failed.failure_kind == CFG::FailureKind::StructuredControlFlow && + !failed.unsupported_reason.empty(), "loop without a merge did not retain its structurization failure"); - Check(graph.failure_block == UINT32_MAX, + Check(failed.failure_block == UINT32_MAX, "failed structurization exposed a discarded synthetic block ID"); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == CfgInstructionCoverage(original, decoded.instructions.size()), "failed structurization changed instruction coverage"); - auto topology = graph; - topology.unsupported = original.unsupported; - topology.failure_kind = original.failure_kind; - topology.failure_block = original.failure_block; - topology.unsupported_reason = original.unsupported_reason; - Check(CFG::GraphToString(topology) == CFG::GraphToString(original), + Check(CFG::GraphToString(graph) == CFG::GraphToString(original), "failed structurization changed CFG topology or analyses"); auto options = MakeCompileOptions(ShaderType::Compute); auto result = RecompileForTest(shader, options); Check(result.program.dispatcher_fallback && - result.program.cfg_failure_kind == graph.failure_kind && - result.program.fallback_reason == graph.unsupported_reason, + result.program.cfg_failure_kind == failed.failure_kind && + result.program.fallback_reason == failed.unsupported_reason, "compiler did not consume CFG failure diagnostics directly"); CheckSpirvBinaryValidates(result.spirv); } @@ -12128,7 +12393,7 @@ void BuildTypedPlan(const uint32_t *code, uint32_t words, ShaderRecompiler::IR::RemoveIdentities(ir.blocks); ShaderRecompiler::IR::EliminateDeadCode(ir.blocks); } - ShaderRecompiler::IR::BuildSrtPlan(ir); + ShaderRecompiler::IR::TrackResources(ir, decoded, cfg); ShaderRecompiler::IR::EliminateDeadCode(ir.blocks); } @@ -12225,9 +12490,20 @@ void TestTypedDescriptorRealWideMoveTranslation() { EncodeMubuf1(0, 0, 1), // buffer_store_dword via copied s[0:3] EncodeSopp(0x01), }; - ShaderRecompiler::IR::Program ir; + ShaderRecompiler::Decoder::Program decoded; + ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded); + const auto native = ShaderRecompiler::CFG::BuildGraph(decoded); + ShaderComputeInputInfo compute; + auto ir = ShaderRecompiler::Frontend::TranslateProgram( + decoded, native, {.stage = ShaderType::Compute, .wave_size = 64u, + .input_info = {.compute = &compute}}); + for (uint32_t reg = 0; reg < 4; ++reg) { + Check(std::ranges::any_of(ir.scalar_writes, [&](const auto &write) { + return write.pc == (reg / 2u) * 4u && + ShaderRecompiler::IR::RegIndex(write.reg) == reg; + }), "wide scalar move lost a native descriptor component write"); + } BuildTypedPlan(shader, static_cast(std::size(shader)), ir); - ShaderRecompiler::IR::TrackResources(ir); Check(ir.info.buffers.size() == 1, "real wide-move shader did not track one buffer use"); const auto *source = TypedDescriptorSource(ir, ir.info.buffers[0].source); @@ -12401,8 +12677,7 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { ShaderRecompiler::IR::Program load_ir; BuildTypedPlan(load_shader, static_cast(std::size(load_shader)), load_ir); - Check(load_ir.srt_reads.size() == 8 && - load_ir.dynamic_reads.empty(), + Check(load_ir.srt_reads.size() == 8, "real scalar loads did not build eight flattened reads"); uint32_t address_reads = 0; uint32_t buffer_reads = 0; @@ -12417,7 +12692,6 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { } Check(address_reads == 4 && buffer_reads == 4, "real scalar loads used the wrong raw typed operations"); - ShaderRecompiler::IR::TrackResources(load_ir); Check(load_ir.info.buffers.size() == 2, "real scalar-load descriptor sources were not attached"); for (const auto &buffer : load_ir.info.buffers) { @@ -12448,7 +12722,6 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { BuildTypedPlan(inline_sampler_shader, static_cast(std::size(inline_sampler_shader)), inline_sampler_ir); - ShaderRecompiler::IR::TrackResources(inline_sampler_ir); ShaderRecompiler::IR::DescriptorValue sampler; ShaderRecompiler::IR::SrtRuntime runtime; Check(inline_sampler_ir.info.samplers.size() == 1 && @@ -12479,7 +12752,7 @@ void TestSrtWalkerRealSmemTranslation() { }; ShaderRecompiler::IR::Program ir; BuildTypedPlan(shader, static_cast(std::size(shader)), ir); - Check(ir.srt_reads.size() == 4 && ir.dynamic_reads.empty(), + Check(ir.srt_reads.size() == 4, "real SMEM translation did not build four compact SRT reads"); const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, @@ -12535,7 +12808,7 @@ void TestSrtWalkerRealSBufferTranslation() { }; ShaderRecompiler::IR::Program ir; BuildTypedPlan(shader, static_cast(std::size(shader)), ir); - Check(ir.srt_reads.size() == 4 && ir.dynamic_reads.empty(), + Check(ir.srt_reads.size() == 4, "real S_BUFFER_LOAD translation did not build four compact reads"); const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, @@ -12551,10 +12824,10 @@ void TestSrtWalkerRealSBufferTranslation() { const auto walked = ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat); Check(walked, "SRT walk failed"); Check(flat.size() == 4 && - std::equal(flat.begin(), flat.end(), table.begin() + 1), - "real S_BUFFER_LOAD walk used the wrong final alignment"); + std::equal(flat.begin(), flat.end(), table.begin()), + "real S_BUFFER_LOAD walk did not align offset components independently"); - user_data[10] = 4 * sizeof(uint32_t); + user_data[10] = 3 * sizeof(uint32_t); const auto bounds_walked = ShaderRecompiler::IR::SrtWalker(ir, runtime).RefreshFlatBuffer(flat); Check(!bounds_walked, "real S_BUFFER_LOAD walk ignored descriptor bounds"); CheckFlattenedReadSlots( @@ -12590,7 +12863,7 @@ void TestScalarMemorySourcesCapturedBeforeWrites() { EncodeSopp(0x01)}); ShaderRecompiler::IR::Program ir; BuildTypedPlan(shader.data(), static_cast(shader.size()), ir); - Check(ir.srt_reads.size() == 4 && ir.dynamic_reads.empty(), + Check(ir.srt_reads.size() == 4, opcode == 0x02 ? "overlapping S_LOAD operands were evaluated after a " "component write" : "overlapping S_BUFFER_LOAD operands were evaluated " @@ -12635,11 +12908,10 @@ void TestScalarMemoryLoadCrossesIntoVcc() { }; ShaderRecompiler::IR::Program ir; BuildTypedPlan(shader, static_cast(std::size(shader)), ir); - Check(ir.srt_reads.size() == 4 && ir.dynamic_reads.empty(), + Check(ir.srt_reads.size() == 4, "wide SMEM destination crossing into VCC lost scalar provenance"); CheckFlattenedReadSlots( ir, 4, "wide SMEM destination crossing into VCC used wrong flat offsets"); - ShaderRecompiler::IR::TrackResources(ir); Check(ir.info.buffers.size() == 1, "wide SMEM destination crossing into VCC lost its buffer use"); const auto *source = @@ -12701,7 +12973,6 @@ void TestResourceTrackingRealDensePatching() { }; ShaderRecompiler::IR::Program ir; BuildTypedPlan(shader, static_cast(std::size(shader)), ir); - ShaderRecompiler::IR::TrackResources(ir); Check(ir.info.buffers.size() == 2 && ir.info.images.size() == 2 && ir.info.samplers.size() == 1, "real resource tracking produced the wrong dense list sizes"); @@ -12768,7 +13039,6 @@ void TestDirectTranslationResetsAnalysisState() { ShaderRecompiler::IR::Program ir; BuildTypedPlan(first_shader, static_cast(std::size(first_shader)), ir); - ShaderRecompiler::IR::TrackResources(ir); ShaderComputeInputInfo compute; ShaderRecompiler::IR::CollectShaderInfo(ir, {.compute = &compute}); Check(ir.resource_tracking_complete && ir.shader_info_complete && @@ -12777,10 +13047,13 @@ void TestDirectTranslationResetsAnalysisState() { ir.shader_hash = 0xdeadbeef; const uint32_t second_shader[] = {EncodeSopp(0x01)}; - BuildTypedPlan(second_shader, static_cast(std::size(second_shader)), - ir); + ShaderRecompiler::Decoder::Program decoded; + ShaderRecompiler::Decoder::DecodeProgram(std::span{second_shader}, decoded); + ir = ShaderRecompiler::Frontend::TranslateProgram( + decoded, ShaderRecompiler::CFG::BuildGraph(decoded), + {.stage = ShaderType::Compute, .input_info = {.compute = &compute}}); Check(!ir.resource_tracking_complete && !ir.shader_info_complete && - ir.srt_plan_complete && + !ir.srt_plan_complete && ir.srt_reads.empty() && ir.shader_hash == 0 && ir.info.buffers.empty() && ir.info.images.empty() && ir.info.samplers.empty() && ir.info.sampled_pairs.empty() && @@ -13607,6 +13880,7 @@ int main() { TestNewShaderRecompilerCapturedVopcSdwaCmpxLtU16(); TestNewShaderRecompilerIrLookupMissFailsExplicitly(); TestNewShaderRecompilerRejectsDppOn64BitCompares(); + TestFloatComparisonInputModes(); TestPsInputCountRegisterDecode(); TestPixelAncillaryLayerInput(); TestNewShaderRecompilerUnbasedFlatUsesBda(); @@ -13629,13 +13903,13 @@ int main() { TestNewShaderRecompilerCfgSharedOuterAndLoopMerge(); TestNewShaderRecompilerCfgLoopExitSharedWithSelection(); TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection(); - TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher(); + TestNewShaderRecompilerCfgNestedLoopNonlocalExitStructured(); TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection(); TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit(); - TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher(); + TestNewShaderRecompilerCfgMixedContinueNonmergeExitStructured(); TestNewShaderRecompilerCfgConditionalLatchNoSelection(); TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection(); - TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection(); + TestNewShaderRecompilerCfgLoopEarlyContinuesStructured(); TestNewShaderRecompilerCfgLoopGatewaySelection(); TestNewShaderRecompilerCfgConditionalLoopHeaderSelection(); TestNewShaderRecompilerCfgMultipleLoopLatches(); @@ -13643,6 +13917,12 @@ int main() { TestNewShaderRecompilerCfgNestedEarlyExitLoopForwarders(); TestNewShaderRecompilerCfgExecSccSharedArm(); TestSharedReturnPreservesDescriptorDominance(); + TestSharedExitPreservesNativeDescriptorSources(); + TestNestedSelectionPreservesDescriptorSources(); + TestNativeScalarReadDescriptorPlanning(); + TestNativeGuardedSamplerSource(); + TestNativeDescriptorProvenanceRejectsGpuSelection(); + TestCfgSiblingSharedExit(); TestNewShaderRecompilerCfgNestedTailEarlyExit(); TestNewShaderRecompilerCfgSharedReturnAfterNestedSelections(); TestNewShaderRecompilerCfgAlternatingSharedReturns();