From ffa69d48de245132beefdfaa946365d672f133db Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 26 Aug 2026 07:42:51 +0200 Subject: [PATCH] shader: remove second IR layer --- CMakeLists.txt | 1 + .../host_gpu/renderer/pipeline/shaders.cpp | 7 +- .../shader/recompiler/ShaderRecompiler.cpp | 212 +-- .../recompiler/backend/spirv/SpirvEmitter.cpp | 77 +- .../backend/spirv/spirvEmitterAnalysis.cpp | 2 +- .../backend/spirv/spirvEmitterFlow.cpp | 12 +- .../backend/spirv/spirvEmitterInternal.h | 8 +- .../backend/spirv/spirvEmitterProgram.cpp | 10 +- .../recompiler/frontend/cfg/ShaderCFG.cpp | 2 +- .../recompiler/frontend/decode/MemoryOps.cpp | 2 +- .../frontend/decode/VectorAluOps.cpp | 2 +- .../frontend/translate/Attribute.cpp | 122 +- .../recompiler/frontend/translate/Compare.cpp | 233 +-- .../recompiler/frontend/translate/Control.cpp | 474 ++++-- .../recompiler/frontend/translate/Convert.cpp | 120 +- .../frontend/translate/Dispatch.cpp | 150 +- .../recompiler/frontend/translate/Float.cpp | 240 +-- .../recompiler/frontend/translate/Integer.cpp | 618 ++++---- .../recompiler/frontend/translate/Memory.cpp | 1310 ++++++++++++----- .../frontend/translate/Translate.cpp | 781 ++++++---- .../recompiler/frontend/translate/Translate.h | 42 +- .../frontend/translate/Translator.h | 168 +-- .../ir/{ValueProgram.cpp => Program.cpp} | 26 +- .../shader/recompiler/ir/ShaderIR.cpp | 1250 ---------------- src/graphics/shader/recompiler/ir/ShaderIR.h | 150 +- .../shader/recompiler/ir/ShaderIRInternal.h | 51 - .../shader/recompiler/ir/ShaderIRLog.cpp | 214 --- .../shader/recompiler/ir/ShaderIRMemory.cpp | 812 ---------- .../shader/recompiler/ir/ValueProgram.h | 76 - .../shader/recompiler/ir/opcodes/Opcodes.h | 71 - .../recompiler/ir/opcodes/ShaderIROpcodes.cpp | 548 ------- .../recompiler/ir/opcodes/ShaderIROpcodes.inc | 354 ----- .../recompiler/ir/passes/BindingLayout.cpp | 18 +- .../ir/passes/ReadLaneElimination.cpp | 2 +- .../ir/passes/ReadLaneElimination.h | 4 +- .../ir/passes/ResourceMaterialization.cpp | 27 +- .../recompiler/ir/passes/ResourceTracking.cpp | 50 +- .../ir/passes/ShaderInfoCollection.cpp | 24 +- .../ir/passes/SharedMemoryBarrier.cpp | 8 +- .../ir/passes/SharedMemoryBarrier.h | 2 +- .../shader/recompiler/ir/passes/SrtWalker.cpp | 91 +- src/graphics/shader/shader.cpp | 41 +- src/graphics/shader/shader.h | 1 - src/graphics/shader/shaderPixelParameter.cpp | 48 + tests/ResourceTrackingTests.cpp | 75 +- tests/ScalarProvenanceTests.cpp | 79 +- tests/ShaderRecompilerComputeTests.cpp | 205 ++- tests/ShaderStageRuntimeTests.cpp | 17 +- tests/shaderCfgTests.cpp | 855 ++++++----- 49 files changed, 3691 insertions(+), 6001 deletions(-) rename src/graphics/shader/recompiler/ir/{ValueProgram.cpp => Program.cpp} (97%) delete mode 100644 src/graphics/shader/recompiler/ir/ShaderIR.cpp delete mode 100644 src/graphics/shader/recompiler/ir/ShaderIRInternal.h delete mode 100644 src/graphics/shader/recompiler/ir/ShaderIRLog.cpp delete mode 100644 src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp delete mode 100644 src/graphics/shader/recompiler/ir/ValueProgram.h delete mode 100644 src/graphics/shader/recompiler/ir/opcodes/Opcodes.h delete mode 100644 src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp delete mode 100644 src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc create mode 100644 src/graphics/shader/shaderPixelParameter.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ebe8d3607..60721050a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -430,6 +430,7 @@ add_executable(resource_tracking_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/ResourceTrackingTests.cpp" "${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp" "${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/shaderPixelParameter.cpp" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp" diff --git a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp index 23b0953af..939f67243 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp @@ -548,9 +548,10 @@ void CreatePipelineInternal( uint32_t attr_size = 4; auto registers_num = vs_input_info.resources_dst[index].registers_num; - auto used_components = (vs_input_info.resource_fetch_components[index] > 0 - ? vs_input_info.resource_fetch_components[index] - : registers_num); + const auto compiled_components = + vs_input_info.stage.program->info.vertex_fetch_components[index]; + auto used_components = + compiled_components > 0 ? static_cast(compiled_components) : registers_num; GetInputFormat(vs_input_info.resources[index], input_attr[index].format, attr_size, static_cast(used_components)); diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 9c49ae8ff..7f80291ca 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -7,7 +7,6 @@ #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include "graphics/shader/recompiler/frontend/translate/Translate.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" #include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" @@ -49,7 +48,7 @@ std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) { dump += "\nIR:\n"; dump += fmt::format("mode={} scratch_dwords={}\n", ir.dispatcher_fallback ? "dispatcher" : "structured", ir.scratch_dwords); - dump += ir.values != nullptr ? IR::ValueProgramToString(*ir.values) : IR::ProgramToString(ir); + dump += IR::ProgramToString(ir); return dump; } @@ -117,17 +116,8 @@ void ClearEmbeddedFetchVectorLanes(EmbeddedFetchVectorLanes* lanes, uint32_t reg lanes->erase(first, last); } -struct EmbeddedFetchLoad { - uint32_t pc = 0; - int attrib_id = -1; - uint32_t components = 0; - std::vector prolog_loads; -}; - -struct EmbeddedFetchData { - std::vector loads; - int32_t vertex_offset_sgpr = -1; -}; +using EmbeddedFetchLoad = Frontend::EmbeddedFetchLoad; +using EmbeddedFetchData = Frontend::EmbeddedFetchPlan; bool IsDecodedSgpr(const Decoder::Operand& op) { return op.kind == Decoder::OperandKind::Sgpr || op.kind == Decoder::OperandKind::VccLo || @@ -246,15 +236,6 @@ int BufferTableAttribFromOffset(uint32_t raw_offset, int dword) { return static_cast((raw_offset + static_cast(dword) * 4u) / 16u); } -void AppendUniquePcs(std::vector& dst, const std::vector& src) { - dst.reserve(dst.size() + src.size()); - for (auto pc: src) { - if (std::find(dst.begin(), dst.end(), pc) == dst.end()) { - dst.push_back(pc); - } - } -} - EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded, const ShaderVertexInputInfo* input_info, uint32_t user_data_base, uint32_t user_data_count, @@ -489,103 +470,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded return data; } -const EmbeddedFetchLoad* FindEmbeddedFetchLoad(const std::vector& loads, - uint32_t pc) { - for (const auto& load: loads) { - if (load.pc == pc) { - return &load; - } - } - return nullptr; -} - -bool EmbeddedFetchPcInList(const std::vector& pcs, uint32_t pc) { - return std::find(pcs.begin(), pcs.end(), pc) != pcs.end(); -} - -bool IsIrFetchPrologLoad(const IR::Instruction& inst) { - return inst.op == IR::Opcode::SLoadDword || inst.op == IR::Opcode::SBufferLoadDword; -} - -int ResolveEmbeddedFetchResource(const ShaderVertexInputInfo* input_info, - const EmbeddedFetchLoad& load) { - if (load.attrib_id >= 0 && load.attrib_id < input_info->resources_num && - input_info->resources_dst[load.attrib_id].attr_id == load.attrib_id) { - return load.attrib_id; - } - for (int i = 0; i < input_info->resources_num; i++) { - const auto& dst = input_info->resources_dst[i]; - if (dst.attr_id == load.attrib_id && - load.components <= static_cast(std::max(dst.registers_num, 1))) { - return i; - } - } - for (int i = 0; i < input_info->resources_num; i++) { - if (input_info->resources_dst[i].attr_id == load.attrib_id) { - return i; - } - } - return -1; -} - -uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputInfo* input_info, - const std::vector& loads) { - if (loads.empty()) { - return 0; - } - - std::vector prolog_pcs; - prolog_pcs.reserve(loads.size()); - for (const auto& load: loads) { - AppendUniquePcs(prolog_pcs, load.prolog_loads); - } - - auto* mutable_input_info = const_cast(input_info); - uint32_t rewritten = 0; - for (auto& block: ir.blocks) { - for (auto& inst: block.instructions) { - if (IsIrFetchPrologLoad(inst) && EmbeddedFetchPcInList(prolog_pcs, inst.pc)) { - auto pc = inst.pc; - inst = {}; - inst.pc = pc; - inst.op = IR::Opcode::ControlNop; - continue; - } - - if (inst.op != IR::Opcode::BufferLoadDword) { - continue; - } - const auto* load = FindEmbeddedFetchLoad(loads, inst.pc); - if (load == nullptr || inst.memory.data_dwords != load->components || - inst.dst.kind != IR::OperandKind::Register || - inst.dst.reg.file != IR::RegisterFile::Vector) { - continue; - } - - const auto resource_id = ResolveEmbeddedFetchResource(input_info, *load); - if (resource_id < 0 || resource_id >= input_info->resources_num) { - LOGF("ShaderRecompiler VS embedded fetch remap failed: pc=0x%08" PRIx32 - " attrib=%d resources=%d\n", - inst.pc, load->attrib_id, input_info->resources_num); - continue; - } - - inst.op = IR::Opcode::LoadInputF32; - inst.input_info.attr = static_cast(resource_id); - inst.input_info.chan = 0; - inst.input_info.component_count = load->components; - inst.memory = {}; - inst.src_count = 0; - mutable_input_info->resource_fetch_components[resource_id] = - std::max(mutable_input_info->resource_fetch_components[resource_id], - static_cast(load->components)); - rewritten += load->components; - } - } - - return rewritten; -} - } // namespace bool TryRecompile(std::span code, const CompileOptions& options, @@ -677,27 +561,6 @@ bool TryRecompile(std::span code, const CompileOptions& options, } IR::Program ir; - LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR LowerProgram\n", GetDumpLabel(options), - StageName(options.stage), options.shader_hash); - const auto lower_program = [&](IR::Program& output) { - output = {}; - if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, output, error)) { - return false; - } - output.shader_hash = options.shader_hash; - output.user_data_base = options.user_data_base; - output.user_data_count = options.user_data_count; - output.scratch_dwords = options.scratch_dwords; - return true; - }; - if (!lower_program(ir)) { - return false; - } - - LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR LowerProgram blocks=%" PRIu64 - " elapsed_ms=%" PRIu64 "\n", - GetDumpLabel(options), StageName(options.stage), options.shader_hash, - static_cast(ir.blocks.size()), phase_ms()); const ShaderVertexInputInfo* vertex = nullptr; const ShaderPixelInputInfo* pixel = nullptr; const ShaderComputeInputInfo* compute = nullptr; @@ -709,42 +572,55 @@ bool TryRecompile(std::span code, const CompileOptions& options, } EmbeddedFetchData embedded_fetch; if (options.stage == ShaderType::Vertex && vertex->fetch_embedded) { - embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, ir.user_data_base, - ir.user_data_count, options.wave_size); - auto rewritten = RewriteEmbeddedVertexFetches(ir, vertex, embedded_fetch.loads); - if (rewritten > 0 || !embedded_fetch.loads.empty()) { - LOGF("%s embedded vertex fetch rewrite: detected=%" PRIu64 " rewritten=%" PRIu32 "\n", - GetDumpLabel(options), static_cast(embedded_fetch.loads.size()), - rewritten); + embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, options.user_data_base, + options.user_data_count, options.wave_size); + if (!embedded_fetch.loads.empty()) { + LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options), + static_cast(embedded_fetch.loads.size())); } } - ir.dispatcher_fallback = dispatcher_fallback; - if (!dispatcher_reason.empty()) { - ir.fallback_reason = dispatcher_reason; - } - - ir.values = std::make_shared(); - if (!Frontend::TranslateProgram(ir, *ir.values, vertex, pixel, compute, error)) { + Frontend::TranslateOptions translate_options { + .stage = options.stage, + .wave_size = options.wave_size, + .shader_hash = options.shader_hash, + .user_data_base = options.user_data_base, + .user_data_count = options.user_data_count, + .scratch_dwords = options.scratch_dwords, + .dispatcher_fallback = dispatcher_fallback, + .cfg_failure_kind = cfg.failure_kind, + .fallback_reason = dispatcher_reason.empty() ? cfg.unsupported_reason + : dispatcher_reason, + .vertex = vertex, + .pixel = pixel, + .compute = compute, + .embedded_fetch = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch, + }; + LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash); + if (!Frontend::TranslateProgram(decoded, cfg, translate_options, ir, error)) { return false; } - ir.blocks.clear(); - IR::RewriteToSsa(ir.values->blocks); - IR::ConstantPropagationPass(ir.values->blocks); - IR::ResolveControlFlowIdentities(*ir.values); - IR::RemoveIdentities(ir.values->blocks); - IR::EliminateDeadCode(ir.values->blocks); - const auto read_lane_stats = IR::EliminateReadLane(*ir.values, ir.wave_size); + LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64 + " elapsed_ms=%" PRIu64 "\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash, + static_cast(ir.blocks.size()), phase_ms()); + IR::RewriteToSsa(ir.blocks); + IR::ConstantPropagationPass(ir.blocks); + IR::ResolveControlFlowIdentities(ir); + IR::RemoveIdentities(ir.blocks); + IR::EliminateDeadCode(ir.blocks); + const auto read_lane_stats = IR::EliminateReadLane(ir, ir.wave_size); if (read_lane_stats.rewritten_reads != 0) { LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options), read_lane_stats.rewritten_reads); - IR::ConstantPropagationPass(ir.values->blocks); - IR::ResolveControlFlowIdentities(*ir.values); - IR::RemoveIdentities(ir.values->blocks); - IR::EliminateDeadCode(ir.values->blocks); + IR::ConstantPropagationPass(ir.blocks); + IR::ResolveControlFlowIdentities(ir); + IR::RemoveIdentities(ir.blocks); + IR::EliminateDeadCode(ir.blocks); } if (options.stage == ShaderType::Compute) { const auto lds_barriers = - IR::InsertSharedMemoryBarriers(*ir.values, ir.wave_size, *compute); + IR::InsertSharedMemoryBarriers(ir, ir.wave_size, *compute); if (lds_barriers.inserted_barriers != 0) { LOGF("%s wave64 LDS synchronization: barriers=%" PRIu32 "\n", GetDumpLabel(options), lds_barriers.inserted_barriers); @@ -758,11 +634,11 @@ bool TryRecompile(std::span code, const CompileOptions& options, } return false; } - IR::EliminateDeadCode(ir.values->blocks); + IR::EliminateDeadCode(ir.blocks); if (!IR::TrackResources(ir, error)) { return false; } - IR::EliminateDeadCode(ir.values->blocks); + IR::EliminateDeadCode(ir.blocks); if (options.stage == ShaderType::Vertex) { ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr; } diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index bf0cf4721..9aaf3975e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -1,7 +1,7 @@ #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/shader.h" #include @@ -51,22 +51,20 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { Expect(Kind::Samplers, Dense(program.info.samplers.size())); } bool uses_gds = false; - if (program.values != nullptr) { - for (const auto* block: program.values->blocks) { - for (const auto& inst: *block) { - if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) { - continue; - } - const auto index = inst.Flags().index; - if (index >= program.values->memory_info.size()) { - return Fail(error, "shared operation has invalid memory metadata"); - } - const auto kind = program.values->memory_info[index].kind; - if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { - return Fail(error, "shared operation has invalid resource kind"); - } - uses_gds |= kind == IR::ResourceKind::Gds; + for (const auto* block: program.blocks) { + for (const auto& inst: *block) { + if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) { + continue; } + const auto index = inst.Flags().index; + if (index >= program.memory_info.size()) { + return Fail(error, "shared operation has invalid memory metadata"); + } + const auto kind = program.memory_info[index].kind; + if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { + return Fail(error, "shared operation has invalid resource kind"); + } + uses_gds |= kind == IR::ResourceKind::Gds; } } if (uses_gds) { @@ -76,11 +74,10 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { Expect(Kind::AddressMemory, Dense(program.info.addresses.size())); } const bool uses_flattened_runtime = - program.values != nullptr && - (!program.values->srt_reads.empty() || + !program.srt_reads.empty() || std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) { return image.indirect_mapping_capacity != 0u; - })); + }); if (uses_flattened_runtime) { Expect(Kind::FlattenedSrt); } @@ -120,9 +117,6 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { return Fail(error, "native user-data layout is inconsistent"); } - if (program.values == nullptr) { - return Fail(error, "native shader plan has no typed SSA"); - } const auto planning_only_handle = [&](const IR::Inst& handle) { return !handle.Uses().empty() && std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) { @@ -132,11 +126,11 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { return false; } const auto index = use.user->Flags().index; - return index < program.values->memory_info.size() && - program.values->memory_info[index].planning_only; + return index < program.memory_info.size() && + program.memory_info[index].planning_only; }); }; - for (const auto* block: program.values->blocks) { + for (const auto* block: program.blocks) { for (const auto& inst: *block) { const auto dense = inst.Flags(); switch (inst.GetOpcode()) { @@ -174,7 +168,7 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { case IR::ValueOpcode::ReadConst: { const auto slot = inst.Arg(1).Resolve(); if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 || - slot.U32() >= program.values->srt_reads.size()) { + slot.U32() >= program.srt_reads.size()) { return Fail(error, "flattened SRT read has an invalid dense slot"); } break; @@ -191,19 +185,15 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { program.spirv_requirements.reset(); IR::SpirvRequirements requirements {}; - if (program.values == nullptr) { - program.spirv_requirements.emplace(requirements); - return true; - } const auto MarkBallot = [&] { requirements.subgroup_ballot = true; }; - for (const auto* block: program.values->blocks) { + for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (IR::AddressOpcodeInfoOf(inst.GetOpcode()).access != IR::AddressAccess::None) { const auto memory_index = inst.Flags().index; - if (memory_index >= program.values->memory_info.size()) { + if (memory_index >= program.memory_info.size()) { return Fail(error, "address operation has invalid memory metadata"); } - if (program.values->memory_info[memory_index].kind == IR::ResourceKind::Scratch) { + if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) { if (program.scratch_dwords == 0) { return Fail(error, "scratch operation has no per-thread storage"); } @@ -212,10 +202,10 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { } if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) { const auto memory_index = inst.Flags().index; - if (memory_index >= program.values->memory_info.size()) { + if (memory_index >= program.memory_info.size()) { return Fail(error, "buffer operation has invalid memory metadata"); } - const auto& memory = program.values->memory_info[memory_index]; + const auto& memory = program.memory_info[memory_index]; if (memory.kind == IR::ResourceKind::Buffer) { if (memory.resource >= program.info.buffers.size()) { return Fail(error, "buffer operation has invalid resource metadata"); @@ -231,10 +221,10 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { const auto shared_access = IR::SharedAccessOf(inst.GetOpcode()); if (shared_access != IR::SharedAccess::None) { const auto index = inst.Flags().index; - if (index >= program.values->memory_info.size()) { + if (index >= program.memory_info.size()) { return Fail(error, "shared operation has invalid memory metadata"); } - const auto kind = program.values->memory_info[index].kind; + const auto kind = program.memory_info[index].kind; if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { return Fail(error, "shared operation has invalid resource kind"); } @@ -288,11 +278,11 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { break; case IR::ValueOpcode::SetAttribute: { const auto index = inst.Flags().index; - if (index >= program.values->export_info.size()) { + if (index >= program.export_info.size()) { return Fail(error, "attribute export has invalid metadata"); } if (program.stage == ShaderType::Pixel && - program.values->export_info[index].vm) { + program.export_info[index].vm) { requirements.pixel_valid_mask = true; } break; @@ -330,12 +320,7 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc if (!ValidateNativeProgram(program, error)) { return false; } - if (program.values == nullptr) { - SetError(error, "SPIR-V emitter requires planned typed SSA"); - return false; - } - const auto& value_program = *program.values; - if (!IR::ValidateValueProgram(value_program, true, error)) { + if (!IR::ValidateProgram(program, true, error)) { return false; } EmitterState state(program, resources, input_info); @@ -348,7 +333,7 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc AllocateInputVariables(state); AllocateOutputVariables(state); DefineModule(state); - if (!EmitValueProgram(state, value_program, error)) { + if (!EmitProgram(state, program, error)) { return false; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index 7bd2a92ad..b0a47064e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -2,7 +2,7 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index df313cd6e..a2433a766 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -5,13 +5,11 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { namespace { -bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) { - if (reg.file != IR::RegisterFile::Scalar) { - return false; - } +bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t& dword_index) { + const auto register_index = IR::RegIndex(reg); const auto& registers = state.program.bindings.user_data_registers; - const auto found = std::lower_bound(registers.begin(), registers.end(), reg.index); - if (found == registers.end() || *found != reg.index) { + const auto found = std::lower_bound(registers.begin(), registers.end(), register_index); + if (found == registers.end() || *found != register_index) { return false; } dword_index = static_cast(found - registers.begin()); @@ -504,7 +502,7 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) { case IR::ValueOpcode::GetUserData: { const auto reg = inst.Arg(0).ScalarRegister(); uint32_t dword = 0; - if (!UserDataDwordIndex(state, {IR::RegisterFile::Scalar, IR::RegIndex(reg)}, dword)) { + if (!UserDataDwordIndex(state, reg, dword)) { ctx.Define(inst, ConstantU32(state, 0)); } else { ctx.Define(inst, EmitShaderDataDwordLoad(state, dword)); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 32468ca69..03f02f813 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -6,7 +6,7 @@ #include "graphics/shader/recompiler/BufferFormat.h" #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" @@ -407,7 +407,7 @@ inline void EmitLabel(EmitterState& state, uint32_t label) { } struct ValueEmitContext { - ValueEmitContext(EmitterState& state_, const IR::ValueProgram& program_) + ValueEmitContext(EmitterState& state_, const IR::Program& program_) : state(state_), program(program_) {} uint32_t Def(IR::Value value); @@ -425,7 +425,7 @@ struct ValueEmitContext { void Fail(const IR::Inst& inst, const char* reason); EmitterState& state; - const IR::ValueProgram& program; + const IR::Program& program; std::unordered_map definitions; std::unordered_map labels; const std::unordered_map* dispatcher_spills = nullptr; @@ -847,7 +847,7 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst); bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst); -bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error); +bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error); // These templates accept local lambdas from several emitter translation units. template diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 031da76cf..dd182acb8 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -79,9 +79,9 @@ void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState } } -const IR::Block* TargetBlock(const IR::ValueProgram& program, uint32_t id) { +const IR::Block* TargetBlock(const IR::Program& program, uint32_t id) { const auto found = std::ranges::find_if( - program.block_info, [&](const IR::ValueBlockInfo& info) { return info.id == id; }); + program.block_info, [&](const IR::BlockInfo& info) { return info.id == id; }); if (found == program.block_info.end()) { return nullptr; } @@ -94,7 +94,7 @@ void EmitReturn(ValueEmitContext& ctx) { } void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block, - const IR::ValueBlockInfo& info) { + const IR::BlockInfo& info) { const auto& term = info.terminator; const auto emit_merge = [&]() { if (term.loop_header) { @@ -150,7 +150,7 @@ void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState& } uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher, - const IR::Block* block, const IR::ValueBlockInfo& info) { + const IR::Block* block, const IR::BlockInfo& info) { const auto& term = info.terminator; switch (term.kind) { case CFG::TerminatorKind::Branch: @@ -503,7 +503,7 @@ void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) { error = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason); } -bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error) { +bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error) { ValueEmitContext ctx(state, program); std::optional dispatcher; if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) { diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp index 9e3f0acce..2cf5c158d 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp @@ -1342,7 +1342,7 @@ bool IsolateSemanticLoopHeaders(Graph& graph, std::string* error) { // SPIR-V requires OpLoopMerge and its branch to remain in the loop header's // physical block. Keep that header as a dedicated control node, exactly like // the separate Loop node in shadPS4's structured AST. Guest instructions live - // in the body so later lowering may introduce bounds/EXEC control flow without + // in the body so later translation may introduce bounds/EXEC control flow without // displacing OpLoopMerge into a helper-created block. if (!IsolateSemanticLoopHeader(graph, loop->header)) { SetFailure(graph, FailureKind::StructuredControlFlow, loop->header, diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index ead6951a1..eceeb0571 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -423,7 +423,7 @@ void DecodeDs(uint32_t pc, std::span code, uint32_t word_index, DecodeVectorGpr(vdst, inst.dst); if (inst.opcode == Opcode::DS_READ_U16_D16) { // Native D16 reads update only the selected destination half. Reuse the partial - // destination representation so the typed lowering preserves the upper word. + // destination representation so typed IR translation preserves the upper word. inst.dst.sdwa_sel = 4u; } DecodeVectorGpr(addr, inst.src0); diff --git a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp index 7aa2bf23e..8101894ce 100644 --- a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp @@ -7,7 +7,7 @@ namespace { using Detail::OpcodeMap; -// These profiles describe the selectors that the current lowering implements. They are not +// These profiles describe the selectors that the current translator implements. They are not // architectural SDWA legality classes. enum class Vop2SdwaProfile { None, diff --git a/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp b/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp index ee9ebf2af..19316eb7a 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp @@ -1,42 +1,104 @@ #include "graphics/shader/recompiler/frontend/translate/Translator.h" +#include "graphics/shader/shader.h" #include #include +#include -namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { +namespace Libs::Graphics::ShaderRecompiler::Frontend { -IR::ExportFlags Translator::AddExportInfo(const IR::Instruction& inst) { - const auto index = static_cast(value_program.export_info.size()); - value_program.export_info.push_back(inst.export_info); +namespace { + +IR::ExportTargetKind ExportTargetKindFromTarget(uint32_t target, uint32_t& index) { + index = 0; + switch (target) { + case 0x08u: return IR::ExportTargetKind::MrtZ; + case 0x09u: return IR::ExportTargetKind::Null; + case 0x14u: return IR::ExportTargetKind::Primitive; + default: break; + } + if (target <= 0x07u) { + index = target; + return IR::ExportTargetKind::Mrt; + } + if (target >= 0x0cu && target <= 0x0fu) { + index = target - 0x0cu; + return IR::ExportTargetKind::Position; + } + if (target >= 0x20u && target <= 0x3fu) { + index = target - 0x20u; + return IR::ExportTargetKind::Parameter; + } + return IR::ExportTargetKind::Unknown; +} + +} // namespace + +IR::ExportFlags Translator::AddExportInfo(const Decoder::Instruction& inst) { + IR::ExportInfo info; + info.kind = ExportTargetKindFromTarget(inst.exp.target, info.index); + info.target = inst.exp.target; + info.en = inst.exp.en; + info.done = inst.exp.done; + info.compr = inst.exp.compr; + info.vm = inst.exp.vm; + const auto index = static_cast(program.export_info.size()); + program.export_info.push_back(info); return {.index = index, .pc = inst.pc}; } -bool Translator::TranslateAttributeOperation(const IR::Instruction& inst) { - if (inst.op == IR::Opcode::LoadInputF32) { - for (uint32_t component = 0; component < inst.input_info.component_count; component++) { - const auto chan = inst.input_info.chan + component; - const auto value = inst.input_info.interpolation_mode < 3u - ? ir.Emit(IR::ValueOpcode::GetInterpolationParameter, - {IR::Value(inst.input_info.attr), IR::Value(chan), - IR::Value(inst.input_info.interpolation_mode)}) - : ir.Emit(IR::ValueOpcode::GetAttribute, - {IR::Value(inst.input_info.attr), IR::Value(chan)}); - WriteOperand(OffsetOperand(inst.dst, component), value); - } - return true; +bool Translator::TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute, + uint32_t component_count) { + for (uint32_t component = 0; component < component_count; component++) { + const auto value = + ir.Emit(IR::ValueOpcode::GetAttribute, {IR::Value(attribute), IR::Value(component)}); + WriteOperand(OffsetOperand(inst.dst, component), value); } - if (inst.op == IR::Opcode::Export) { - std::array components {IR::Value(0u), IR::Value(0u), IR::Value(0u), - IR::Value(0u)}; - for (uint32_t index = 0; index < std::min(inst.src_count, 4u); index++) { - components[index] = ReadRawU32(PlainOperand(inst.src[index])); - } - const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4, - {components[0], components[1], components[2], components[3]}); - ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst)); - return true; - } - return false; + return true; } -} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail +bool Translator::TranslateAttributeOperation(const Decoder::Instruction& inst, std::string* error) { + switch (inst.opcode) { + case Decoder::Opcode::V_INTERP_P1_F32: return true; + case Decoder::Opcode::V_INTERP_P2_F32: + case Decoder::Opcode::V_INTERP_MOV_F32: { + if (inst.opcode == Decoder::Opcode::V_INTERP_MOV_F32 && inst.src0.value >= 3u) { + return ShaderError::Fail( + error, fmt::format("v_interp_mov_f32 mode {} is reserved at pc 0x{:08x}", + inst.src0.value, inst.pc)); + } + const auto interpolation_mode = + inst.opcode == Decoder::Opcode::V_INTERP_MOV_F32 ? inst.src0.value : 3u; + const auto value = + interpolation_mode < 3u + ? ir.Emit(IR::ValueOpcode::GetInterpolationParameter, + {IR::Value(inst.src1.value), IR::Value(inst.src2.value), + IR::Value(interpolation_mode)}) + : ir.Emit(IR::ValueOpcode::GetAttribute, + {IR::Value(inst.src1.value), IR::Value(inst.src2.value)}); + WriteOperand(inst.dst, value); + return true; + } + case Decoder::Opcode::EXP: { + uint32_t index = 0; + if (ExportTargetKindFromTarget(inst.exp.target, index) == + IR::ExportTargetKind::Unknown) { + return ShaderError::Fail( + error, fmt::format("unsupported EXP target 0x{:02x} at pc 0x{:08x}", + inst.exp.target, inst.pc)); + } + std::array components {IR::Value(0u), IR::Value(0u), IR::Value(0u), + IR::Value(0u)}; + for (uint32_t source = 0; source < std::min(inst.src_count, 4u); source++) { + components[source] = ReadRawU32(PlainOperand(SourceAt(inst, source))); + } + const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4, + {components[0], components[1], components[2], components[3]}); + ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst)); + return true; + } + default: return false; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp index 8cd406ba4..9827856a9 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp @@ -1,187 +1,210 @@ #include "graphics/shader/recompiler/frontend/translate/Translator.h" -namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { +namespace Libs::Graphics::ShaderRecompiler::Frontend { -bool Translator::TranslateIntegerCompare(const IR::Instruction& inst) { +bool Translator::TranslateIntegerCompare(const Decoder::Instruction& inst) { IR::ValueOpcode opcode {}; IR::Type type = IR::Type::U32; - switch (inst.op) { - case IR::Opcode::CompareFalse: + switch (inst.opcode) { + case Decoder::Opcode::V_CMP_F_I32: + case Decoder::Opcode::V_CMP_F_U32: WriteCompareResult(inst.dst, IR::U1(IR::Value(false))); return true; - case IR::Opcode::CompareTrue: + case Decoder::Opcode::V_CMP_T_I32: + case Decoder::Opcode::V_CMP_T_U32: WriteCompareResult(inst.dst, IR::U1(IR::Value(true))); return true; - case IR::Opcode::CompareEqU32: - case IR::Opcode::CompareMaskEqU32: - case IR::Opcode::CompareEqI32: - case IR::Opcode::CompareMaskEqI32: opcode = IR::ValueOpcode::IEqual32; break; - case IR::Opcode::CompareNeU32: - case IR::Opcode::CompareMaskNeU32: - case IR::Opcode::CompareNeI32: - case IR::Opcode::CompareMaskNeI32: opcode = IR::ValueOpcode::INotEqual32; break; - case IR::Opcode::CompareGtU32: - case IR::Opcode::CompareMaskGtU32: opcode = IR::ValueOpcode::UGreaterThan32; break; - case IR::Opcode::CompareGeU32: - case IR::Opcode::CompareMaskGeU32: opcode = IR::ValueOpcode::UGreaterThanEqual32; break; - case IR::Opcode::CompareLtU32: - case IR::Opcode::CompareMaskLtU32: opcode = IR::ValueOpcode::ULessThan32; break; - case IR::Opcode::CompareLeU32: - case IR::Opcode::CompareMaskLeU32: opcode = IR::ValueOpcode::ULessThanEqual32; break; - case IR::Opcode::CompareGtI32: - case IR::Opcode::CompareMaskGtI32: opcode = IR::ValueOpcode::SGreaterThan32; break; - case IR::Opcode::CompareGeI32: - case IR::Opcode::CompareMaskGeI32: opcode = IR::ValueOpcode::SGreaterThanEqual32; break; - case IR::Opcode::CompareLtI32: - case IR::Opcode::CompareMaskLtI32: opcode = IR::ValueOpcode::SLessThan32; break; - case IR::Opcode::CompareLeI32: - case IR::Opcode::CompareMaskLeI32: opcode = IR::ValueOpcode::SLessThanEqual32; break; - case IR::Opcode::CompareEqU64: + case Decoder::Opcode::S_CMP_EQ_U32: + case Decoder::Opcode::V_CMP_EQ_U32: + case Decoder::Opcode::V_CMPX_EQ_U32: + case Decoder::Opcode::S_CMP_EQ_I32: + case Decoder::Opcode::V_CMP_EQ_I32: + case Decoder::Opcode::V_CMPX_EQ_I32: opcode = IR::ValueOpcode::IEqual32; break; + case Decoder::Opcode::S_CMP_LG_U32: + case Decoder::Opcode::V_CMP_NE_U32: + case Decoder::Opcode::V_CMPX_NE_U32: + case Decoder::Opcode::S_CMP_LG_I32: + case Decoder::Opcode::V_CMP_NE_I32: + case Decoder::Opcode::V_CMPX_NE_I32: opcode = IR::ValueOpcode::INotEqual32; break; + case Decoder::Opcode::S_CMP_GT_U32: + case Decoder::Opcode::V_CMP_GT_U32: + case Decoder::Opcode::V_CMPX_GT_U32: opcode = IR::ValueOpcode::UGreaterThan32; break; + case Decoder::Opcode::S_CMP_GE_U32: + case Decoder::Opcode::V_CMP_GE_U32: + case Decoder::Opcode::V_CMPX_GE_U32: opcode = IR::ValueOpcode::UGreaterThanEqual32; break; + case Decoder::Opcode::S_CMP_LT_U32: + case Decoder::Opcode::V_CMP_LT_U32: + case Decoder::Opcode::V_CMPX_LT_U32: opcode = IR::ValueOpcode::ULessThan32; break; + case Decoder::Opcode::S_CMP_LE_U32: + case Decoder::Opcode::V_CMP_LE_U32: + case Decoder::Opcode::V_CMPX_LE_U32: opcode = IR::ValueOpcode::ULessThanEqual32; break; + case Decoder::Opcode::S_CMP_GT_I32: + case Decoder::Opcode::V_CMP_GT_I32: + case Decoder::Opcode::V_CMPX_GT_I32: opcode = IR::ValueOpcode::SGreaterThan32; break; + case Decoder::Opcode::S_CMP_GE_I32: + case Decoder::Opcode::V_CMP_GE_I32: + case Decoder::Opcode::V_CMPX_GE_I32: opcode = IR::ValueOpcode::SGreaterThanEqual32; break; + case Decoder::Opcode::S_CMP_LT_I32: + case Decoder::Opcode::V_CMP_LT_I32: + case Decoder::Opcode::V_CMPX_LT_I32: opcode = IR::ValueOpcode::SLessThan32; break; + case Decoder::Opcode::S_CMP_LE_I32: + case Decoder::Opcode::V_CMP_LE_I32: + case Decoder::Opcode::V_CMPX_LE_I32: opcode = IR::ValueOpcode::SLessThanEqual32; break; + case Decoder::Opcode::S_CMP_EQ_U64: + case Decoder::Opcode::V_CMP_EQ_I64: opcode = IR::ValueOpcode::IEqual64; type = IR::Type::U64; break; - case IR::Opcode::CompareGtU64: + case Decoder::Opcode::V_CMP_GT_U64: opcode = IR::ValueOpcode::UGreaterThan64; type = IR::Type::U64; break; - case IR::Opcode::CompareNeU64: + case Decoder::Opcode::S_CMP_LG_U64: + case Decoder::Opcode::V_CMP_NE_U64: + case Decoder::Opcode::V_CMPX_NE_I64: + case Decoder::Opcode::V_CMPX_NE_U64: opcode = IR::ValueOpcode::INotEqual64; type = IR::Type::U64; break; default: return false; } - const auto lhs = ReadOperand(inst.src[0], type); - const auto rhs = ReadOperand(inst.src[1], type); + const auto lhs = ReadOperand(inst.src0, type); + const auto rhs = ReadOperand(inst.src1, type); WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs}))); return true; } -bool Translator::TranslateInteger16Compare(const IR::Instruction& inst) { +bool Translator::TranslateInteger16Compare(const Decoder::Instruction& inst) { IR::ValueOpcode opcode {}; bool signed_value = false; - switch (inst.op) { - case IR::Opcode::CompareEqU16: - case IR::Opcode::CompareEqI16: opcode = IR::ValueOpcode::IEqual32; break; - case IR::Opcode::CompareNeU16: - case IR::Opcode::CompareNeI16: opcode = IR::ValueOpcode::INotEqual32; break; - case IR::Opcode::CompareGtU16: opcode = IR::ValueOpcode::UGreaterThan32; break; - case IR::Opcode::CompareGeU16: opcode = IR::ValueOpcode::UGreaterThanEqual32; break; - case IR::Opcode::CompareLtU16: opcode = IR::ValueOpcode::ULessThan32; break; - case IR::Opcode::CompareLeU16: opcode = IR::ValueOpcode::ULessThanEqual32; break; - case IR::Opcode::CompareGtI16: + switch (inst.opcode) { + case Decoder::Opcode::V_CMP_EQ_U16: + case Decoder::Opcode::V_CMP_EQ_I16: opcode = IR::ValueOpcode::IEqual32; break; + case Decoder::Opcode::V_CMP_NE_U16: + case Decoder::Opcode::V_CMP_NE_I16: opcode = IR::ValueOpcode::INotEqual32; break; + case Decoder::Opcode::V_CMP_GT_U16: opcode = IR::ValueOpcode::UGreaterThan32; break; + case Decoder::Opcode::V_CMP_GE_U16: opcode = IR::ValueOpcode::UGreaterThanEqual32; break; + case Decoder::Opcode::V_CMP_LT_U16: opcode = IR::ValueOpcode::ULessThan32; break; + case Decoder::Opcode::V_CMP_LE_U16: opcode = IR::ValueOpcode::ULessThanEqual32; break; + case Decoder::Opcode::V_CMP_GT_I16: opcode = IR::ValueOpcode::SGreaterThan32; signed_value = true; break; - case IR::Opcode::CompareGeI16: + case Decoder::Opcode::V_CMP_GE_I16: opcode = IR::ValueOpcode::SGreaterThanEqual32; signed_value = true; break; - case IR::Opcode::CompareLtI16: + case Decoder::Opcode::V_CMP_LT_I16: opcode = IR::ValueOpcode::SLessThan32; signed_value = true; break; - case IR::Opcode::CompareLeI16: + case Decoder::Opcode::V_CMP_LE_I16: opcode = IR::ValueOpcode::SLessThanEqual32; signed_value = true; break; default: return false; } - if (inst.op == IR::Opcode::CompareEqI16 || inst.op == IR::Opcode::CompareNeI16) { + if (inst.opcode == Decoder::Opcode::V_CMP_EQ_I16 || + inst.opcode == Decoder::Opcode::V_CMP_NE_I16) { signed_value = true; } - WriteCompareResult(inst.dst, - IR::U1(ir.Emit(opcode, {ReadU16AsU32(inst.src[0], signed_value), - ReadU16AsU32(inst.src[1], signed_value)}))); + WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {ReadU16AsU32(inst.src0, signed_value), + ReadU16AsU32(inst.src1, signed_value)}))); return true; } -bool Translator::TranslateFloatCompare(const IR::Instruction& inst) { +bool Translator::TranslateFloatCompare(const Decoder::Instruction& inst) { IR::ValueOpcode opcode {}; bool half = false; - switch (inst.op) { - case IR::Opcode::CompareEqF32: - case IR::Opcode::CompareMaskEqF32: opcode = IR::ValueOpcode::FPOrdEqual32; break; - case IR::Opcode::CompareNeF32: - case IR::Opcode::CompareMaskNeF32: opcode = IR::ValueOpcode::FPOrdNotEqual32; break; - case IR::Opcode::CompareGtF32: - case IR::Opcode::CompareMaskGtF32: opcode = IR::ValueOpcode::FPOrdGreaterThan32; break; - case IR::Opcode::CompareGeF32: - case IR::Opcode::CompareMaskGeF32: opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32; break; - case IR::Opcode::CompareLtF32: - case IR::Opcode::CompareMaskLtF32: opcode = IR::ValueOpcode::FPOrdLessThan32; break; - case IR::Opcode::CompareLeF32: - case IR::Opcode::CompareMaskLeF32: opcode = IR::ValueOpcode::FPOrdLessThanEqual32; break; - case IR::Opcode::CompareUnordEqF32: - case IR::Opcode::CompareMaskUnordEqF32: opcode = IR::ValueOpcode::FPUnordEqual32; break; - case IR::Opcode::CompareUnordNeF32: - case IR::Opcode::CompareMaskUnordNeF32: opcode = IR::ValueOpcode::FPUnordNotEqual32; break; - case IR::Opcode::CompareUnordGtF32: - case IR::Opcode::CompareMaskUnordGtF32: - opcode = IR::ValueOpcode::FPUnordGreaterThan32; + switch (inst.opcode) { + case Decoder::Opcode::V_CMP_F_F32: + WriteCompareResult(inst.dst, IR::U1(IR::Value(false))); + return true; + case Decoder::Opcode::V_CMP_TRU_F32: + WriteCompareResult(inst.dst, IR::U1(IR::Value(true))); + return true; + case Decoder::Opcode::V_CMP_EQ_F32: + case Decoder::Opcode::V_CMPX_EQ_F32: opcode = IR::ValueOpcode::FPOrdEqual32; break; + case Decoder::Opcode::V_CMP_LG_F32: + case Decoder::Opcode::V_CMPX_LG_F32: opcode = IR::ValueOpcode::FPOrdNotEqual32; break; + case Decoder::Opcode::V_CMP_GT_F32: + case Decoder::Opcode::V_CMPX_GT_F32: opcode = IR::ValueOpcode::FPOrdGreaterThan32; break; + case Decoder::Opcode::V_CMP_GE_F32: + case Decoder::Opcode::V_CMPX_GE_F32: + opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32; break; - case IR::Opcode::CompareUnordGeF32: - case IR::Opcode::CompareMaskUnordGeF32: + case Decoder::Opcode::V_CMP_LT_F32: + case Decoder::Opcode::V_CMPX_LT_F32: opcode = IR::ValueOpcode::FPOrdLessThan32; break; + case Decoder::Opcode::V_CMP_LE_F32: + case Decoder::Opcode::V_CMPX_LE_F32: opcode = IR::ValueOpcode::FPOrdLessThanEqual32; break; + case Decoder::Opcode::V_CMP_NLG_F32: + case Decoder::Opcode::V_CMPX_NLG_F32: opcode = IR::ValueOpcode::FPUnordEqual32; break; + case Decoder::Opcode::V_CMP_NEQ_F32: + case Decoder::Opcode::V_CMPX_NEQ_F32: opcode = IR::ValueOpcode::FPUnordNotEqual32; break; + case Decoder::Opcode::V_CMP_NLE_F32: + case Decoder::Opcode::V_CMPX_NLE_F32: opcode = IR::ValueOpcode::FPUnordGreaterThan32; break; + case Decoder::Opcode::V_CMP_NLT_F32: + case Decoder::Opcode::V_CMPX_NLT_F32: opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32; break; - case IR::Opcode::CompareUnordLtF32: - case IR::Opcode::CompareMaskUnordLtF32: opcode = IR::ValueOpcode::FPUnordLessThan32; break; - case IR::Opcode::CompareUnordLeF32: - case IR::Opcode::CompareMaskUnordLeF32: + case Decoder::Opcode::V_CMP_NGE_F32: + case Decoder::Opcode::V_CMPX_NGE_F32: opcode = IR::ValueOpcode::FPUnordLessThan32; break; + case Decoder::Opcode::V_CMP_NGT_F32: + case Decoder::Opcode::V_CMPX_NGT_F32: opcode = IR::ValueOpcode::FPUnordLessThanEqual32; break; - case IR::Opcode::CompareEqF16: - case IR::Opcode::CompareMaskEqF16: + case Decoder::Opcode::V_CMP_EQ_F16: + case Decoder::Opcode::V_CMPX_EQ_F16: opcode = IR::ValueOpcode::FPOrdEqual32; half = true; break; - case IR::Opcode::CompareNeF16: - case IR::Opcode::CompareMaskNeF16: + case Decoder::Opcode::V_CMP_LG_F16: opcode = IR::ValueOpcode::FPOrdNotEqual32; half = true; break; - case IR::Opcode::CompareGtF16: - case IR::Opcode::CompareMaskGtF16: + case Decoder::Opcode::V_CMP_GT_F16: + case Decoder::Opcode::V_CMPX_GT_F16: opcode = IR::ValueOpcode::FPOrdGreaterThan32; half = true; break; - case IR::Opcode::CompareGeF16: - case IR::Opcode::CompareMaskGeF16: + case Decoder::Opcode::V_CMP_GE_F16: + case Decoder::Opcode::V_CMPX_GE_F16: opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32; half = true; break; - case IR::Opcode::CompareLtF16: - case IR::Opcode::CompareMaskLtF16: + case Decoder::Opcode::V_CMP_LT_F16: + case Decoder::Opcode::V_CMPX_LT_F16: opcode = IR::ValueOpcode::FPOrdLessThan32; half = true; break; - case IR::Opcode::CompareLeF16: - case IR::Opcode::CompareMaskLeF16: + case Decoder::Opcode::V_CMP_LE_F16: + case Decoder::Opcode::V_CMPX_LE_F16: opcode = IR::ValueOpcode::FPOrdLessThanEqual32; half = true; break; - case IR::Opcode::CompareUnordNeF16: - case IR::Opcode::CompareMaskUnordNeF16: + case Decoder::Opcode::V_CMP_NEQ_F16: + case Decoder::Opcode::V_CMPX_NEQ_F16: opcode = IR::ValueOpcode::FPUnordNotEqual32; half = true; break; - case IR::Opcode::CompareMaskUnordGeF16: + case Decoder::Opcode::V_CMPX_NLT_F16: opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32; half = true; break; - case IR::Opcode::CompareOrderedF32: - case IR::Opcode::CompareUnorderedF32: { - const auto lhs = IR::F32(ReadOperand(inst.src[0], IR::Type::F32)); - const auto rhs = IR::F32(ReadOperand(inst.src[1], IR::Type::F32)); + case Decoder::Opcode::V_CMP_O_F32: + case Decoder::Opcode::V_CMP_U_F32: { + const auto lhs = IR::F32(ReadOperand(inst.src0, IR::Type::F32)); + const auto rhs = IR::F32(ReadOperand(inst.src1, IR::Type::F32)); const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {lhs})), IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {rhs}))); const auto result = - inst.op == IR::Opcode::CompareUnorderedF32 ? unordered : ir.LogicalNot(unordered); + inst.opcode == Decoder::Opcode::V_CMP_U_F32 ? unordered : ir.LogicalNot(unordered); WriteCompareResult(inst.dst, result); return true; } - case IR::Opcode::CompareClassF32: { - const auto value = ReadOperand(inst.src[0], IR::Type::F32); - const auto mask = ReadOperand(inst.src[1], IR::Type::U32); + case Decoder::Opcode::V_CMP_CLASS_F32: { + const auto value = ReadOperand(inst.src0, IR::Type::F32); + const auto mask = ReadOperand(inst.src1, IR::Type::U32); WriteCompareResult(inst.dst, IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask}))); return true; @@ -189,11 +212,11 @@ bool Translator::TranslateFloatCompare(const IR::Instruction& inst) { default: return false; } const auto lhs = - half ? IR::Value(ReadF16AsF32(inst.src[0])) : ReadOperand(inst.src[0], IR::Type::F32); + half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32); const auto rhs = - half ? IR::Value(ReadF16AsF32(inst.src[1])) : ReadOperand(inst.src[1], IR::Type::F32); + half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32); WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs}))); return true; } -} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index dbfd29b21..53bd79462 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -1,23 +1,52 @@ #include "graphics/shader/recompiler/frontend/translate/Translator.h" -namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { +namespace Libs::Graphics::ShaderRecompiler::Frontend { +namespace { -bool Translator::TranslateStateOperation(const IR::Instruction& inst) { +bool IsExecOrVcc(const Decoder::Operand& operand) { + switch (operand.kind) { + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: return true; + default: return false; + } +} + +Decoder::Operand ConditionOperand(Decoder::OperandKind kind) { + Decoder::Operand operand; + operand.kind = kind; + return operand; +} + +} // namespace + +bool Translator::TranslateStateOperation(const Decoder::Instruction& inst) { const auto zero = IR::U32(IR::Value(0u)); - switch (inst.op) { - case IR::Opcode::SaveexecB32: - case IR::Opcode::SaveexecB64: { + switch (inst.opcode) { + case Decoder::Opcode::S_AND_SAVEEXEC_B32: + case Decoder::Opcode::S_ANDN1_SAVEEXEC_B32: + case Decoder::Opcode::S_AND_SAVEEXEC_B64: + case Decoder::Opcode::S_ANDN1_SAVEEXEC_B64: + case Decoder::Opcode::S_ORN2_SAVEEXEC_B64: { const auto old = ir.GetExec(); - const auto src = ReadMask(inst.src[0]); + const auto src = ReadMask(inst.src0); IR::U1 result; - switch (inst.saveexec_mode) { - case IR::SaveexecMode::And: result = ir.LogicalAnd(old, src); break; - case IR::SaveexecMode::Andn1: + switch (inst.opcode) { + case Decoder::Opcode::S_AND_SAVEEXEC_B32: + case Decoder::Opcode::S_AND_SAVEEXEC_B64: result = ir.LogicalAnd(old, src); break; + case Decoder::Opcode::S_ANDN1_SAVEEXEC_B32: + case Decoder::Opcode::S_ANDN1_SAVEEXEC_B64: result = ir.LogicalAnd(old, ir.LogicalNot(src)); break; - case IR::SaveexecMode::Orn2: result = ir.LogicalOr(ir.LogicalNot(old), src); break; + case Decoder::Opcode::S_ORN2_SAVEEXEC_B64: + result = ir.LogicalOr(ir.LogicalNot(old), src); + break; + default: EXIT("invalid saveexec opcode"); } - if (inst.op == IR::Opcode::SaveexecB64) { + if (inst.opcode == Decoder::Opcode::S_AND_SAVEEXEC_B64 || + inst.opcode == Decoder::Opcode::S_ANDN1_SAVEEXEC_B64 || + inst.opcode == Decoder::Opcode::S_ORN2_SAVEEXEC_B64) { WriteMask64(inst.dst, old); } else { WriteMask(inst.dst, old); @@ -29,63 +58,75 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) { ir.SetScc(result); return true; } - case IR::Opcode::ScalarAddCarryU32: - case IR::Opcode::IAddCarryU32: { - const auto lhs = ReadU32(inst.src[0]); - const auto rhs = ReadU32(inst.src[1]); - const auto carry_in = ConditionBit(inst.src[2]); + case Decoder::Opcode::S_ADD_U32: + case Decoder::Opcode::S_ADDC_U32: + case Decoder::Opcode::V_ADD_I32: + case Decoder::Opcode::V_ADDC_U32: { + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + const auto carry_in = inst.opcode == Decoder::Opcode::S_ADDC_U32 + ? ConditionBit(ConditionOperand(Decoder::OperandKind::Scc)) + : inst.opcode == Decoder::Opcode::V_ADDC_U32 + ? ConditionBit(inst.src2) + : zero; const auto add0 = ir.Emit(IR::ValueOpcode::IAddCarry32, {lhs, rhs}); const auto partial = ir.CompositeExtract(add0, 0); const auto carry0 = ir.CompositeExtract(add0, 1); const auto add1 = ir.Emit(IR::ValueOpcode::IAddCarry32, {partial, carry_in}); const auto result = ir.CompositeExtract(add1, 0); const auto carry1 = ir.CompositeExtract(add1, 1); - const auto carry = ir.INotEqual(ir.BitwiseOr(carry0, carry1), IR::U32(IR::Value(0u))); - WriteOperand(inst.dst, result); - if (inst.op == IR::Opcode::ScalarAddCarryU32) { + const auto carry = ir.INotEqual(ir.BitwiseOr(carry0, carry1), zero); + WriteOperand(DestinationOperand(inst), result); + if (inst.opcode == Decoder::Opcode::S_ADD_U32 || + inst.opcode == Decoder::Opcode::S_ADDC_U32) { ir.SetScc(carry); } else { WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), carry)); } return true; } - case IR::Opcode::ScalarSubBorrowU32: - case IR::Opcode::ISubBorrowU32: { - const auto lhs = ReadU32(inst.src[0]); - const auto rhs = ReadU32(inst.src[1]); - const auto result = ir.ISub(lhs, rhs); - const auto borrow = ir.UGreaterThan(rhs, lhs); - WriteOperand(inst.dst, result); - if (inst.op == IR::Opcode::ScalarSubBorrowU32) { + case Decoder::Opcode::S_SUB_U32: + case Decoder::Opcode::V_SUB_I32: + case Decoder::Opcode::V_SUBREV_I32: { + const bool reverse = inst.opcode == Decoder::Opcode::V_SUBREV_I32; + const auto lhs = ReadU32(reverse ? inst.src1 : inst.src0); + const auto rhs = ReadU32(reverse ? inst.src0 : inst.src1); + const auto result = ir.ISub(lhs, rhs); + const auto borrow = ir.UGreaterThan(rhs, lhs); + WriteOperand(DestinationOperand(inst), result); + if (inst.opcode == Decoder::Opcode::S_SUB_U32) { ir.SetScc(borrow); } else { WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow)); } return true; } - case IR::Opcode::ScalarSubBorrowCarryU32: - case IR::Opcode::ISubBorrowCarryU32: { - const auto lhs = ReadU32(inst.src[0]); - const auto rhs = ReadU32(inst.src[1]); - const auto borrow_in = ConditionBit(inst.src[2]); + case Decoder::Opcode::S_SUBB_U32: + case Decoder::Opcode::V_SUBREV_CO_CI_U32: { + const bool vector = inst.opcode == Decoder::Opcode::V_SUBREV_CO_CI_U32; + const auto lhs = ReadU32(vector ? inst.src1 : inst.src0); + const auto rhs = ReadU32(vector ? inst.src0 : inst.src1); + const auto borrow_in = vector + ? ConditionBit(inst.src2) + : ConditionBit(ConditionOperand(Decoder::OperandKind::Scc)); const auto partial = ir.ISub(lhs, rhs); const auto result = ir.ISub(partial, borrow_in); const auto borrow0 = ir.UGreaterThan(rhs, lhs); const auto borrow1 = ir.UGreaterThan(borrow_in, partial); - WriteOperand(inst.dst, result); - const auto borrow = ir.LogicalOr(borrow0, borrow1); - if (inst.op == IR::Opcode::ScalarSubBorrowCarryU32) { - ir.SetScc(borrow); - } else { + const auto borrow = ir.LogicalOr(borrow0, borrow1); + WriteOperand(DestinationOperand(inst), result); + if (vector) { WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow)); + } else { + ir.SetScc(borrow); } return true; } - case IR::Opcode::ScalarSignedAddOverflowI32: - case IR::Opcode::ScalarSignedSubOverflowI32: { - const auto lhs = ReadU32(inst.src[0]); - const auto rhs = ReadU32(inst.src[1]); - const bool subtract = inst.op == IR::Opcode::ScalarSignedSubOverflowI32; + case Decoder::Opcode::S_ADD_I32: + case Decoder::Opcode::S_SUB_I32: { + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + const bool subtract = inst.opcode == Decoder::Opcode::S_SUB_I32; const auto result = subtract ? ir.ISub(lhs, rhs) : ir.IAdd(lhs, rhs); const auto shift = IR::U32(IR::Value(31u)); const auto lhs_sign = ir.ShiftRightLogical(lhs, shift); @@ -94,116 +135,201 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) { const auto inputs = subtract ? ir.INotEqual(lhs_sign, rhs_sign) : ir.IEqual(lhs_sign, rhs_sign); const auto changed = ir.INotEqual(lhs_sign, out_sign); - WriteOperand(inst.dst, result); + WriteOperand(DestinationOperand(inst), result); ir.SetScc(ir.LogicalAnd(inputs, changed)); return true; } - case IR::Opcode::ScalarShiftLeftAddCarryU32: { - const auto lhs = ReadU32(inst.src[0]); - const auto shift = ReadU32(inst.src[1]); - const auto rhs = ReadU32(inst.src[2]); - const auto shifted = ir.ShiftLeftLogical(lhs, shift); - const auto result = ir.IAdd(shifted, rhs); - const auto add_carry = ir.ULessThan(result, shifted); - const auto inverse_shift = ir.ISub(IR::U32(IR::Value(32u)), shift); - const auto shifted_out = ir.ShiftRightLogical(lhs, inverse_shift); - const auto shift_carry = ir.INotEqual(shifted_out, zero); - WriteOperand(inst.dst, result); + case Decoder::Opcode::S_LSHL1_ADD_U32: + case Decoder::Opcode::S_LSHL2_ADD_U32: + case Decoder::Opcode::S_LSHL3_ADD_U32: + case Decoder::Opcode::S_LSHL4_ADD_U32: { + const uint32_t shift_amount = 1u + static_cast(inst.opcode) - + static_cast(Decoder::Opcode::S_LSHL1_ADD_U32); + const auto lhs = ReadU32(inst.src0); + const auto shift = IR::U32(IR::Value(shift_amount)); + const auto rhs = ReadU32(inst.src1); + const auto shifted = ir.ShiftLeftLogical(lhs, shift); + const auto result = ir.IAdd(shifted, rhs); + const auto add_carry = ir.ULessThan(result, shifted); + const auto inverse_shift = ir.ISub(IR::U32(IR::Value(32u)), shift); + const auto shifted_out = ir.ShiftRightLogical(lhs, inverse_shift); + const auto shift_carry = ir.INotEqual(shifted_out, zero); + WriteOperand(DestinationOperand(inst), result); ir.SetScc(ir.LogicalOr(add_carry, shift_carry)); return true; } - default: return false; - } -} - -bool Translator::TranslateControlOperation(const IR::Instruction& inst) { - switch (inst.op) { - case IR::Opcode::ControlNop: ir.Emit(IR::ValueOpcode::ControlNop); return true; - case IR::Opcode::Waitcnt: ir.Emit(IR::ValueOpcode::Waitcnt); return true; - case IR::Opcode::Barrier: ir.Emit(IR::ValueOpcode::Barrier); return true; - case IR::Opcode::Sendmsg: ir.Emit(IR::ValueOpcode::Sendmsg); return true; - case IR::Opcode::TtraceData: ir.Emit(IR::ValueOpcode::TtraceData); return true; - case IR::Opcode::InstPrefetch: ir.Emit(IR::ValueOpcode::InstPrefetch); return true; - default: return false; - } -} - -bool Translator::TranslateMove(const IR::Instruction& inst) { - switch (inst.op) { - case IR::Opcode::MoveU32: - if (inst.src[0].kind == IR::OperandKind::Register && - (inst.src[0].reg.file == IR::RegisterFile::Exec || - inst.src[0].reg.file == IR::RegisterFile::Vcc) && - inst.dst.kind == IR::OperandKind::Register && - (inst.dst.reg.file == IR::RegisterFile::Exec || - inst.dst.reg.file == IR::RegisterFile::Vcc)) { - WriteMask(inst.dst, ReadMask(inst.src[0])); - } else { - WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U32)); + case Decoder::Opcode::S_MIN_I32: + case Decoder::Opcode::S_MAX_I32: + case Decoder::Opcode::S_MIN_U32: + case Decoder::Opcode::S_MAX_U32: { + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + IR::ValueOpcode value_opcode; + IR::ValueOpcode compare_opcode; + switch (inst.opcode) { + case Decoder::Opcode::S_MIN_I32: + value_opcode = IR::ValueOpcode::SMin32; + compare_opcode = IR::ValueOpcode::SLessThan32; + break; + case Decoder::Opcode::S_MAX_I32: + value_opcode = IR::ValueOpcode::SMax32; + compare_opcode = IR::ValueOpcode::SGreaterThan32; + break; + case Decoder::Opcode::S_MIN_U32: + value_opcode = IR::ValueOpcode::UMin32; + compare_opcode = IR::ValueOpcode::ULessThan32; + break; + case Decoder::Opcode::S_MAX_U32: + value_opcode = IR::ValueOpcode::UMax32; + compare_opcode = IR::ValueOpcode::UGreaterThan32; + break; + default: EXIT("invalid scalar min/max opcode"); } + WriteOperand(DestinationOperand(inst), ir.Emit(value_opcode, {lhs, rhs})); + ir.SetScc(IR::U1(ir.Emit(compare_opcode, {lhs, rhs}))); return true; - case IR::Opcode::MoveF32Bits: - WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32)); - return true; - case IR::Opcode::MoveU64: { - const auto is_exec_or_vcc = [](const IR::Operand& operand) { - return operand.kind == IR::OperandKind::Register && - (operand.reg.file == IR::RegisterFile::Exec || - operand.reg.file == IR::RegisterFile::Vcc); - }; - if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0])) { - WriteMask64(inst.dst, ReadMask(inst.src[0])); + } + default: return false; + } +} + +bool Translator::TranslateControlOperation(const Decoder::Instruction& inst) { + switch (inst.opcode) { + case Decoder::Opcode::V_NOP: return true; + case Decoder::Opcode::S_NOP: + case Decoder::Opcode::S_SETREG_B32: + case Decoder::Opcode::S_SLEEP: + case Decoder::Opcode::S_TRAP: ir.Emit(IR::ValueOpcode::ControlNop); return true; + case Decoder::Opcode::S_WAITCNT: + case Decoder::Opcode::S_WAITCNT_DEPCTR: ir.Emit(IR::ValueOpcode::Waitcnt); return true; + case Decoder::Opcode::S_BARRIER: ir.Emit(IR::ValueOpcode::Barrier); return true; + case Decoder::Opcode::S_SENDMSG: ir.Emit(IR::ValueOpcode::Sendmsg); return true; + case Decoder::Opcode::S_TTRACEDATA: ir.Emit(IR::ValueOpcode::TtraceData); return true; + case Decoder::Opcode::S_INST_PREFETCH: ir.Emit(IR::ValueOpcode::InstPrefetch); return true; + default: return false; + } +} + +bool Translator::TranslateMove(const Decoder::Instruction& inst, std::string* error) { + switch (inst.opcode) { + case Decoder::Opcode::S_GETPC_B64: { + const auto base = IR::U64(ir.Emit(IR::ValueOpcode::GetShaderBase)); + const auto address = IR::U64(ir.Emit( + IR::ValueOpcode::IAdd64, {base, IR::Value(static_cast(inst.pc) + 4u)})); + if (inst.dst.kind == Decoder::OperandKind::Null) { return true; } - const bool scalar_copy = inst.dst.kind == IR::OperandKind::Register && - inst.src[0].kind == IR::OperandKind::Register && - inst.dst.reg.file == IR::RegisterFile::Scalar && - inst.src[0].reg.file == IR::RegisterFile::Scalar; - IR::U1 source_mask; - IR::U1 source_mask_valid; - if (scalar_copy) { - const auto src = static_cast(inst.src[0].reg.index); - source_mask = ir.GetThreadBitScalarReg(src); - source_mask_valid = ir.GetScalarMaskTag(src); + auto high = PlainOperand(inst.dst); + switch (inst.dst.kind) { + case Decoder::OperandKind::Sgpr: + if (inst.dst.reg < 105u) { + high.reg++; + } else if (inst.dst.reg == 105u) { + high.kind = Decoder::OperandKind::VccLo; + high.reg = 0u; + } else { + if (error != nullptr) { + *error = "S_GETPC_B64 destination does not name a valid scalar pair"; + } + return false; + } + break; + case Decoder::OperandKind::VccLo: high.kind = Decoder::OperandKind::VccHi; break; + case Decoder::OperandKind::M0: high.kind = Decoder::OperandKind::Null; break; + case Decoder::OperandKind::ExecLo: high.kind = Decoder::OperandKind::ExecHi; break; + default: + if (error != nullptr) { + *error = "S_GETPC_B64 destination does not name a valid scalar pair"; + } + return false; } - WriteU32Pair(inst.dst, ReadU32Pair(inst.src[0])); - if (scalar_copy) { - ir.SetThreadBitScalarReg(static_cast(inst.dst.reg.index), - source_mask); - ir.SetScalarMaskTag(static_cast(inst.dst.reg.index), - source_mask_valid); + const auto words = ExtractU64(address); + WriteOperand(inst.dst, words[0]); + WriteOperand(high, words[1]); + return true; + } + case Decoder::Opcode::S_CSELECT_B32: { + const auto result = ir.Select(ir.GetScc(), ReadU32(inst.src0), ReadU32(inst.src1)); + WriteOperand(DestinationOperand(inst), result); + return true; + } + case Decoder::Opcode::S_CSELECT_B64: { + const auto condition = ir.GetScc(); + const auto lhs = ReadU32Pair(inst.src0); + const auto rhs = ReadU32Pair(inst.src1); + const auto selected_mask = IR::U1(ir.Emit( + IR::ValueOpcode::SelectU1, {condition, ReadMask(inst.src0), ReadMask(inst.src1)})); + const auto selected_mask_valid = + IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, + {condition, ReadMaskValid(inst.src0), ReadMaskValid(inst.src1)})); + if (IsExecOrVcc(inst.dst)) { + WriteMask64(inst.dst, selected_mask); + return true; + } + WriteU32Pair(inst.dst, {ir.Select(condition, lhs[0], rhs[0]), + ir.Select(condition, lhs[1], rhs[1])}); + if (inst.dst.kind == Decoder::OperandKind::Sgpr) { + const auto dst = static_cast(inst.dst.reg); + ir.SetThreadBitScalarReg(dst, selected_mask); + ir.SetScalarMaskTag(dst, selected_mask_valid); } return true; } - case IR::Opcode::WqmB64: { - const auto is_exec_or_vcc = [](const IR::Operand& operand) { - return operand.kind == IR::OperandKind::Register && - (operand.reg.file == IR::RegisterFile::Exec || - operand.reg.file == IR::RegisterFile::Vcc); - }; - if (!is_exec_or_vcc(inst.dst) && !is_exec_or_vcc(inst.src[0])) { - const auto mask_valid = ReadMaskValid(inst.src[0]); + case Decoder::Opcode::S_MOV_B32: + case Decoder::Opcode::S_MOVK_I32: + case Decoder::Opcode::V_MOV_B32: + if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) { + WriteMask(inst.dst, ReadMask(inst.src0)); + } else if (inst.opcode == Decoder::Opcode::V_MOV_B32 && + (inst.src0.negate || inst.src0.absolute)) { + WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::F32)); + } else { + WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::U32)); + } + return true; + case Decoder::Opcode::S_MOV_B64: { + if (IsExecOrVcc(inst.dst) || IsExecOrVcc(inst.src0)) { + WriteMask64(inst.dst, ReadMask(inst.src0)); + return true; + } + const bool scalar_copy = inst.dst.kind == Decoder::OperandKind::Sgpr && + inst.src0.kind == Decoder::OperandKind::Sgpr; + IR::U1 source_mask; + IR::U1 source_mask_valid; + if (scalar_copy) { + source_mask = ir.GetThreadBitScalarReg(static_cast(inst.src0.reg)); + source_mask_valid = ir.GetScalarMaskTag(static_cast(inst.src0.reg)); + } + WriteU32Pair(inst.dst, ReadU32Pair(inst.src0)); + if (scalar_copy) { + ir.SetThreadBitScalarReg(static_cast(inst.dst.reg), source_mask); + ir.SetScalarMaskTag(static_cast(inst.dst.reg), source_mask_valid); + } + return true; + } + case Decoder::Opcode::S_WQM_B64: { + if (!IsExecOrVcc(inst.dst) && !IsExecOrVcc(inst.src0)) { + const auto mask_valid = ReadMaskValid(inst.src0); const auto invocation_result = - IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])})); + IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)})); const auto result = IR::U64( - ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)})); - WriteOperand(inst.dst, result); - if (inst.dst.kind == IR::OperandKind::Register && - inst.dst.reg.file == IR::RegisterFile::Scalar) { - const auto dst = static_cast(inst.dst.reg.index); + ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)})); + WriteOperand(DestinationOperand(inst), result); + if (inst.dst.kind == Decoder::OperandKind::Sgpr) { + const auto dst = static_cast(inst.dst.reg); ir.SetThreadBitScalarReg(dst, invocation_result); ir.SetScalarMaskTag(dst, mask_valid); const auto raw_nonzero = IR::U1( ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})); - ir.SetScc(IR::U1(ir.Emit( - IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero}))); + ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, + {mask_valid, invocation_result, raw_nonzero}))); } else { ir.SetScc(IR::U1( ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); } return true; } - const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])})); + const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)})); WriteMask64(inst.dst, result); ir.SetScc(result); return true; @@ -212,29 +338,56 @@ bool Translator::TranslateMove(const IR::Instruction& inst) { } } -bool Translator::TranslateLaneOperation(const IR::Instruction& inst) { +bool Translator::TranslateLaneOperation(const Decoder::Instruction& inst, std::string* error) { const auto lane_mask = IR::U32(IR::Value(current_wave_size == 32u ? 31u : 63u)); - switch (inst.op) { - case IR::Opcode::MoveRelSourceU32: { - EXIT_IF(inst.src[0].kind != IR::OperandKind::Register || - inst.src[0].reg.file != IR::RegisterFile::Vector); - const auto base = inst.src[0].reg.index; - const auto m0 = ir.BitwiseAnd(ReadU32(inst.src[1]), IR::U32(IR::Value(0xffu))); + switch (inst.opcode) { + case Decoder::Opcode::V_MOVRELS_B32: { + if (inst.dst.kind != Decoder::OperandKind::Vgpr || + inst.src0.kind != Decoder::OperandKind::Vgpr) { + if (error != nullptr) { + *error = "V_MOVRELS_B32 requires VGPR source and destination"; + } + return false; + } + if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp || + inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || + inst.src0.absolute || inst.src0.dpp) { + if (error != nullptr) { + *error = "V_MOVRELS_B32 modifiers are not implemented"; + } + return false; + } + const auto base = inst.src0.reg; + const auto m0 = ir.BitwiseAnd(ReadU32(ConditionOperand(Decoder::OperandKind::M0)), + IR::U32(IR::Value(0xffu))); auto selected = ir.GetVectorReg(static_cast(base)); for (uint32_t index = base + 1u; index < current_vector_limit; index++) { const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base))); selected = ir.Select(match, ir.GetVectorReg(static_cast(index)), selected); } - WriteOperand(inst.dst, selected); + WriteOperand(DestinationOperand(inst), selected); return true; } - case IR::Opcode::MoveRelDestU32: { - EXIT_IF(inst.dst.kind != IR::OperandKind::Register || - inst.dst.reg.file != IR::RegisterFile::Vector); - const auto base = inst.dst.reg.index; - const auto value = ReadU32(inst.src[0]); - const auto m0 = ir.BitwiseAnd(ReadU32(inst.src[1]), IR::U32(IR::Value(0xffu))); + case Decoder::Opcode::V_MOVRELD_B32: { + if (inst.dst.kind != Decoder::OperandKind::Vgpr) { + if (error != nullptr) { + *error = "V_MOVRELD_B32 requires VGPR destination"; + } + return false; + } + if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp || + inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || + inst.src0.absolute || inst.src0.dpp) { + if (error != nullptr) { + *error = "V_MOVRELD_B32 modifiers are not implemented"; + } + return false; + } + const auto base = inst.dst.reg; + const auto value = ReadU32(inst.src0); + const auto m0 = ir.BitwiseAnd(ReadU32(ConditionOperand(Decoder::OperandKind::M0)), + IR::U32(IR::Value(0xffu))); for (uint32_t index = base; index < current_vector_limit; index++) { const auto reg = static_cast(index); const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base))); @@ -243,41 +396,38 @@ bool Translator::TranslateLaneOperation(const IR::Instruction& inst) { } return true; } - case IR::Opcode::ReadFirstLaneU32: { + case Decoder::Opcode::V_READFIRSTLANE_B32: { const auto result = - ir.Emit(IR::ValueOpcode::ReadFirstLane, {ReadU32(inst.src[0]), ir.GetExec()}); - WriteOperand(inst.dst, result); + ir.Emit(IR::ValueOpcode::ReadFirstLane, {ReadU32(inst.src0), ir.GetExec()}); + WriteOperand(DestinationOperand(inst), result); return true; } - case IR::Opcode::ReadLaneU32: { - const auto lane = ir.BitwiseAnd(ReadU32(inst.src[1]), lane_mask); - WriteOperand(inst.dst, - ir.Emit(IR::ValueOpcode::ReadLane, {ReadU32(inst.src[0]), lane})); + case Decoder::Opcode::V_READLANE_B32: { + const auto lane = ir.BitwiseAnd(ReadU32(inst.src1), lane_mask); + WriteOperand(DestinationOperand(inst), + ir.Emit(IR::ValueOpcode::ReadLane, {ReadU32(inst.src0), lane})); return true; } - case IR::Opcode::WriteLaneU32: { - EXIT_IF(inst.dst.kind != IR::OperandKind::Register || - inst.dst.reg.file != IR::RegisterFile::Vector); - const auto reg = static_cast(inst.dst.reg.index); - const auto lane = ir.BitwiseAnd(ReadU32(inst.src[1]), lane_mask); - const auto result = IR::U32(ir.Emit( - IR::ValueOpcode::WriteLane, {ir.GetVectorReg(reg), ReadU32(inst.src[0]), lane})); - // V_WRITELANE_B32 ignores EXEC; bypass the ordinary VGPR destination path. + case Decoder::Opcode::V_WRITELANE_B32: { + EXIT_IF(inst.dst.kind != Decoder::OperandKind::Vgpr); + const auto reg = static_cast(inst.dst.reg); + const auto lane = ir.BitwiseAnd(ReadU32(inst.src1), lane_mask); + const auto result = IR::U32(ir.Emit(IR::ValueOpcode::WriteLane, + {ir.GetVectorReg(reg), ReadU32(inst.src0), lane})); ir.SetVectorReg(reg, result); return true; } - case IR::Opcode::Permlane16B32: - case IR::Opcode::Permlanex16B32: { + case Decoder::Opcode::V_PERMLANE16_B32: + case Decoder::Opcode::V_PERMLANEX16_B32: { const IR::PermlaneFlags flags { - .x16 = inst.op == IR::Opcode::Permlanex16B32, + .x16 = inst.opcode == Decoder::Opcode::V_PERMLANEX16_B32, .fetch_inactive = inst.dst.op_sel, .bound_control = inst.dst.op_sel_hi, }; const auto result = ir.Emit( IR::ValueOpcode::Permlane16U32, - {ReadU32(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2]), ir.GetExec()}, - flags); - auto dst = inst.dst; + {ReadU32(inst.src0), ReadU32(inst.src1), ReadU32(inst.src2), ir.GetExec()}, flags); + auto dst = DestinationOperand(inst); dst.op_sel = false; dst.op_sel_hi = false; WriteOperand(dst, result); @@ -287,4 +437,4 @@ bool Translator::TranslateLaneOperation(const IR::Instruction& inst) { } } -} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Convert.cpp b/src/graphics/shader/recompiler/frontend/translate/Convert.cpp index 7ef1a4584..30bd8a9b9 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Convert.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Convert.cpp @@ -1,6 +1,6 @@ #include "graphics/shader/recompiler/frontend/translate/Translator.h" -namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { +namespace Libs::Graphics::ShaderRecompiler::Frontend { IR::F32 Translator::SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value) { return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value})); @@ -38,11 +38,11 @@ IR::U32 Translator::ConvertF32ToI32Saturated(IR::F32 value, float lower_bound, f return ir.Select(nan, IR::U32(IR::Value(0u)), clamped); } -bool Translator::TranslateConversion(const IR::Instruction& inst) { +bool Translator::TranslateConversion(const Decoder::Instruction& inst) { const auto f32 = [&](uint32_t index) { - return IR::F32(ReadOperand(inst.src[index], IR::Type::F32)); + return IR::F32(ReadOperand(SourceAt(inst, index), IR::Type::F32)); }; - const auto u32 = [&](uint32_t index) { return ReadU32(inst.src[index]); }; + const auto u32 = [&](uint32_t index) { return ReadU32(SourceAt(inst, index)); }; const auto convert_u32 = [&](IR::F32 value) { return ConvertF32ToU32Saturated(value, 4294967296.0f, 4294967040.0f, 0xffffffffu); }; @@ -51,65 +51,77 @@ bool Translator::TranslateConversion(const IR::Instruction& inst) { 0x80000000u, 0x7fffffffu); }; - switch (inst.op) { - case IR::Opcode::ConvertByteU32ToF32: { - const auto index = - inst.src[1].kind == IR::OperandKind::ImmediateU32 ? inst.src[1].imm & 3u : 0u; + switch (inst.opcode) { + case Decoder::Opcode::V_CVT_F32_UBYTE0: + case Decoder::Opcode::V_CVT_F32_UBYTE1: + case Decoder::Opcode::V_CVT_F32_UBYTE2: + case Decoder::Opcode::V_CVT_F32_UBYTE3: { + const auto index = static_cast(inst.opcode) - + static_cast(Decoder::Opcode::V_CVT_F32_UBYTE0); const auto byte = ir.BitwiseAnd(ir.ShiftRightLogical(u32(0), IR::U32(IR::Value(index * 8u))), IR::U32(IR::Value(0xffu))); - WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32U32, {byte})); + WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::ConvertF32U32, {byte})); return true; } - case IR::Opcode::ConvertU32ToF32: - WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32U32, {u32(0)})); + case Decoder::Opcode::V_CVT_F32_U32: + WriteOperand(DestinationOperand(inst), + ir.Emit(IR::ValueOpcode::ConvertF32U32, {u32(0)})); return true; - case IR::Opcode::ConvertI32ToF32: - WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32S32, {u32(0)})); + case Decoder::Opcode::V_CVT_F32_I32: + WriteOperand(DestinationOperand(inst), + ir.Emit(IR::ValueOpcode::ConvertF32S32, {u32(0)})); return true; - case IR::Opcode::ConvertF32ToU32: WriteOperand(inst.dst, convert_u32(f32(0))); return true; - case IR::Opcode::ConvertF32ToI32: WriteOperand(inst.dst, convert_i32(f32(0))); return true; - case IR::Opcode::ConvertF32ToF16: WriteF16(inst.dst, f32(0)); return true; - case IR::Opcode::ConvertF16ToF32: - WriteOperand(inst.dst, ReadF16AsF32(inst.src[0])); + case Decoder::Opcode::V_CVT_U32_F32: + WriteOperand(DestinationOperand(inst), convert_u32(f32(0))); return true; - case IR::Opcode::ConvertU16ToF16: { - const auto value = IR::F32( - ir.Emit(IR::ValueOpcode::ConvertF32U32, {ReadU16AsU32(inst.src[0], false)})); - WriteF16(inst.dst, value); + case Decoder::Opcode::V_CVT_I32_F32: + WriteOperand(DestinationOperand(inst), convert_i32(f32(0))); return true; - } - case IR::Opcode::ConvertI16ToF16: { + case Decoder::Opcode::V_CVT_F16_F32: + WriteF16(DestinationOperand(inst), f32(0)); + return true; + case Decoder::Opcode::V_CVT_F32_F16: + WriteOperand(DestinationOperand(inst), ReadF16AsF32(inst.src0)); + return true; + case Decoder::Opcode::V_CVT_F16_U16: { const auto value = - IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {ReadU16AsU32(inst.src[0], true)})); - WriteF16(inst.dst, value); + IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32U32, {ReadU16AsU32(inst.src0, false)})); + WriteF16(DestinationOperand(inst), value); return true; } - case IR::Opcode::ConvertF16ToU16: { + case Decoder::Opcode::V_CVT_F16_I16: { + const auto value = + IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {ReadU16AsU32(inst.src0, true)})); + WriteF16(DestinationOperand(inst), value); + return true; + } + case Decoder::Opcode::V_CVT_U16_F16: { const auto converted = - ConvertF32ToU32Saturated(ReadF16AsF32(inst.src[0]), 65536.0f, 65535.0f, 0xffffu); - WriteU16(inst.dst, converted); + ConvertF32ToU32Saturated(ReadF16AsF32(inst.src0), 65536.0f, 65535.0f, 0xffffu); + WriteU16(DestinationOperand(inst), converted); return true; } - case IR::Opcode::ConvertF16ToI16: { + case Decoder::Opcode::V_CVT_I16_F16: { const auto converted = ConvertF32ToI32Saturated( - ReadF16AsF32(inst.src[0]), -32768.0f, 32768.0f, 32767.0f, 0xffff8000u, 0x7fffu); - WriteU16(inst.dst, ir.BitwiseAnd(converted, IR::U32(IR::Value(0xffffu)))); + ReadF16AsF32(inst.src0), -32768.0f, 32768.0f, 32767.0f, 0xffff8000u, 0x7fffu); + WriteU16(DestinationOperand(inst), + ir.BitwiseAnd(converted, IR::U32(IR::Value(0xffffu)))); return true; } - case IR::Opcode::ConvertRoundPlusInfF32ToI32: { + case Decoder::Opcode::V_CVT_RPI_I32_F32: { const auto biased = IR::F32(ir.Emit(IR::ValueOpcode::FPAdd32, {f32(0), IR::Value::F32(0.5f)})); const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {biased})); - WriteOperand(inst.dst, convert_i32(rounded)); + WriteOperand(DestinationOperand(inst), convert_i32(rounded)); return true; } - case IR::Opcode::ConvertFloorF32ToI32: { + case Decoder::Opcode::V_CVT_FLR_I32_F32: { const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {f32(0)})); - WriteOperand(inst.dst, convert_i32(rounded)); + WriteOperand(DestinationOperand(inst), convert_i32(rounded)); return true; } - case IR::Opcode::FrexpExpI32F32: { + case Decoder::Opcode::V_FREXP_EXP_I32_F32: { const auto bits = ir.BitCastU32(f32(0)); const auto exponent = IR::U32( ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)})); @@ -123,43 +135,45 @@ bool Translator::TranslateConversion(const IR::Instruction& inst) { ir.INotEqual(exponent, IR::U32(IR::Value(0xffu))), ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal, denormal), IR::U32(IR::Value(0u))); - WriteOperand(inst.dst, finite); + WriteOperand(DestinationOperand(inst), finite); return true; } - case IR::Opcode::ConvertI4ToOffsetF32: { + case Decoder::Opcode::V_CVT_OFF_F32_I4: { const auto nibble = IR::U32( ir.Emit(IR::ValueOpcode::BitFieldSExtract, {u32(0), IR::Value(0u), IR::Value(4u)})); const auto value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {nibble})); - WriteOperand(inst.dst, + WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::FPMul32, {value, IR::Value::F32(1.0f / 16.0f)})); return true; } - case IR::Opcode::PackF32ToF16Rtz: - WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::PackFloat2x16Rtz, {f32(0), f32(1)})); + case Decoder::Opcode::V_CVT_PKRTZ_F16_F32: + WriteOperand(DestinationOperand(inst), + ir.Emit(IR::ValueOpcode::PackFloat2x16Rtz, {f32(0), f32(1)})); return true; - case IR::Opcode::PackSnorm2x16F32: - case IR::Opcode::PackUnorm2x16F32: { + case Decoder::Opcode::V_CVT_PKNORM_I16_F32: + case Decoder::Opcode::V_CVT_PKNORM_U16_F32: { const auto pair = ir.Emit(IR::ValueOpcode::CompositeConstructF32x2, {f32(0), f32(1)}); - const auto opcode = inst.op == IR::Opcode::PackSnorm2x16F32 + const auto opcode = inst.opcode == Decoder::Opcode::V_CVT_PKNORM_I16_F32 ? IR::ValueOpcode::PackSnorm2x16 : IR::ValueOpcode::PackUnorm2x16; - WriteOperand(inst.dst, ir.Emit(opcode, {pair})); + WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {pair})); return true; } - case IR::Opcode::PackU8F32: { + case Decoder::Opcode::V_CVT_PK_U8_F32: { const auto byte = ConvertF32ToU32Saturated(f32(0), 255.0f, 255.0f, 255u); const auto index = ir.BitwiseAnd(u32(1), IR::U32(IR::Value(3u))); const auto shift = ir.ShiftLeftLogical(index, IR::U32(IR::Value(3u))); const auto mask = ir.ShiftLeftLogical(IR::U32(IR::Value(0xffu)), shift); const auto base = ir.BitwiseAnd(u32(2), ir.BitwiseNot(mask)); - WriteOperand(inst.dst, ir.BitwiseOr(base, ir.ShiftLeftLogical(byte, shift))); + WriteOperand(DestinationOperand(inst), + ir.BitwiseOr(base, ir.ShiftLeftLogical(byte, shift))); return true; } - case IR::Opcode::PackB32F16: { - const auto low = ReadF16LaneBits(inst.src[0], false); + case Decoder::Opcode::V_PACK_B32_F16: { + const auto low = ReadF16LaneBits(inst.src0, false); const auto high = - ir.ShiftLeftLogical(ReadF16LaneBits(inst.src[1], false), IR::U32(IR::Value(16u))); - WriteOperand(inst.dst, ir.BitwiseOr(low, high)); + ir.ShiftLeftLogical(ReadF16LaneBits(inst.src1, false), IR::U32(IR::Value(16u))); + WriteOperand(DestinationOperand(inst), ir.BitwiseOr(low, high)); return true; } default: return false; @@ -172,4 +186,4 @@ IR::U32 Translator::PackU16Lanes(IR::U32 low, IR::U32 high) { ir.ShiftLeftLogical(ir.BitwiseAnd(high, mask), IR::U32(IR::Value(16u)))); } -} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp index fbee46aed..4222efb1d 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp @@ -1,57 +1,73 @@ #include "graphics/shader/recompiler/frontend/translate/Translator.h" -namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { +namespace Libs::Graphics::ShaderRecompiler::Frontend { -bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) { +bool Translator::TranslateU64MaskOperation(const Decoder::Instruction& inst) { + switch (inst.opcode) { + case Decoder::Opcode::S_AND_B64: + case Decoder::Opcode::S_ANDN2_B64: + case Decoder::Opcode::S_OR_B64: + case Decoder::Opcode::S_ORN2_B64: + case Decoder::Opcode::S_XOR_B64: + case Decoder::Opcode::S_NAND_B64: + case Decoder::Opcode::S_NOR_B64: + case Decoder::Opcode::S_XNOR_B64: + case Decoder::Opcode::S_NOT_B64: break; + default: return false; + } const auto invocation_result = EvaluateU64Mask(inst); - const auto is_exec_or_vcc = [](const IR::Operand& operand) { - return operand.kind == IR::OperandKind::Register && - (operand.reg.file == IR::RegisterFile::Exec || - operand.reg.file == IR::RegisterFile::Vcc); + const auto is_exec_or_vcc = [](const Decoder::Operand& operand) { + switch (operand.kind) { + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: return true; + default: return false; + } }; - if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0]) || - (inst.src_count > 1u && is_exec_or_vcc(inst.src[1]))) { + if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) || + (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) { WriteMask64(inst.dst, invocation_result); ir.SetScc(invocation_result); return true; } - const auto lhs = ReadU32Pair(inst.src[0]); - auto mask_valid = ReadMaskValid(inst.src[0]); + const auto lhs = ReadU32Pair(inst.src0); + auto mask_valid = ReadMaskValid(inst.src0); if (inst.src_count > 1u) { - mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src[1])); + mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src1)); } std::array result; - if (inst.op == IR::Opcode::BitwiseNotU64) { + if (inst.opcode == Decoder::Opcode::S_NOT_B64) { result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])}; } else { - const auto rhs = ReadU32Pair(inst.src[1]); + const auto rhs = ReadU32Pair(inst.src1); for (uint32_t component = 0; component < 2u; component++) { - switch (inst.op) { - case IR::Opcode::BitwiseAndU64: + switch (inst.opcode) { + case Decoder::Opcode::S_AND_B64: result[component] = ir.BitwiseAnd(lhs[component], rhs[component]); break; - case IR::Opcode::BitwiseAndNotU64: + case Decoder::Opcode::S_ANDN2_B64: result[component] = ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component])); break; - case IR::Opcode::BitwiseOrU64: + case Decoder::Opcode::S_OR_B64: result[component] = ir.BitwiseOr(lhs[component], rhs[component]); break; - case IR::Opcode::BitwiseOrNotU64: + case Decoder::Opcode::S_ORN2_B64: result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component])); break; - case IR::Opcode::BitwiseXorU64: + case Decoder::Opcode::S_XOR_B64: result[component] = ir.BitwiseXor(lhs[component], rhs[component]); break; - case IR::Opcode::BitwiseNandU64: + case Decoder::Opcode::S_NAND_B64: result[component] = ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component])); break; - case IR::Opcode::BitwiseNorU64: + case Decoder::Opcode::S_NOR_B64: result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component])); break; - case IR::Opcode::BitwiseXnorU64: + case Decoder::Opcode::S_XNOR_B64: result[component] = ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component])); break; @@ -60,9 +76,8 @@ bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) { } } WriteU32Pair(inst.dst, result); - if (inst.dst.kind == IR::OperandKind::Register && - inst.dst.reg.file == IR::RegisterFile::Scalar) { - const auto dst = static_cast(inst.dst.reg.index); + if (inst.dst.kind == Decoder::OperandKind::Sgpr) { + const auto dst = static_cast(inst.dst.reg); ir.SetThreadBitScalarReg(dst, invocation_result); ir.SetScalarMaskTag(dst, mask_valid); const auto raw_nonzero = @@ -75,29 +90,68 @@ bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) { return true; } -bool Translator::TranslateInstruction(const IR::Instruction& inst) { - switch (IR::GetOpcodeInfo(inst.op).lowering_class) { - case IR::LoweringClass::Control: return TranslateControlOperation(inst); - case IR::LoweringClass::Move: return TranslateMove(inst); - case IR::LoweringClass::Lane: return TranslateLaneOperation(inst); - case IR::LoweringClass::State: return TranslateStateOperation(inst); - case IR::LoweringClass::Memory: return TranslateMemoryOperation(inst); - case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst); - case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst); - case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst); - case IR::LoweringClass::FloatCompare: return TranslateFloatCompare(inst); - case IR::LoweringClass::Conversion: return TranslateConversion(inst); - case IR::LoweringClass::Integer16: return TranslateInteger16Operation(inst); - case IR::LoweringClass::PackedInteger16: return TranslatePackedInteger16(inst); - case IR::LoweringClass::PackedFloat16: return TranslatePackedFloat16(inst); - case IR::LoweringClass::Float16: return TranslateFloat16Operation(inst); - case IR::LoweringClass::Float: return TranslateFloatOperation(inst); - case IR::LoweringClass::U64Mask: return TranslateU64MaskOperation(inst); - case IR::LoweringClass::SimpleInteger: return TranslateSimpleInteger(inst); - case IR::LoweringClass::ComposedInteger: return TranslateComposedInteger(inst); - case IR::LoweringClass::ExtendedInteger: return TranslateExtendedInteger(inst); +bool Translator::TranslateInstruction(const Decoder::Instruction& inst, std::string* error) { + current_opcode = inst.opcode; + current_pc = inst.pc; + + switch (inst.opcode) { + case Decoder::Opcode::UNKNOWN: + case Decoder::Opcode::COUNT: + if (error != nullptr) { + *error = "decoded opcode has no IR translation"; + } + return false; + case Decoder::Opcode::UNSUPPORTED: + if (error != nullptr) { + *error = "unsupported decoded instruction: " + Decoder::InstructionToString(inst); + } + return false; + case Decoder::Opcode::S_GETPC_B64: return TranslateMove(inst, error); + case Decoder::Opcode::V_MOVRELS_B32: + case Decoder::Opcode::V_MOVRELD_B32: return TranslateLaneOperation(inst, error); + default: break; } - return false; + + switch (inst.family) { + case Decoder::Family::SMEM: + case Decoder::Family::MUBUF: + case Decoder::Family::MTBUF: + case Decoder::Family::FLAT: + case Decoder::Family::DS: + case Decoder::Family::MIMG: return TranslateMemoryOperation(inst, error); + case Decoder::Family::EXP: return TranslateAttributeOperation(inst, error); + default: break; + } + + switch (inst.opcode) { + case Decoder::Opcode::V_INTERP_P1_F32: + case Decoder::Opcode::V_INTERP_P2_F32: + case Decoder::Opcode::V_INTERP_MOV_F32: return TranslateAttributeOperation(inst, error); + case Decoder::Opcode::S_BRANCH: + case Decoder::Opcode::S_CBRANCH_SCC0: + case Decoder::Opcode::S_CBRANCH_SCC1: + case Decoder::Opcode::S_CBRANCH_VCCZ: + case Decoder::Opcode::S_CBRANCH_VCCNZ: + case Decoder::Opcode::S_CBRANCH_EXECZ: + case Decoder::Opcode::S_CBRANCH_EXECNZ: + case Decoder::Opcode::S_SETPC_B64: + case Decoder::Opcode::S_ENDPGM: return true; + default: break; + } + + const bool translated = TranslateControlOperation(inst) || TranslateStateOperation(inst) || + TranslateMove(inst, error) || TranslateLaneOperation(inst, error) || + TranslateIntegerCompare(inst) || TranslateInteger16Compare(inst) || + TranslateFloatCompare(inst) || TranslateConversion(inst) || + TranslateInteger16Operation(inst) || TranslatePackedInteger16(inst) || + TranslatePackedFloat16(inst) || TranslateFloat16Operation(inst) || + TranslateFloatOperation(inst) || TranslateU64MaskOperation(inst) || + TranslateSimpleInteger(inst) || TranslateComposedInteger(inst) || + TranslateExtendedInteger(inst); + if (!translated && error != nullptr) { + *error = "decoded opcode has no IR translation"; + } + return translated; } -} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Float.cpp b/src/graphics/shader/recompiler/frontend/translate/Float.cpp index f17946f84..9387f631d 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Float.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Float.cpp @@ -3,36 +3,39 @@ #include #include -namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { +namespace Libs::Graphics::ShaderRecompiler::Frontend { -bool Translator::TranslatePackedFloat16(const IR::Instruction& inst) { +bool Translator::TranslatePackedFloat16(const Decoder::Instruction& inst) { IR::ValueOpcode opcode; - switch (inst.op) { - case IR::Opcode::PackedAddF16: opcode = IR::ValueOpcode::FPAdd32; break; - case IR::Opcode::PackedMulF16: opcode = IR::ValueOpcode::FPMul32; break; - case IR::Opcode::PackedMinF16: opcode = IR::ValueOpcode::FPMin32; break; - case IR::Opcode::PackedMaxF16: opcode = IR::ValueOpcode::FPMax32; break; - case IR::Opcode::PackedFmaF16: opcode = IR::ValueOpcode::FPFma32; break; + switch (inst.opcode) { + case Decoder::Opcode::V_PK_ADD_F16: opcode = IR::ValueOpcode::FPAdd32; break; + case Decoder::Opcode::V_PK_MUL_F16: opcode = IR::ValueOpcode::FPMul32; break; + case Decoder::Opcode::V_PK_MIN_F16: opcode = IR::ValueOpcode::FPMin32; break; + case Decoder::Opcode::V_PK_MAX_F16: opcode = IR::ValueOpcode::FPMax32; break; + case Decoder::Opcode::V_PK_FMA_F16: opcode = IR::ValueOpcode::FPFma32; break; + case Decoder::Opcode::V_PK_FMAC_F16: opcode = IR::ValueOpcode::FPFma32; break; default: return false; } const auto translate_lane = [&](bool high) { - const auto lhs = ReadF16LaneAsF32(inst.src[0], high, true); - const auto rhs = ReadF16LaneAsF32(inst.src[1], high, true); + const auto lhs = ReadF16LaneAsF32(inst.src0, high, true); + const auto rhs = ReadF16LaneAsF32(inst.src1, high, true); IR::F32 result; - if (inst.src_count == 3u) { - result = IR::F32( - ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src[2], high, true)})); + if (inst.opcode == Decoder::Opcode::V_PK_FMAC_F16) { + result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.dst, high, true)})); + } else if (inst.src_count == 3u) { + result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src2, high, true)})); } else { result = IR::F32(ir.Emit(opcode, {lhs, rhs})); } return ApplyF32ResultModifiers(inst.dst, result); }; - auto raw = inst.dst; + auto raw = DestinationOperand(inst); raw.omod = 0u; raw.clamp = false; auto result = PackHalf2x16(translate_lane(false), translate_lane(true)); - if (inst.op == IR::Opcode::PackedMinF16 || inst.op == IR::Opcode::PackedMaxF16) { - const auto quiet_snan = [&](const IR::Operand& operand, bool high) { + if (inst.opcode == Decoder::Opcode::V_PK_MIN_F16 || + inst.opcode == Decoder::Opcode::V_PK_MAX_F16) { + const auto quiet_snan = [&](const Decoder::Operand& operand, bool high) { const auto bits = ReadU16LaneAsU32(operand, high, false); const auto exponent = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x7c00u))); const auto payload = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x01ffu))); @@ -41,8 +44,8 @@ bool Translator::TranslatePackedFloat16(const IR::Instruction& inst) { return std::pair {snan, ir.BitwiseOr(bits, IR::U32(IR::Value(0x0200u)))}; }; const auto override_lane = [&](bool high) { - const auto [lhs_snan, lhs_quiet] = quiet_snan(inst.src[0], high); - const auto [rhs_snan, rhs_quiet] = quiet_snan(inst.src[1], high); + const auto [lhs_snan, lhs_quiet] = quiet_snan(inst.src0, high); + const auto [rhs_snan, rhs_quiet] = quiet_snan(inst.src1, high); const auto normal = high ? ir.ShiftRightLogical(result, IR::U32(IR::Value(16u))) : ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))); return ir.Select(lhs_snan, lhs_quiet, ir.Select(rhs_snan, rhs_quiet, normal)); @@ -53,59 +56,71 @@ bool Translator::TranslatePackedFloat16(const IR::Instruction& inst) { return true; } -bool Translator::TranslateFloat16Operation(const IR::Instruction& inst) { +bool Translator::TranslateFloat16Operation(const Decoder::Instruction& inst) { IR::ValueOpcode opcode; - switch (inst.op) { - case IR::Opcode::AddF16: opcode = IR::ValueOpcode::FPAdd32; break; - case IR::Opcode::SubF16: opcode = IR::ValueOpcode::FPSub32; break; - case IR::Opcode::MulF16: opcode = IR::ValueOpcode::FPMul32; break; - case IR::Opcode::MinF16: opcode = IR::ValueOpcode::FPMin32; break; - case IR::Opcode::MaxF16: opcode = IR::ValueOpcode::FPMax32; break; - case IR::Opcode::FmaF16: - case IR::Opcode::MadMixF16: opcode = IR::ValueOpcode::FPFma32; break; - case IR::Opcode::RcpF16: opcode = IR::ValueOpcode::FPRecip32; break; - case IR::Opcode::SqrtF16: opcode = IR::ValueOpcode::FPSqrt; break; - case IR::Opcode::InverseSqrtF16: opcode = IR::ValueOpcode::FPRecipSqrt32; break; - case IR::Opcode::Log2F16: opcode = IR::ValueOpcode::FPLog2; break; - case IR::Opcode::Exp2F16: opcode = IR::ValueOpcode::FPExp2; break; - case IR::Opcode::FloorF16: opcode = IR::ValueOpcode::FPFloor32; break; - case IR::Opcode::CeilF16: opcode = IR::ValueOpcode::FPCeil32; break; - case IR::Opcode::TruncF16: opcode = IR::ValueOpcode::FPTrunc32; break; - case IR::Opcode::RoundEvenF16: opcode = IR::ValueOpcode::FPRoundEven32; break; - case IR::Opcode::Min3F16: opcode = IR::ValueOpcode::FPMinTri32; break; - case IR::Opcode::Max3F16: opcode = IR::ValueOpcode::FPMaxTri32; break; - case IR::Opcode::Med3F16: opcode = IR::ValueOpcode::FPMedTri32; break; + switch (inst.opcode) { + case Decoder::Opcode::V_ADD_F16: opcode = IR::ValueOpcode::FPAdd32; break; + case Decoder::Opcode::V_SUB_F16: opcode = IR::ValueOpcode::FPSub32; break; + case Decoder::Opcode::V_SUBREV_F16: opcode = IR::ValueOpcode::FPSub32; break; + case Decoder::Opcode::V_MUL_F16: opcode = IR::ValueOpcode::FPMul32; break; + case Decoder::Opcode::V_MIN_F16: opcode = IR::ValueOpcode::FPMin32; break; + case Decoder::Opcode::V_MAX_F16: opcode = IR::ValueOpcode::FPMax32; break; + case Decoder::Opcode::V_FMAC_F16: + case Decoder::Opcode::V_FMAMK_F16: + case Decoder::Opcode::V_FMAAK_F16: + case Decoder::Opcode::V_FMA_F16: + case Decoder::Opcode::V_MAD_MIXLO_F16: opcode = IR::ValueOpcode::FPFma32; break; + case Decoder::Opcode::V_MAD_MIXHI_F16: opcode = IR::ValueOpcode::FPFma32; break; + case Decoder::Opcode::V_RCP_F16: opcode = IR::ValueOpcode::FPRecip32; break; + case Decoder::Opcode::V_SQRT_F16: opcode = IR::ValueOpcode::FPSqrt; break; + case Decoder::Opcode::V_RSQ_F16: opcode = IR::ValueOpcode::FPRecipSqrt32; break; + case Decoder::Opcode::V_LOG_F16: opcode = IR::ValueOpcode::FPLog2; break; + case Decoder::Opcode::V_EXP_F16: opcode = IR::ValueOpcode::FPExp2; break; + case Decoder::Opcode::V_FLOOR_F16: opcode = IR::ValueOpcode::FPFloor32; break; + case Decoder::Opcode::V_CEIL_F16: opcode = IR::ValueOpcode::FPCeil32; break; + case Decoder::Opcode::V_TRUNC_F16: opcode = IR::ValueOpcode::FPTrunc32; break; + case Decoder::Opcode::V_RNDNE_F16: opcode = IR::ValueOpcode::FPRoundEven32; break; + case Decoder::Opcode::V_MIN3_F16: opcode = IR::ValueOpcode::FPMinTri32; break; + case Decoder::Opcode::V_MAX3_F16: opcode = IR::ValueOpcode::FPMaxTri32; break; + case Decoder::Opcode::V_MED3_F16: opcode = IR::ValueOpcode::FPMedTri32; break; default: return false; } + const bool accumulator = inst.opcode == Decoder::Opcode::V_FMAC_F16; + const uint32_t source_count = accumulator ? 3u : inst.src_count; std::array args; - for (uint32_t index = 0; index < inst.src_count; index++) { - args[index] = inst.op == IR::Opcode::MadMixF16 ? IR::Value(ReadMixF32(inst.src[index])) - : IR::Value(ReadF16AsF32(inst.src[index])); + for (uint32_t index = 0; index < source_count; index++) { + auto operand = accumulator && index == 2u ? inst.dst : SourceAt(inst, index); + if (inst.opcode == Decoder::Opcode::V_SUBREV_F16 && index < 2u) { + operand = SourceAt(inst, 1u - index); + } + const bool mix = inst.opcode == Decoder::Opcode::V_MAD_MIXLO_F16 || + inst.opcode == Decoder::Opcode::V_MAD_MIXHI_F16; + args[index] = mix ? IR::Value(ReadMixF32(operand)) : IR::Value(ReadF16AsF32(operand)); } IR::F32 result; - switch (inst.src_count) { + switch (source_count) { case 1: result = IR::F32(ir.Emit(opcode, {args[0]})); break; case 2: result = IR::F32(ir.Emit(opcode, {args[0], args[1]})); break; case 3: result = IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]})); break; default: EXIT("invalid half-float source count: %u", inst.src_count); } - if (inst.op == IR::Opcode::SqrtF16 || inst.op == IR::Opcode::InverseSqrtF16 || - inst.op == IR::Opcode::Log2F16) { - const auto negative = IR::U1( - ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {args[0], IR::Value::F32(0.0f)})); - result = ApplyF32ResultModifiers(inst.dst, result); - const auto bits = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f))); + if (inst.opcode == Decoder::Opcode::V_SQRT_F16 || inst.opcode == Decoder::Opcode::V_RSQ_F16 || + inst.opcode == Decoder::Opcode::V_LOG_F16) { + const auto negative = + IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {args[0], IR::Value::F32(0.0f)})); + result = ApplyF32ResultModifiers(inst.dst, result); + const auto bits = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f))); const auto invalid = IR::U32(IR::Value(inst.dst.clamp ? 0u : 0xfe00u)); - WriteU16(inst.dst, ir.Select(negative, invalid, bits)); + WriteU16(DestinationOperand(inst), ir.Select(negative, invalid, bits)); return true; } - WriteF16(inst.dst, result); + WriteF16(DestinationOperand(inst), result); return true; } -bool Translator::TranslateFloatOperation(const IR::Instruction& inst) { - if (inst.op == IR::Opcode::FrexpMantF32) { - const auto bits = ReadU32(inst.src[0]); +bool Translator::TranslateFloatOperation(const Decoder::Instruction& inst) { + if (inst.opcode == Decoder::Opcode::V_FREXP_MANT_F32) { + const auto bits = ReadU32(inst.src0); const auto exponent = IR::U32( ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)})); const auto mantissa = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu))); @@ -121,84 +136,99 @@ bool Translator::TranslateFloatOperation(const IR::Instruction& inst) { const auto finite = ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal, ir.Select(zero, bits, subnormal)); const auto result = ir.Select(ir.IEqual(exponent, IR::U32(IR::Value(0xffu))), bits, finite); - WriteOperand(inst.dst, ir.BitCastF32(result)); + WriteOperand(DestinationOperand(inst), ir.BitCastF32(result)); return true; } IR::ValueOpcode opcode {}; - switch (inst.op) { - case IR::Opcode::RcpF32: opcode = IR::ValueOpcode::FPRecip32; break; - case IR::Opcode::RcpIflagF32: opcode = IR::ValueOpcode::FPRecipIFlag32; break; - case IR::Opcode::FractF32: opcode = IR::ValueOpcode::FPFract32; break; - case IR::Opcode::TruncF32: opcode = IR::ValueOpcode::FPTrunc32; break; - case IR::Opcode::CeilF32: opcode = IR::ValueOpcode::FPCeil32; break; - case IR::Opcode::RoundEvenF32: opcode = IR::ValueOpcode::FPRoundEven32; break; - case IR::Opcode::FloorF32: opcode = IR::ValueOpcode::FPFloor32; break; - case IR::Opcode::Exp2F32: opcode = IR::ValueOpcode::FPExp2; break; - case IR::Opcode::Log2F32: opcode = IR::ValueOpcode::FPLog2; break; - case IR::Opcode::InverseSqrtF32: opcode = IR::ValueOpcode::FPRecipSqrt32; break; - case IR::Opcode::SqrtF32: opcode = IR::ValueOpcode::FPSqrt; break; - case IR::Opcode::SinF32: opcode = IR::ValueOpcode::FPSin; break; - case IR::Opcode::CosF32: opcode = IR::ValueOpcode::FPCos; break; - case IR::Opcode::FAddF32: opcode = IR::ValueOpcode::FPAdd32; break; - case IR::Opcode::FSubF32: opcode = IR::ValueOpcode::FPSub32; break; - case IR::Opcode::FMulF32: opcode = IR::ValueOpcode::FPMul32; break; - case IR::Opcode::FMinF32: opcode = IR::ValueOpcode::FPMin32; break; - case IR::Opcode::FMaxF32: opcode = IR::ValueOpcode::FPMax32; break; - case IR::Opcode::FMadF32: opcode = IR::ValueOpcode::FPFma32; break; - case IR::Opcode::FMin3F32: opcode = IR::ValueOpcode::FPMinTri32; break; - case IR::Opcode::FMax3F32: opcode = IR::ValueOpcode::FPMaxTri32; break; - case IR::Opcode::FMed3F32: opcode = IR::ValueOpcode::FPMedTri32; break; - case IR::Opcode::LdexpF32: opcode = IR::ValueOpcode::FPLdexp; break; + switch (inst.opcode) { + case Decoder::Opcode::V_RCP_F32: opcode = IR::ValueOpcode::FPRecip32; break; + case Decoder::Opcode::V_RCP_IFLAG_F32: opcode = IR::ValueOpcode::FPRecipIFlag32; break; + case Decoder::Opcode::V_FRACT_F32: opcode = IR::ValueOpcode::FPFract32; break; + case Decoder::Opcode::V_TRUNC_F32: opcode = IR::ValueOpcode::FPTrunc32; break; + case Decoder::Opcode::V_CEIL_F32: opcode = IR::ValueOpcode::FPCeil32; break; + case Decoder::Opcode::V_RNDNE_F32: opcode = IR::ValueOpcode::FPRoundEven32; break; + case Decoder::Opcode::V_FLOOR_F32: opcode = IR::ValueOpcode::FPFloor32; break; + case Decoder::Opcode::V_EXP_F32: opcode = IR::ValueOpcode::FPExp2; break; + case Decoder::Opcode::V_LOG_F32: opcode = IR::ValueOpcode::FPLog2; break; + case Decoder::Opcode::V_RSQ_F32: opcode = IR::ValueOpcode::FPRecipSqrt32; break; + case Decoder::Opcode::V_SQRT_F32: opcode = IR::ValueOpcode::FPSqrt; break; + case Decoder::Opcode::V_SIN_F32: opcode = IR::ValueOpcode::FPSin; break; + case Decoder::Opcode::V_COS_F32: opcode = IR::ValueOpcode::FPCos; break; + case Decoder::Opcode::V_ADD_F32: opcode = IR::ValueOpcode::FPAdd32; break; + case Decoder::Opcode::V_SUB_F32: opcode = IR::ValueOpcode::FPSub32; break; + case Decoder::Opcode::V_SUBREV_F32: opcode = IR::ValueOpcode::FPSub32; break; + case Decoder::Opcode::V_MUL_F32: opcode = IR::ValueOpcode::FPMul32; break; + case Decoder::Opcode::V_MIN_F32: opcode = IR::ValueOpcode::FPMin32; break; + case Decoder::Opcode::V_MAX_F32: opcode = IR::ValueOpcode::FPMax32; break; + case Decoder::Opcode::V_MAC_F32: opcode = IR::ValueOpcode::FPFma32; break; + case Decoder::Opcode::V_MADMK_F32: + case Decoder::Opcode::V_MADAK_F32: + case Decoder::Opcode::V_MAD_F32: + case Decoder::Opcode::V_FMA_F32: opcode = IR::ValueOpcode::FPFma32; break; + case Decoder::Opcode::V_MIN3_F32: opcode = IR::ValueOpcode::FPMinTri32; break; + case Decoder::Opcode::V_MAX3_F32: opcode = IR::ValueOpcode::FPMaxTri32; break; + case Decoder::Opcode::V_MED3_F32: opcode = IR::ValueOpcode::FPMedTri32; break; + case Decoder::Opcode::V_LDEXP_F32: opcode = IR::ValueOpcode::FPLdexp; break; default: break; } if (opcode != IR::ValueOpcode {}) { + const bool accumulator = inst.opcode == Decoder::Opcode::V_MAC_F32; + const bool fused = accumulator || inst.opcode == Decoder::Opcode::V_MADMK_F32 || + inst.opcode == Decoder::Opcode::V_MADAK_F32 || + inst.opcode == Decoder::Opcode::V_MAD_F32 || + inst.opcode == Decoder::Opcode::V_FMA_F32; + const uint32_t source_count = accumulator ? 3u : inst.src_count; std::array args; - for (uint32_t index = 0; index < inst.src_count; index++) { + for (uint32_t index = 0; index < source_count; index++) { const auto arg_type = IR::ArgTypeOf(opcode, index); - args[index] = - arg_type == IR::Type::F32 - ? (inst.op == IR::Opcode::FMadF32 ? IR::Value(ReadMixF32(inst.src[index])) - : ReadOperand(inst.src[index], IR::Type::F32)) - : ReadOperand(inst.src[index], arg_type); + auto operand = accumulator && index == 2u ? inst.dst : SourceAt(inst, index); + if (inst.opcode == Decoder::Opcode::V_SUBREV_F32 && index < 2u) { + operand = SourceAt(inst, 1u - index); + } + args[index] = arg_type == IR::Type::F32 ? (fused ? IR::Value(ReadMixF32(operand)) + : ReadOperand(operand, IR::Type::F32)) + : ReadOperand(operand, arg_type); } IR::Value result; - switch (inst.src_count) { + switch (source_count) { case 1: result = ir.Emit(opcode, {args[0]}); break; case 2: result = ir.Emit(opcode, {args[0], args[1]}); break; case 3: result = ir.Emit(opcode, {args[0], args[1], args[2]}); break; default: EXIT("invalid floating-point source count: %u", inst.src_count); } - WriteOperand(inst.dst, result); + WriteOperand(DestinationOperand(inst), result); return true; } const auto select_f32 = [&](IR::U1 condition, IR::F32 true_value, IR::F32 false_value) { return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value})); }; - if (inst.op == IR::Opcode::Dot2AccF32F16) { - auto a = inst.src[0]; + if (inst.opcode == Decoder::Opcode::V_DOT2C_F32_F16) { + auto a = inst.src0; a.op_sel = false; a.op_sel_hi = true; - auto b = inst.src[1]; + auto b = inst.src1; b.op_sel = false; b.op_sel_hi = true; const auto a_lo = ReadF16LaneAsF32(a, false); const auto a_hi = ReadF16LaneAsF32(a, true); const auto b_lo = ReadF16LaneAsF32(b, false); const auto b_hi = ReadF16LaneAsF32(b, true); - const auto acc = ReadMixF32(inst.src[2]); + const auto acc = ReadMixF32(inst.dst); const auto lo = IR::F32(ir.Emit(IR::ValueOpcode::FPFma32, {a_lo, b_lo, acc})); - WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo})); + WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo})); return true; } - if (inst.op != IR::Opcode::CubeIdF32 && inst.op != IR::Opcode::CubeScF32 && - inst.op != IR::Opcode::CubeTcF32 && inst.op != IR::Opcode::CubeMaF32) { + if (inst.opcode != Decoder::Opcode::V_CUBEID_F32 && + inst.opcode != Decoder::Opcode::V_CUBESC_F32 && + inst.opcode != Decoder::Opcode::V_CUBETC_F32 && + inst.opcode != Decoder::Opcode::V_CUBEMA_F32) { return false; } - const auto x = ReadMixF32(inst.src[0]); - const auto y = ReadMixF32(inst.src[1]); - const auto z = ReadMixF32(inst.src[2]); + const auto x = ReadMixF32(inst.src0); + const auto y = ReadMixF32(inst.src1); + const auto z = ReadMixF32(inst.src2); const auto nx = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {x})); const auto ny = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {y})); const auto nz = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {z})); @@ -216,18 +246,20 @@ bool Translator::TranslateFloatOperation(const IR::Instruction& inst) { return select_f32(z_face, z_value, select_f32(y_face, y_value, x_value)); }; IR::F32 result; - switch (inst.op) { - case IR::Opcode::CubeIdF32: + switch (inst.opcode) { + case Decoder::Opcode::V_CUBEID_F32: result = select_face( select_f32(x_neg, IR::F32(IR::Value::F32(1.0f)), IR::F32(IR::Value::F32(0.0f))), select_f32(y_neg, IR::F32(IR::Value::F32(3.0f)), IR::F32(IR::Value::F32(2.0f))), select_f32(z_neg, IR::F32(IR::Value::F32(5.0f)), IR::F32(IR::Value::F32(4.0f)))); break; - case IR::Opcode::CubeScF32: + case Decoder::Opcode::V_CUBESC_F32: result = select_face(select_f32(x_neg, z, nz), x, select_f32(z_neg, nx, x)); break; - case IR::Opcode::CubeTcF32: result = select_face(ny, select_f32(y_neg, nz, z), ny); break; - case IR::Opcode::CubeMaF32: { + case Decoder::Opcode::V_CUBETC_F32: + result = select_face(ny, select_f32(y_neg, nz, z), ny); + break; + case Decoder::Opcode::V_CUBEMA_F32: { const auto two = IR::F32(IR::Value::F32(2.0f)); result = select_face(IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {x, two})), IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {y, two})), @@ -236,8 +268,8 @@ bool Translator::TranslateFloatOperation(const IR::Instruction& inst) { } default: return false; } - WriteOperand(inst.dst, result); + WriteOperand(DestinationOperand(inst), result); return true; } -} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp index bf69c4c7c..a1aecd150 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -2,22 +2,22 @@ #include -namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { +namespace Libs::Graphics::ShaderRecompiler::Frontend { -bool Translator::TranslateInteger16Operation(const IR::Instruction& inst) { +bool Translator::TranslateInteger16Operation(const Decoder::Instruction& inst) { const auto mask16 = [&](IR::U32 value) { return ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu))); }; IR::Value result; - switch (inst.op) { - case IR::Opcode::ShiftLeftLogicalU16: - case IR::Opcode::ShiftRightLogicalU16: - case IR::Opcode::ShiftRightArithmeticI16: { - const bool arithmetic = inst.op == IR::Opcode::ShiftRightArithmeticI16; - const auto value = ReadU16AsU32(inst.src[0], arithmetic); + switch (inst.opcode) { + case Decoder::Opcode::V_LSHLREV_B16: + case Decoder::Opcode::V_LSHRREV_B16: + case Decoder::Opcode::V_ASHRREV_I16: { + const bool arithmetic = inst.opcode == Decoder::Opcode::V_ASHRREV_I16; + const auto value = ReadU16AsU32(inst.src1, arithmetic); const auto count = - ir.BitwiseAnd(ReadU16AsU32(inst.src[1], false), IR::U32(IR::Value(15u))); - if (inst.op == IR::Opcode::ShiftLeftLogicalU16) { + ir.BitwiseAnd(ReadU16AsU32(inst.src0, false), IR::U32(IR::Value(15u))); + if (inst.opcode == Decoder::Opcode::V_LSHLREV_B16) { result = ir.ShiftLeftLogical(value, count); } else if (arithmetic) { result = ir.ShiftRightArithmetic(value, count); @@ -26,176 +26,207 @@ bool Translator::TranslateInteger16Operation(const IR::Instruction& inst) { } break; } - case IR::Opcode::IAddU16: - result = ir.IAdd(ReadU16AsU32(inst.src[0], false), ReadU16AsU32(inst.src[1], false)); + case Decoder::Opcode::V_ADD_NC_U16: + case Decoder::Opcode::V_ADD_NC_I16: + result = ir.IAdd(ReadU16AsU32(inst.src0, false), ReadU16AsU32(inst.src1, false)); break; - case IR::Opcode::ISubI16: - result = ir.ISub(ReadU16AsU32(inst.src[0], false), ReadU16AsU32(inst.src[1], false)); + case Decoder::Opcode::V_SUB_NC_U16: + case Decoder::Opcode::V_SUB_NC_I16: + result = ir.ISub(ReadU16AsU32(inst.src0, false), ReadU16AsU32(inst.src1, false)); break; - case IR::Opcode::IMed3I16: + case Decoder::Opcode::V_MED3_I16: result = ir.Emit(IR::ValueOpcode::SMedTri32, - {ReadU16AsU32(inst.src[0], true), ReadU16AsU32(inst.src[1], true), - ReadU16AsU32(inst.src[2], true)}); + {ReadU16AsU32(inst.src0, true), ReadU16AsU32(inst.src1, true), + ReadU16AsU32(inst.src2, true)}); break; - case IR::Opcode::IMinI16: - case IR::Opcode::IMaxI16: - case IR::Opcode::UMinU16: - case IR::Opcode::UMaxU16: { - const bool sign = inst.op == IR::Opcode::IMinI16 || inst.op == IR::Opcode::IMaxI16; + case Decoder::Opcode::V_MIN_I16: + case Decoder::Opcode::V_MAX_I16: + case Decoder::Opcode::V_MIN_U16: + case Decoder::Opcode::V_MAX_U16: { + const bool sign = inst.opcode == Decoder::Opcode::V_MIN_I16 || + inst.opcode == Decoder::Opcode::V_MAX_I16; IR::ValueOpcode opcode; - switch (inst.op) { - case IR::Opcode::IMinI16: opcode = IR::ValueOpcode::SMin32; break; - case IR::Opcode::IMaxI16: opcode = IR::ValueOpcode::SMax32; break; - case IR::Opcode::UMinU16: opcode = IR::ValueOpcode::UMin32; break; + switch (inst.opcode) { + case Decoder::Opcode::V_MIN_I16: opcode = IR::ValueOpcode::SMin32; break; + case Decoder::Opcode::V_MAX_I16: opcode = IR::ValueOpcode::SMax32; break; + case Decoder::Opcode::V_MIN_U16: opcode = IR::ValueOpcode::UMin32; break; default: opcode = IR::ValueOpcode::UMax32; break; } result = - ir.Emit(opcode, {ReadU16AsU32(inst.src[0], sign), ReadU16AsU32(inst.src[1], sign)}); + ir.Emit(opcode, {ReadU16AsU32(inst.src0, sign), ReadU16AsU32(inst.src1, sign)}); break; } default: return false; } - WriteU16(inst.dst, mask16(IR::U32(result))); + WriteU16(DestinationOperand(inst), mask16(IR::U32(result))); return true; } -bool Translator::TranslatePackedInteger16(const IR::Instruction& inst) { - const auto lane = [&](const IR::Operand& operand, bool high, bool sign) { +bool Translator::TranslatePackedInteger16(const Decoder::Instruction& inst) { + const auto lane = [&](const Decoder::Operand& operand, bool high, bool sign) { return ReadU16LaneAsU32(operand, high, sign); }; const auto mask_count = [&](IR::U32 value) { return ir.BitwiseAnd(value, IR::U32(IR::Value(15u))); }; const auto translate_lane = [&](bool high) -> IR::U32 { - if (inst.op == IR::Opcode::PackedLshlrevB16 || inst.op == IR::Opcode::PackedLshrrevB16 || - inst.op == IR::Opcode::PackedAshrrevI16) { - const auto count = mask_count(lane(inst.src[0], high, false)); - const bool arithmetic = inst.op == IR::Opcode::PackedAshrrevI16; - const auto value = lane(inst.src[1], high, arithmetic); - if (inst.op == IR::Opcode::PackedLshlrevB16) { + if (inst.opcode == Decoder::Opcode::V_PK_LSHLREV_B16 || + inst.opcode == Decoder::Opcode::V_PK_LSHRREV_B16 || + inst.opcode == Decoder::Opcode::V_PK_ASHRREV_I16) { + const auto count = mask_count(lane(inst.src0, high, false)); + const bool arithmetic = inst.opcode == Decoder::Opcode::V_PK_ASHRREV_I16; + const auto value = lane(inst.src1, high, arithmetic); + if (inst.opcode == Decoder::Opcode::V_PK_LSHLREV_B16) { return ir.ShiftLeftLogical(value, count); } return arithmetic ? ir.ShiftRightArithmetic(value, count) : ir.ShiftRightLogical(value, count); } - const bool signed_minmax = - inst.op == IR::Opcode::PackedMaxI16 || inst.op == IR::Opcode::PackedMinI16; - const auto lhs = lane(inst.src[0], high, signed_minmax); - const auto rhs = lane(inst.src[1], high, signed_minmax); - switch (inst.op) { - case IR::Opcode::PackedMadI16: - case IR::Opcode::PackedMadU16: + const bool signed_minmax = inst.opcode == Decoder::Opcode::V_PK_MAX_I16 || + inst.opcode == Decoder::Opcode::V_PK_MIN_I16; + const auto lhs = lane(inst.src0, high, signed_minmax); + const auto rhs = lane(inst.src1, high, signed_minmax); + switch (inst.opcode) { + case Decoder::Opcode::V_PK_MAD_I16: + case Decoder::Opcode::V_PK_MAD_U16: return ir.IAdd(ir.IMul(lhs, rhs), - lane(inst.src[2], high, inst.op == IR::Opcode::PackedMadI16)); - case IR::Opcode::PackedMulLoU16: return ir.IMul(lhs, rhs); - case IR::Opcode::PackedAddI16: - case IR::Opcode::PackedAddU16: return ir.IAdd(lhs, rhs); - case IR::Opcode::PackedSubI16: - case IR::Opcode::PackedSubU16: return ir.ISub(lhs, rhs); - case IR::Opcode::PackedMaxI16: + lane(inst.src2, high, inst.opcode == Decoder::Opcode::V_PK_MAD_I16)); + case Decoder::Opcode::V_PK_MUL_LO_U16: return ir.IMul(lhs, rhs); + case Decoder::Opcode::V_PK_ADD_I16: + case Decoder::Opcode::V_PK_ADD_U16: return ir.IAdd(lhs, rhs); + case Decoder::Opcode::V_PK_SUB_I16: + case Decoder::Opcode::V_PK_SUB_U16: return ir.ISub(lhs, rhs); + case Decoder::Opcode::V_PK_MAX_I16: return IR::U32(ir.Emit(IR::ValueOpcode::SMax32, {lhs, rhs})); - case IR::Opcode::PackedMinI16: + case Decoder::Opcode::V_PK_MIN_I16: return IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {lhs, rhs})); - case IR::Opcode::PackedMaxU16: + case Decoder::Opcode::V_PK_MAX_U16: return IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs})); - case IR::Opcode::PackedMinU16: + case Decoder::Opcode::V_PK_MIN_U16: return IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs})); default: EXIT("invalid packed integer opcode"); } }; - switch (inst.op) { - case IR::Opcode::PackedMadI16: - case IR::Opcode::PackedMulLoU16: - case IR::Opcode::PackedAddI16: - case IR::Opcode::PackedSubI16: - case IR::Opcode::PackedLshlrevB16: - case IR::Opcode::PackedLshrrevB16: - case IR::Opcode::PackedAshrrevI16: - case IR::Opcode::PackedMaxI16: - case IR::Opcode::PackedMinI16: - case IR::Opcode::PackedMadU16: - case IR::Opcode::PackedAddU16: - case IR::Opcode::PackedSubU16: - case IR::Opcode::PackedMaxU16: - case IR::Opcode::PackedMinU16: - WriteOperand(inst.dst, PackU16Lanes(translate_lane(false), translate_lane(true))); + switch (inst.opcode) { + case Decoder::Opcode::V_PK_MAD_I16: + case Decoder::Opcode::V_PK_MUL_LO_U16: + case Decoder::Opcode::V_PK_ADD_I16: + case Decoder::Opcode::V_PK_SUB_I16: + case Decoder::Opcode::V_PK_LSHLREV_B16: + case Decoder::Opcode::V_PK_LSHRREV_B16: + case Decoder::Opcode::V_PK_ASHRREV_I16: + case Decoder::Opcode::V_PK_MAX_I16: + case Decoder::Opcode::V_PK_MIN_I16: + case Decoder::Opcode::V_PK_MAD_U16: + case Decoder::Opcode::V_PK_ADD_U16: + case Decoder::Opcode::V_PK_SUB_U16: + case Decoder::Opcode::V_PK_MAX_U16: + case Decoder::Opcode::V_PK_MIN_U16: + WriteOperand(DestinationOperand(inst), + PackU16Lanes(translate_lane(false), translate_lane(true))); return true; default: return false; } } -IR::U1 Translator::EvaluateU64Mask(const IR::Instruction& inst) { - const auto lhs = [&] { return ReadMask(inst.src[0]); }; - const auto rhs = [&] { return ReadMask(inst.src[1]); }; - switch (inst.op) { - case IR::Opcode::BitwiseAndU64: return ir.LogicalAnd(lhs(), rhs()); - case IR::Opcode::BitwiseOrU64: return ir.LogicalOr(lhs(), rhs()); - case IR::Opcode::BitwiseXorU64: +IR::U1 Translator::EvaluateU64Mask(const Decoder::Instruction& inst) { + const auto lhs = [&] { return ReadMask(inst.src0); }; + const auto rhs = [&] { return ReadMask(inst.src1); }; + switch (inst.opcode) { + case Decoder::Opcode::S_AND_B64: return ir.LogicalAnd(lhs(), rhs()); + case Decoder::Opcode::S_OR_B64: return ir.LogicalOr(lhs(), rhs()); + case Decoder::Opcode::S_XOR_B64: return IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})); - case IR::Opcode::BitwiseAndNotU64: return ir.LogicalAnd(lhs(), ir.LogicalNot(rhs())); - case IR::Opcode::BitwiseOrNotU64: return ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); - case IR::Opcode::BitwiseNandU64: return ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); - case IR::Opcode::BitwiseNorU64: return ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); - case IR::Opcode::BitwiseXnorU64: + case Decoder::Opcode::S_ANDN2_B64: return ir.LogicalAnd(lhs(), ir.LogicalNot(rhs())); + case Decoder::Opcode::S_ORN2_B64: return ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); + case Decoder::Opcode::S_NAND_B64: return ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); + case Decoder::Opcode::S_NOR_B64: return ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); + case Decoder::Opcode::S_XNOR_B64: return ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}))); - case IR::Opcode::BitwiseNotU64: return ir.LogicalNot(lhs()); + case Decoder::Opcode::S_NOT_B64: return ir.LogicalNot(lhs()); default: EXIT("invalid 64-bit mask opcode"); } } -bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) { +bool Translator::TranslateSimpleInteger(const Decoder::Instruction& inst) { IR::ValueOpcode opcode {}; IR::Type type = IR::Type::U32; - switch (inst.op) { - case IR::Opcode::AbsI32: opcode = IR::ValueOpcode::IAbs32; break; - case IR::Opcode::IAddU32: opcode = IR::ValueOpcode::IAdd32; break; - case IR::Opcode::ISubU32: opcode = IR::ValueOpcode::ISub32; break; - case IR::Opcode::IMulU32: opcode = IR::ValueOpcode::IMul32; break; - case IR::Opcode::UMulHighU32: opcode = IR::ValueOpcode::UMulHi; break; - case IR::Opcode::SMulHighI32: opcode = IR::ValueOpcode::SMulHi; break; - case IR::Opcode::IMinI32: opcode = IR::ValueOpcode::SMin32; break; - case IR::Opcode::IMaxI32: opcode = IR::ValueOpcode::SMax32; break; - case IR::Opcode::UMinU32: opcode = IR::ValueOpcode::UMin32; break; - case IR::Opcode::UMaxU32: opcode = IR::ValueOpcode::UMax32; break; - case IR::Opcode::IMin3I32: opcode = IR::ValueOpcode::SMinTri32; break; - case IR::Opcode::IMax3I32: opcode = IR::ValueOpcode::SMaxTri32; break; - case IR::Opcode::IMed3I32: opcode = IR::ValueOpcode::SMedTri32; break; - case IR::Opcode::UMin3U32: opcode = IR::ValueOpcode::UMinTri32; break; - case IR::Opcode::UMax3U32: opcode = IR::ValueOpcode::UMaxTri32; break; - case IR::Opcode::UMed3U32: opcode = IR::ValueOpcode::UMedTri32; break; - case IR::Opcode::BitwiseAndU32: opcode = IR::ValueOpcode::BitwiseAnd32; break; - case IR::Opcode::BitwiseOrU32: opcode = IR::ValueOpcode::BitwiseOr32; break; - case IR::Opcode::BitwiseXorU32: opcode = IR::ValueOpcode::BitwiseXor32; break; - case IR::Opcode::BitwiseNotU32: opcode = IR::ValueOpcode::BitwiseNot32; break; - case IR::Opcode::BitReverseU32: opcode = IR::ValueOpcode::BitReverse32; break; - case IR::Opcode::BitCountU32: opcode = IR::ValueOpcode::BitCount32; break; - case IR::Opcode::BitCountU64: + switch (inst.opcode) { + case Decoder::Opcode::S_ABS_I32: opcode = IR::ValueOpcode::IAbs32; break; + case Decoder::Opcode::S_MUL_I32: + case Decoder::Opcode::S_MULK_I32: + case Decoder::Opcode::V_MUL_LO_U32: + case Decoder::Opcode::V_MUL_LO_I32: opcode = IR::ValueOpcode::IMul32; break; + case Decoder::Opcode::S_MUL_HI_U32: + case Decoder::Opcode::V_MUL_HI_U32: opcode = IR::ValueOpcode::UMulHi; break; + case Decoder::Opcode::V_MUL_HI_I32: opcode = IR::ValueOpcode::SMulHi; break; + case Decoder::Opcode::V_ADD_NC_U32: opcode = IR::ValueOpcode::IAdd32; break; + case Decoder::Opcode::V_SUB_NC_U32: + case Decoder::Opcode::V_SUBREV_NC_U32: opcode = IR::ValueOpcode::ISub32; break; + case Decoder::Opcode::V_MIN_I32: opcode = IR::ValueOpcode::SMin32; break; + case Decoder::Opcode::V_MAX_I32: opcode = IR::ValueOpcode::SMax32; break; + case Decoder::Opcode::V_MIN_U32: opcode = IR::ValueOpcode::UMin32; break; + case Decoder::Opcode::V_MAX_U32: opcode = IR::ValueOpcode::UMax32; break; + case Decoder::Opcode::V_MIN3_I32: opcode = IR::ValueOpcode::SMinTri32; break; + case Decoder::Opcode::V_MAX3_I32: opcode = IR::ValueOpcode::SMaxTri32; break; + case Decoder::Opcode::V_MED3_I32: opcode = IR::ValueOpcode::SMedTri32; break; + case Decoder::Opcode::V_MIN3_U32: opcode = IR::ValueOpcode::UMinTri32; break; + case Decoder::Opcode::V_MAX3_U32: opcode = IR::ValueOpcode::UMaxTri32; break; + case Decoder::Opcode::V_MED3_U32: opcode = IR::ValueOpcode::UMedTri32; break; + case Decoder::Opcode::S_AND_B32: + case Decoder::Opcode::V_AND_B32: opcode = IR::ValueOpcode::BitwiseAnd32; break; + case Decoder::Opcode::S_OR_B32: + case Decoder::Opcode::V_OR_B32: opcode = IR::ValueOpcode::BitwiseOr32; break; + case Decoder::Opcode::S_XOR_B32: + case Decoder::Opcode::V_XOR_B32: opcode = IR::ValueOpcode::BitwiseXor32; break; + case Decoder::Opcode::S_NOT_B32: + case Decoder::Opcode::V_NOT_B32: opcode = IR::ValueOpcode::BitwiseNot32; break; + case Decoder::Opcode::S_BREV_B32: + case Decoder::Opcode::V_BFREV_B32: opcode = IR::ValueOpcode::BitReverse32; break; + case Decoder::Opcode::S_BCNT1_I32_B32: opcode = IR::ValueOpcode::BitCount32; break; + case Decoder::Opcode::S_BCNT1_I32_B64: opcode = IR::ValueOpcode::BitCount64; type = IR::Type::U64; break; - case IR::Opcode::FindLsbU32: opcode = IR::ValueOpcode::FindILsb32; break; - case IR::Opcode::ShiftLeftLogicalU32: opcode = IR::ValueOpcode::ShiftLeftLogical32; break; - case IR::Opcode::ShiftRightLogicalU32: opcode = IR::ValueOpcode::ShiftRightLogical32; break; - case IR::Opcode::ShiftRightArithmeticI32: + case Decoder::Opcode::S_FF1_I32_B32: + case Decoder::Opcode::V_FFBL_B32: opcode = IR::ValueOpcode::FindILsb32; break; + case Decoder::Opcode::S_LSHL_B32: + case Decoder::Opcode::V_LSHL_B32: + case Decoder::Opcode::V_LSHLREV_B32: opcode = IR::ValueOpcode::ShiftLeftLogical32; break; + case Decoder::Opcode::S_LSHR_B32: + case Decoder::Opcode::V_LSHR_B32: + case Decoder::Opcode::V_LSHRREV_B32: opcode = IR::ValueOpcode::ShiftRightLogical32; break; + case Decoder::Opcode::S_ASHR_I32: + case Decoder::Opcode::V_ASHR_I32: + case Decoder::Opcode::V_ASHRREV_I32: opcode = IR::ValueOpcode::ShiftRightArithmetic32; break; - case IR::Opcode::ShiftLeftLogicalU64: + case Decoder::Opcode::S_LSHL_B64: + case Decoder::Opcode::V_LSHLREV_B64: opcode = IR::ValueOpcode::ShiftLeftLogical64; type = IR::Type::U64; break; - case IR::Opcode::ShiftRightLogicalU64: + case Decoder::Opcode::S_LSHR_B64: + case Decoder::Opcode::V_LSHRREV_B64: opcode = IR::ValueOpcode::ShiftRightLogical64; type = IR::Type::U64; break; default: return false; } + const bool reverse = inst.opcode == Decoder::Opcode::V_SUBREV_NC_U32 || + inst.opcode == Decoder::Opcode::V_LSHLREV_B32 || + inst.opcode == Decoder::Opcode::V_LSHRREV_B32 || + inst.opcode == Decoder::Opcode::V_ASHRREV_I32 || + inst.opcode == Decoder::Opcode::V_LSHLREV_B64 || + inst.opcode == Decoder::Opcode::V_LSHRREV_B64; std::array args; for (uint32_t index = 0; index < inst.src_count; index++) { const auto arg_type = IR::ArgTypeOf(opcode, index); - args[index] = ReadOperand(inst.src[index], arg_type == IR::Type::Void ? type : arg_type); + const auto operand = SourceAt(inst, reverse && index < 2u ? 1u - index : index); + args[index] = ReadOperand(operand, arg_type == IR::Type::Void ? type : arg_type); if (index == 1u && (opcode == IR::ValueOpcode::ShiftLeftLogical32 || opcode == IR::ValueOpcode::ShiftRightLogical32 || opcode == IR::ValueOpcode::ShiftRightArithmetic32)) { @@ -209,56 +240,77 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) { case 3: result = ir.Emit(opcode, {args[0], args[1], args[2]}); break; default: EXIT("invalid simple integer source count: %u", inst.src_count); } - WriteOperand(inst.dst, result); + WriteOperand(DestinationOperand(inst), result); + switch (inst.opcode) { + case Decoder::Opcode::S_ABS_I32: + case Decoder::Opcode::S_BCNT1_I32_B32: + case Decoder::Opcode::S_BCNT1_I32_B64: + case Decoder::Opcode::S_AND_B32: + case Decoder::Opcode::S_OR_B32: + case Decoder::Opcode::S_XOR_B32: + case Decoder::Opcode::S_NOT_B32: + case Decoder::Opcode::S_LSHL_B32: + case Decoder::Opcode::S_LSHR_B32: + case Decoder::Opcode::S_ASHR_I32: + ir.SetScc(ir.INotEqual(IR::U32(result), IR::U32(IR::Value(0u)))); + break; + case Decoder::Opcode::S_LSHL_B64: + case Decoder::Opcode::S_LSHR_B64: + ir.SetScc( + IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); + break; + default: break; + } return true; } -bool Translator::TranslateComposedInteger(const IR::Instruction& inst) { +bool Translator::TranslateComposedInteger(const Decoder::Instruction& inst) { const auto binary_u32 = [&](auto operation) { - const auto lhs = ReadU32(inst.src[0]); - const auto rhs = ReadU32(inst.src[1]); + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); return operation(lhs, rhs); }; IR::Value result; - switch (inst.op) { - case IR::Opcode::BitwiseAndNotU32: + switch (inst.opcode) { + case Decoder::Opcode::S_ANDN2_B32: result = binary_u32( [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseAnd(lhs, ir.BitwiseNot(rhs)); }); break; - case IR::Opcode::BitwiseOrNotU32: + case Decoder::Opcode::S_ORN2_B32: result = binary_u32( [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseOr(lhs, ir.BitwiseNot(rhs)); }); break; - case IR::Opcode::BitwiseNandU32: + case Decoder::Opcode::S_NAND_B32: result = binary_u32( [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseAnd(lhs, rhs)); }); break; - case IR::Opcode::BitwiseNorU32: + case Decoder::Opcode::S_NOR_B32: result = binary_u32( [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseOr(lhs, rhs)); }); break; - case IR::Opcode::BitwiseXnorU32: + case Decoder::Opcode::S_XNOR_B32: + case Decoder::Opcode::V_XNOR_B32: result = binary_u32( [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); }); break; - case IR::Opcode::BitwiseAndOrU32: { - const auto lhs = ReadU32(inst.src[0]); - const auto rhs = ReadU32(inst.src[1]); - const auto add = ReadU32(inst.src[2]); + case Decoder::Opcode::V_AND_OR_B32: { + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + const auto add = ReadU32(inst.src2); result = ir.BitwiseOr(ir.BitwiseAnd(lhs, rhs), add); break; } - case IR::Opcode::BitwiseOr3U32: - result = ir.BitwiseOr(ir.BitwiseOr(ReadU32(inst.src[0]), ReadU32(inst.src[1])), - ReadU32(inst.src[2])); + case Decoder::Opcode::V_OR3_B32: + result = ir.BitwiseOr(ir.BitwiseOr(ReadU32(inst.src0), ReadU32(inst.src1)), + ReadU32(inst.src2)); break; - case IR::Opcode::BitwiseXor3U32: - result = ir.BitwiseXor(ir.BitwiseXor(ReadU32(inst.src[0]), ReadU32(inst.src[1])), - ReadU32(inst.src[2])); + case Decoder::Opcode::V_XOR3_B32: + result = ir.BitwiseXor(ir.BitwiseXor(ReadU32(inst.src0), ReadU32(inst.src1)), + ReadU32(inst.src2)); break; - case IR::Opcode::FindLsbU64: { - const auto source = ExtractU64(ReadU64(inst.src[0])); + case Decoder::Opcode::S_FF1_I32_B64: { + const auto source = ExtractU64(ReadU64(inst.src0)); const auto low_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]})); const auto high_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]})); const auto high_position = ir.IAdd(high_lsb, IR::U32(IR::Value(32u))); @@ -267,16 +319,16 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) { high_position, IR::U32(IR::Value(0xffffffffu)))); break; } - case IR::Opcode::FindMsbFromHighU32: { - const auto source = ReadU32(inst.src[0]); + case Decoder::Opcode::V_FFBH_U32: { + const auto source = ReadU32(inst.src0); const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {source})); const auto position = ir.ISub(IR::U32(IR::Value(31u)), msb); result = ir.Select(ir.INotEqual(source, IR::U32(IR::Value(0u))), position, IR::U32(IR::Value(0xffffffffu))); break; } - case IR::Opcode::FindMsbFromHighU64: { - const auto source = ReadU64(inst.src[0]); + case Decoder::Opcode::S_FLBIT_I32_B64: { + const auto source = ReadU64(inst.src0); const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb64, {source})); const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb); const auto nonzero = IR::U1( @@ -287,11 +339,21 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) { } default: return false; } - WriteOperand(inst.dst, result); + WriteOperand(DestinationOperand(inst), result); + switch (inst.opcode) { + case Decoder::Opcode::S_ANDN2_B32: + case Decoder::Opcode::S_ORN2_B32: + case Decoder::Opcode::S_NAND_B32: + case Decoder::Opcode::S_NOR_B32: + case Decoder::Opcode::S_XNOR_B32: + ir.SetScc(ir.INotEqual(IR::U32(result), IR::U32(IR::Value(0u)))); + break; + default: break; + } return true; } -bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { +bool Translator::TranslateExtendedInteger(const Decoder::Instruction& inst) { const auto imm = [](uint32_t value) { return IR::U32(IR::Value(value)); }; const auto mask = [&](IR::U32 value, uint32_t bits) { return ir.BitwiseAnd(value, imm(bits)); }; const auto extract = [&](IR::U32 value, IR::U32 offset, IR::U32 width, bool sign) { @@ -312,26 +374,26 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { }; IR::Value result; - switch (inst.op) { - case IR::Opcode::IMadI24U32: - case IR::Opcode::UMadU24U32: - case IR::Opcode::IMulI24U32: - case IR::Opcode::UMulU24U32: { - const bool sign = - inst.op == IR::Opcode::IMadI24U32 || inst.op == IR::Opcode::IMulI24U32; - const auto lhs = extract(ReadU32(inst.src[0]), imm(0), imm(24), sign); - const auto rhs = extract(ReadU32(inst.src[1]), imm(0), imm(24), sign); + switch (inst.opcode) { + case Decoder::Opcode::V_MAD_I32_I24: + case Decoder::Opcode::V_MAD_U32_U24: + case Decoder::Opcode::V_MUL_I32_I24: + case Decoder::Opcode::V_MUL_U32_U24: { + const bool sign = inst.opcode == Decoder::Opcode::V_MAD_I32_I24 || + inst.opcode == Decoder::Opcode::V_MUL_I32_I24; + const auto lhs = extract(ReadU32(inst.src0), imm(0), imm(24), sign); + const auto rhs = extract(ReadU32(inst.src1), imm(0), imm(24), sign); auto value = ir.IMul(lhs, rhs); if (inst.src_count == 3) { - value = ir.IAdd(value, ReadU32(inst.src[2])); + value = ir.IAdd(value, ReadU32(inst.src2)); } result = value; break; } - case IR::Opcode::UMadU64U32: { - const auto lhs = ReadU32(inst.src[0]); - const auto rhs = ReadU32(inst.src[1]); - const auto add = ExtractU64(ReadU64(inst.src[2])); + case Decoder::Opcode::V_MAD_U64_U32: { + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + const auto add = ExtractU64(ReadU64(inst.src2)); const auto mul_low = ir.IMul(lhs, rhs); const auto mul_high = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})); const auto low = ir.IAdd(mul_low, add[0]); @@ -340,52 +402,52 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { const auto carry0 = ir.ULessThan(high0, mul_high); const auto high = ir.IAdd(high0, ir.Select(carry_low, imm(1), imm(0))); const auto carry1 = ir.ULessThan(high, high0); - WriteOperand(inst.dst, ir.ConstructU64(low, high)); - if (inst.dst2.kind != IR::OperandKind::Null) { + WriteOperand(DestinationOperand(inst), ir.ConstructU64(low, high)); + if (inst.dst2.kind != Decoder::OperandKind::Null && + inst.dst2.kind != Decoder::OperandKind::Unknown) { WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1)); } return true; } - case IR::Opcode::SadU32: { - const auto lhs = ReadU32(inst.src[0]); - const auto rhs = ReadU32(inst.src[1]); + case Decoder::Opcode::V_SAD_U32: { + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); const auto lo = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs})); const auto hi = IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs})); - result = ir.IAdd(ir.ISub(hi, lo), ReadU32(inst.src[2])); + result = ir.IAdd(ir.ISub(hi, lo), ReadU32(inst.src2)); break; } - case IR::Opcode::IAdd3U32: - result = - ir.IAdd(ir.IAdd(ReadU32(inst.src[0]), ReadU32(inst.src[1])), ReadU32(inst.src[2])); + case Decoder::Opcode::V_ADD3_U32: + result = ir.IAdd(ir.IAdd(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2)); break; - case IR::Opcode::BitClearU32: - case IR::Opcode::BitSetU32: { - const auto bit = ir.ShiftLeftLogical(imm(1), mask(ReadU32(inst.src[1]), 31u)); - result = inst.op == IR::Opcode::BitClearU32 - ? IR::Value(ir.BitwiseAnd(ReadU32(inst.src[0]), ir.BitwiseNot(bit))) - : IR::Value(ir.BitwiseOr(ReadU32(inst.src[0]), bit)); + case Decoder::Opcode::S_BITSET0_B32: + case Decoder::Opcode::S_BITSET1_B32: { + const auto bit = ir.ShiftLeftLogical(imm(1), mask(ReadU32(inst.src0), 31u)); + result = inst.opcode == Decoder::Opcode::S_BITSET0_B32 + ? IR::Value(ir.BitwiseAnd(ReadU32(inst.dst), ir.BitwiseNot(bit))) + : IR::Value(ir.BitwiseOr(ReadU32(inst.dst), bit)); break; } - case IR::Opcode::BitCountAddU32: - result = ir.IAdd(IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {ReadU32(inst.src[0])})), - ReadU32(inst.src[1])); + case Decoder::Opcode::V_BCNT_U32_B32: + result = ir.IAdd(IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {ReadU32(inst.src0)})), + ReadU32(inst.src1)); break; - case IR::Opcode::MaskedBitCountLowU32: - case IR::Opcode::MaskedBitCountHighU32: { + case Decoder::Opcode::V_MBCNT_LO_U32_B32: + case Decoder::Opcode::V_MBCNT_HI_U32_B32: { const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); const auto local = mask(lane, 31u); const auto below = ir.ISub(ir.ShiftLeftLogical(imm(1), local), imm(1)); const auto high_lane = IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, imm(32)})); - const auto thread_mask = inst.op == IR::Opcode::MaskedBitCountLowU32 + const auto thread_mask = inst.opcode == Decoder::Opcode::V_MBCNT_LO_U32_B32 ? ir.Select(high_lane, imm(0xffffffffu), below) : ir.Select(high_lane, below, imm(0)); - const auto active = ir.BitwiseAnd(ReadU32(inst.src[0]), thread_mask); + const auto active = ir.BitwiseAnd(ReadU32(inst.src0), thread_mask); result = ir.IAdd(IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {active})), - ReadU32(inst.src[1])); + ReadU32(inst.src1)); break; } - case IR::Opcode::BitReplicateB64B32: { + case Decoder::Opcode::S_BITREPLICATE_B64_B32: { const auto replicate = [&](IR::U32 value) { auto bits = ir.BitwiseOr(value, ir.ShiftLeftLogical(value, imm(8))); bits = ir.BitwiseAnd(bits, imm(0x00ff00ffu)); @@ -397,12 +459,12 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { bits = ir.BitwiseAnd(bits, imm(0x55555555u)); return ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(1))); }; - const auto source = ReadU32(inst.src[0]); + const auto source = ReadU32(inst.src0); result = ir.ConstructU64(replicate(mask(source, 0xffffu)), replicate(ir.ShiftRightLogical(source, imm(16)))); break; } - case IR::Opcode::QuadmaskB64: { + case Decoder::Opcode::S_QUADMASK_B64: { const auto compact = [&](IR::U32 value) { auto bits = ir.BitwiseOr(value, ir.ShiftRightLogical(value, imm(1))); bits = ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(2))); @@ -411,42 +473,43 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { bits = mask(ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(6))), 0x000f000fu); return mask(ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(12))), 0xffu); }; - const auto source = ReadU32Pair(inst.src[0]); + const auto source = ReadU32Pair(inst.src0); const auto quads = ir.BitwiseOr(compact(source[0]), ir.ShiftLeftLogical(compact(source[1]), imm(8))); result = ir.ConstructU64(quads, imm(0)); break; } - case IR::Opcode::BitFieldMaskU32: { - const auto count = mask(ReadU32(inst.src[0]), 31u); - const auto offset = mask(ReadU32(inst.src[1]), 31u); + case Decoder::Opcode::S_BFM_B32: + case Decoder::Opcode::V_BFM_B32: { + const auto count = mask(ReadU32(inst.src0), 31u); + const auto offset = mask(ReadU32(inst.src1), 31u); result = ir.Emit(IR::ValueOpcode::BitFieldInsert, {imm(0), imm(0xffffffffu), offset, count}); break; } - case IR::Opcode::BitFieldMaskU64: { - const auto count = mask(ReadU32(inst.src[0]), 63u); - const auto offset = mask(ReadU32(inst.src[1]), 63u); + case Decoder::Opcode::S_BFM_B64: { + const auto count = mask(ReadU32(inst.src0), 63u); + const auto offset = mask(ReadU32(inst.src1), 63u); result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {right_mask64(count), offset}); break; } - case IR::Opcode::BitFieldExtractU32: - case IR::Opcode::BitFieldExtractI32: { - const auto source = ReadU32(inst.src[0]); - const auto field = ReadU32(inst.src[1]); + case Decoder::Opcode::S_BFE_U32: + case Decoder::Opcode::S_BFE_I32: { + const auto source = ReadU32(inst.src0); + const auto field = ReadU32(inst.src1); const auto offset = extract(field, imm(0), imm(5), false); const auto count = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {extract(field, imm(16), imm(7), false), ir.ISub(imm(32), offset)})); - const auto opcode = inst.op == IR::Opcode::BitFieldExtractI32 + const auto opcode = inst.opcode == Decoder::Opcode::S_BFE_I32 ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract; result = ir.Emit(opcode, {source, offset, count}); break; } - case IR::Opcode::BitFieldExtractU64: { - const auto source = ReadU64(inst.src[0]); - const auto field = ReadU32(inst.src[1]); + case Decoder::Opcode::S_BFE_U64: { + const auto source = ReadU64(inst.src0); + const auto field = ReadU32(inst.src1); const auto offset = extract(field, imm(0), imm(6), false); const auto raw_count = extract(field, imm(16), imm(7), false); const auto available = ir.ISub(imm(64), offset); @@ -456,49 +519,49 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { result = ir.Emit(IR::ValueOpcode::BitwiseAnd64, {shifted, right_mask64(count)}); break; } - case IR::Opcode::BitFieldExtract3U32: - case IR::Opcode::BitFieldExtract3I32: { - const auto source = ReadU32(inst.src[0]); - const auto offset = mask(ReadU32(inst.src[1]), 31u); + case Decoder::Opcode::V_BFE_U32: + case Decoder::Opcode::V_BFE_I32: { + const auto source = ReadU32(inst.src0); + const auto offset = mask(ReadU32(inst.src1), 31u); const auto count = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, - {mask(ReadU32(inst.src[2]), 31u), ir.ISub(imm(32), offset)})); - const auto opcode = inst.op == IR::Opcode::BitFieldExtract3I32 + {mask(ReadU32(inst.src2), 31u), ir.ISub(imm(32), offset)})); + const auto opcode = inst.opcode == Decoder::Opcode::V_BFE_I32 ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract; result = ir.Emit(opcode, {source, offset, count}); break; } - case IR::Opcode::BitFieldInsertSelectU32: { - const auto bits = ReadU32(inst.src[0]); - const auto insert = ReadU32(inst.src[1]); - const auto base = ReadU32(inst.src[2]); + case Decoder::Opcode::V_BFI_B32: { + const auto bits = ReadU32(inst.src0); + const auto insert = ReadU32(inst.src1); + const auto base = ReadU32(inst.src2); result = ir.BitwiseOr(ir.BitwiseAnd(bits, insert), ir.BitwiseAnd(ir.BitwiseNot(bits), base)); break; } - case IR::Opcode::BitCompare0B32: - case IR::Opcode::BitCompare1B32: { - const auto value = ReadU32(inst.src[0]); - const auto bit = extract(value, mask(ReadU32(inst.src[1]), 31u), imm(1), false); - const auto expected = imm(inst.op == IR::Opcode::BitCompare1B32 ? 1u : 0u); - WriteOperand(inst.dst, ir.IEqual(bit, expected)); + case Decoder::Opcode::S_BITCMP0_B32: + case Decoder::Opcode::S_BITCMP1_B32: { + const auto value = ReadU32(inst.src0); + const auto bit = extract(value, mask(ReadU32(inst.src1), 31u), imm(1), false); + const auto expected = imm(inst.opcode == Decoder::Opcode::S_BITCMP1_B32 ? 1u : 0u); + WriteCompareResult(inst.dst, ir.IEqual(bit, expected)); return true; } - case IR::Opcode::AlignBitU32: { - const auto hi = ReadU32(inst.src[0]); - const auto lo = ReadU32(inst.src[1]); - const auto shift = mask(ReadU32(inst.src[2]), 31u); + case Decoder::Opcode::V_ALIGNBIT_B32: { + const auto hi = ReadU32(inst.src0); + const auto lo = ReadU32(inst.src1); + const auto shift = mask(ReadU32(inst.src2), 31u); const auto lo_part = ir.ShiftRightLogical(lo, shift); const auto hi_part_raw = ir.ShiftLeftLogical(hi, mask(ir.ISub(imm(32), shift), 31u)); const auto hi_part = ir.Select(ir.INotEqual(shift, imm(0)), hi_part_raw, imm(0)); result = ir.BitwiseOr(lo_part, hi_part); break; } - case IR::Opcode::AlignByteU32: { - const auto hi = ReadU32(inst.src[0]); - const auto lo = ReadU32(inst.src[1]); - const auto byte_offset = mask(ReadU32(inst.src[2]), 31u); + case Decoder::Opcode::V_ALIGNBYTE_B32: { + const auto hi = ReadU32(inst.src0); + const auto lo = ReadU32(inst.src1); + const auto byte_offset = mask(ReadU32(inst.src2), 31u); const auto bit_offset = ir.ShiftLeftLogical(byte_offset, imm(3)); const auto concatenated = ir.ConstructU64(lo, hi); const auto shifted = IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, @@ -508,85 +571,68 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { result = ir.Select(in_range, ExtractU64(shifted)[0], imm(0)); break; } - case IR::Opcode::ShiftLeftAddU32: + case Decoder::Opcode::V_LSHL_ADD_U32: + result = ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src0), mask(ReadU32(inst.src1), 31u)), + ReadU32(inst.src2)); + break; + case Decoder::Opcode::V_ADD_LSHL_U32: + result = ir.ShiftLeftLogical(ir.IAdd(ReadU32(inst.src0), ReadU32(inst.src1)), + mask(ReadU32(inst.src2), 31u)); + break; + case Decoder::Opcode::V_XAD_U32: result = - ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src[0]), mask(ReadU32(inst.src[1]), 31u)), - ReadU32(inst.src[2])); + ir.IAdd(ir.BitwiseXor(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2)); break; - case IR::Opcode::AddShiftLeftU32: - result = ir.ShiftLeftLogical(ir.IAdd(ReadU32(inst.src[0]), ReadU32(inst.src[1])), - mask(ReadU32(inst.src[2]), 31u)); - break; - case IR::Opcode::XorAddU32: - result = ir.IAdd(ir.BitwiseXor(ReadU32(inst.src[0]), ReadU32(inst.src[1])), - ReadU32(inst.src[2])); - break; - case IR::Opcode::ShiftLeftOrU32: - result = ir.BitwiseOr( - ir.ShiftLeftLogical(ReadU32(inst.src[0]), mask(ReadU32(inst.src[1]), 31u)), - ReadU32(inst.src[2])); - break; - case IR::Opcode::SelectU32: + case Decoder::Opcode::V_LSHL_OR_B32: result = - ir.Select(ReadCondition(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2])); + ir.BitwiseOr(ir.ShiftLeftLogical(ReadU32(inst.src0), mask(ReadU32(inst.src1), 31u)), + ReadU32(inst.src2)); break; - case IR::Opcode::SelectMaskU32: - result = ir.Select(ReadMask(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2])); - break; - case IR::Opcode::SelectF32Bits: - case IR::Opcode::SelectMaskF32Bits: { - const auto condition = inst.op == IR::Opcode::SelectMaskF32Bits - ? ReadMask(inst.src[0]) - : ReadCondition(inst.src[0]); - result = ir.Emit(IR::ValueOpcode::SelectF32, - {condition, ReadOperand(inst.src[1], IR::Type::F32), - ReadOperand(inst.src[2], IR::Type::F32)}); + case Decoder::Opcode::V_CNDMASK_B32: { + Decoder::Operand mask_operand; + mask_operand.kind = Decoder::OperandKind::VccLo; + if (inst.src_count >= 3u) { + mask_operand = inst.src2; + } + const auto condition = ReadMask(mask_operand); + if (inst.src0.negate || inst.src0.absolute || inst.src1.negate || inst.src1.absolute) { + result = ir.Emit(IR::ValueOpcode::SelectF32, + {condition, ReadOperand(inst.src1, IR::Type::F32), + ReadOperand(inst.src0, IR::Type::F32)}); + } else { + result = ir.Select(condition, ReadU32(inst.src1), ReadU32(inst.src0)); + } break; } - case IR::Opcode::SelectU64: { - const auto condition = ReadCondition(inst.src[0]); - const auto selected_mask = - IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, - {condition, ReadMask(inst.src[1]), ReadMask(inst.src[2])})); - const auto selected_mask_valid = - IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMaskValid(inst.src[1]), - ReadMaskValid(inst.src[2])})); - if (inst.dst.kind == IR::OperandKind::Register && - (inst.dst.reg.file == IR::RegisterFile::Exec || - inst.dst.reg.file == IR::RegisterFile::Vcc)) { - WriteMask64(inst.dst, selected_mask); - return true; - } - const auto lhs = ReadU32Pair(inst.src[1]); - const auto rhs = ReadU32Pair(inst.src[2]); - WriteU32Pair(inst.dst, {ir.Select(condition, lhs[0], rhs[0]), - ir.Select(condition, lhs[1], rhs[1])}); - if (inst.dst.kind == IR::OperandKind::Register && - inst.dst.reg.file == IR::RegisterFile::Scalar) { - const auto dst = static_cast(inst.dst.reg.index); - ir.SetThreadBitScalarReg(dst, selected_mask); - ir.SetScalarMaskTag(dst, selected_mask_valid); - } - return true; - } - case IR::Opcode::PackLowLowU16: - case IR::Opcode::PackLowHighU16: - case IR::Opcode::PackHighHighU16: - case IR::Opcode::PackU16U32: { - const bool high0 = inst.op == IR::Opcode::PackHighHighU16; - const bool high1 = inst.op == IR::Opcode::PackLowHighU16 || high0; + case Decoder::Opcode::S_PACK_LL_B32_B16: + case Decoder::Opcode::S_PACK_LH_B32_B16: + case Decoder::Opcode::S_PACK_HH_B32_B16: + case Decoder::Opcode::V_CVT_PK_U16_U32: + case Decoder::Opcode::V_CVT_PK_I16_I32: { + const bool high0 = inst.opcode == Decoder::Opcode::S_PACK_HH_B32_B16; + const bool high1 = inst.opcode == Decoder::Opcode::S_PACK_LH_B32_B16 || high0; const auto lo = - high0 ? ir.ShiftRightLogical(ReadU32(inst.src[0]), imm(16)) : ReadU32(inst.src[0]); + high0 ? ir.ShiftRightLogical(ReadU32(inst.src0), imm(16)) : ReadU32(inst.src0); const auto hi = - high1 ? ir.ShiftRightLogical(ReadU32(inst.src[1]), imm(16)) : ReadU32(inst.src[1]); + high1 ? ir.ShiftRightLogical(ReadU32(inst.src1), imm(16)) : ReadU32(inst.src1); result = ir.BitwiseOr(mask(lo, 0xffffu), ir.ShiftLeftLogical(mask(hi, 0xffffu), imm(16))); break; } default: return false; } - WriteOperand(inst.dst, result); + WriteOperand(DestinationOperand(inst), result); + switch (inst.opcode) { + case Decoder::Opcode::S_BFE_U32: + case Decoder::Opcode::S_BFE_I32: ir.SetScc(ir.INotEqual(IR::U32(result), imm(0u))); break; + case Decoder::Opcode::S_BFE_U64: + case Decoder::Opcode::S_QUADMASK_B64: + ir.SetScc( + IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); + break; + default: break; + } return true; } -} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 042f643c2..651c948f5 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -1,14 +1,431 @@ +#include "common/magicEnum.h" #include "graphics/shader/recompiler/frontend/translate/Translator.h" #include #include +#include -namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { +namespace Libs::Graphics::ShaderRecompiler::Frontend { -IR::MemoryFlags Translator::AddMemoryInfo(const IR::Instruction& inst) { - const auto index = static_cast(value_program.memory_info.size()); - value_program.memory_info.push_back(inst.memory); - return {.index = index, .pc = inst.pc}; +namespace { + +using IR::ResourceKind; + +const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index) { + switch (index) { + case 0: return decoded.src0; + case 1: return decoded.src1; + case 2: return decoded.src2; + default: return decoded.src3; + } +} + +Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index) { + if (index == 0) { + return operand; + } + auto result = operand; + result.sdwa_sel = 6; + result.sdwa_dst_unused = 2; + result.omod = 0; + result.sdwa_sext = false; + result.op_sel = false; + result.op_sel_hi = false; + result.negate = false; + result.negate_hi = false; + result.absolute = false; + result.dpp_ctrl = 0; + result.dpp_row_mask = 0xf; + result.dpp_bank_mask = 0xf; + result.dpp_fetch_inactive = false; + result.dpp_bound_ctrl = false; + result.dpp = false; + if (result.kind == Decoder::OperandKind::Vgpr || result.kind == Decoder::OperandKind::Sgpr) { + result.reg += index; + } + return result; +} + +uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) { + switch (operand.kind) { + case Decoder::OperandKind::Sgpr: return operand.reg / 4u; + case Decoder::OperandKind::Vgpr: return operand.reg; + case Decoder::OperandKind::IntegerInlineConstant: + case Decoder::OperandKind::LiteralConstant: return operand.value; + default: return 0; + } +} + +uint32_t RawScalarLoadBase(const Decoder::Operand& operand) { + if (operand.kind == Decoder::OperandKind::Sgpr) { + return operand.reg; + } + return operand.kind == Decoder::OperandKind::VccLo ? 106u : 0u; +} + +bool TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code) { + switch (operand.kind) { + case Decoder::OperandKind::Sgpr: code = operand.reg; return true; + case Decoder::OperandKind::VccLo: code = 106u; return true; + case Decoder::OperandKind::VccHi: code = 107u; return true; + case Decoder::OperandKind::ExecLo: code = 126u; return true; + case Decoder::OperandKind::ExecHi: code = 127u; return true; + default: return false; + } +} + +ResourceKind FlatSegmentResourceKind(uint32_t segment) { + switch (segment) { + case 1u: return ResourceKind::Scratch; + case 2u: return ResourceKind::Global; + default: return ResourceKind::Flat; + } +} + +ResourceKind MemoryKind(const Decoder::Instruction& decoded) { + switch (decoded.family) { + case Decoder::Family::SMEM: + return decoded.opcode == Decoder::Opcode::S_LOAD_DWORD || + decoded.opcode == Decoder::Opcode::S_LOAD_DWORDX2 || + decoded.opcode == Decoder::Opcode::S_LOAD_DWORDX4 || + decoded.opcode == Decoder::Opcode::S_LOAD_DWORDX8 || + decoded.opcode == Decoder::Opcode::S_LOAD_DWORDX16 + ? ResourceKind::ScalarAddress + : ResourceKind::ScalarBuffer; + case Decoder::Family::MUBUF: + case Decoder::Family::MTBUF: return ResourceKind::Buffer; + case Decoder::Family::FLAT: return FlatSegmentResourceKind(decoded.memory_segment); + case Decoder::Family::DS: return decoded.gds ? ResourceKind::Gds : ResourceKind::Lds; + case Decoder::Family::MIMG: + switch (decoded.opcode) { + case Decoder::Opcode::IMAGE_STORE: + case Decoder::Opcode::IMAGE_STORE_MIP: return ResourceKind::StorageImage; + case Decoder::Opcode::IMAGE_ATOMIC_ADD: + case Decoder::Opcode::IMAGE_ATOMIC_UMIN: + case Decoder::Opcode::IMAGE_ATOMIC_UMAX: + case Decoder::Opcode::IMAGE_ATOMIC_AND: + case Decoder::Opcode::IMAGE_ATOMIC_OR: + case Decoder::Opcode::IMAGE_ATOMIC_XOR: return ResourceKind::StorageImageUint; + default: return ResourceKind::Image; + } + default: return ResourceKind::None; + } +} + +IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) { + IR::MemoryInfo memory; + memory.kind = MemoryKind(decoded); + memory.offset = decoded.offset; + memory.secondary_offset = decoded.secondary_offset; + memory.dmask = decoded.dmask; + memory.data_dwords = decoded.data_dwords; + memory.data_bits = decoded.data_bits; + memory.component_count = + decoded.data_components != 0u ? decoded.data_components : decoded.data_dwords; + memory.data_format = decoded.data_format; + memory.number_format = decoded.number_format; + memory.image_sample_flags = decoded.image_sample_flags; + memory.image_dimension = decoded.image_dimension; + memory.image_address_components = decoded.image_address_components; + memory.image_nsa_dwords = decoded.image_nsa_dwords; + for (uint32_t index = 0; index < Decoder::MaxImageNsaAddressComponents; index++) { + memory.image_nsa_addr[index] = decoded.image_nsa_addr[index]; + } + memory.memory_segment = decoded.memory_segment; + memory.address_is_full = + memory.kind == ResourceKind::Flat || + (memory.kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr); + memory.data_signed = decoded.data_signed; + memory.typed = decoded.typed; + memory.formatted = decoded.formatted; + memory.image_has_mip = decoded.opcode == Decoder::Opcode::IMAGE_LOAD_MIP || + decoded.opcode == Decoder::Opcode::IMAGE_STORE_MIP; + memory.image_r128 = decoded.image_r128; + memory.glc = decoded.glc; + memory.slc = decoded.slc; + memory.idxen = decoded.idxen; + memory.offen = decoded.offen; + memory.resource = ResourceIndexFromOperand(decoded.src1); + memory.sampler = ResourceIndexFromOperand(decoded.src2); + if (memory.kind == ResourceKind::ScalarAddress) { + memory.resource = RawScalarLoadBase(decoded.src0); + } else if (memory.kind == ResourceKind::ScalarBuffer) { + memory.resource = ResourceIndexFromOperand(decoded.src0); + } else if (memory.kind == ResourceKind::Lds || memory.kind == ResourceKind::Gds || + memory.kind == ResourceKind::Scratch) { + memory.resource = 0; + memory.sampler = 0; + } + if (decoded.opcode == Decoder::Opcode::DS_READ2_B64 || + decoded.opcode == Decoder::Opcode::DS_READ2ST64_B64 || + decoded.opcode == Decoder::Opcode::DS_WRITE2_B64 || + decoded.opcode == Decoder::Opcode::DS_WRITE2ST64_B64) { + memory.data_dwords = 4u; + } else if (decoded.opcode == Decoder::Opcode::DS_READ2_B32 || + decoded.opcode == Decoder::Opcode::DS_READ2ST64_B32 || + decoded.opcode == Decoder::Opcode::DS_WRITE2_B32 || + decoded.opcode == Decoder::Opcode::DS_WRITE2ST64_B32) { + memory.data_dwords = 2u; + } + if (memory.kind == ResourceKind::StorageImage || + memory.kind == ResourceKind::StorageImageUint) { + memory.sampler = 0; + } + return memory; +} + +bool IsScalarAddressLoad(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::S_LOAD_DWORD: + case Decoder::Opcode::S_LOAD_DWORDX2: + case Decoder::Opcode::S_LOAD_DWORDX4: + case Decoder::Opcode::S_LOAD_DWORDX8: + case Decoder::Opcode::S_LOAD_DWORDX16: return true; + default: return false; + } +} + +bool IsScalarBufferLoad(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::S_BUFFER_LOAD_DWORD: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX2: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16: return true; + default: return false; + } +} + +bool IsDsReturnAtomic(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::DS_ADD_RTN_U32: + case Decoder::Opcode::DS_SUB_RTN_U32: + case Decoder::Opcode::DS_MIN_RTN_I32: + case Decoder::Opcode::DS_MAX_RTN_I32: + case Decoder::Opcode::DS_MIN_RTN_U32: + case Decoder::Opcode::DS_MAX_RTN_U32: + case Decoder::Opcode::DS_AND_RTN_B32: + case Decoder::Opcode::DS_OR_RTN_B32: + case Decoder::Opcode::DS_XOR_RTN_B32: + case Decoder::Opcode::DS_WRXCHG_RTN_B32: return true; + default: return false; + } +} + +bool MemoryOpcodeMatchesFamily(Decoder::Opcode opcode, Decoder::Family family) { + switch (family) { + case Decoder::Family::SMEM: + return IsScalarAddressLoad(opcode) || IsScalarBufferLoad(opcode); + case Decoder::Family::MUBUF: + return (opcode >= Decoder::Opcode::BUFFER_LOAD_FORMAT_X && + opcode <= Decoder::Opcode::BUFFER_STORE_DWORDX4) || + (opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP && + opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX); + case Decoder::Family::MTBUF: + return opcode >= Decoder::Opcode::TBUFFER_LOAD_FORMAT_X && + opcode <= Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW; + case Decoder::Family::FLAT: + return opcode >= Decoder::Opcode::FLAT_LOAD_UBYTE && + opcode <= Decoder::Opcode::FLAT_STORE_DWORDX4; + case Decoder::Family::DS: + return opcode >= Decoder::Opcode::DS_ADD_U32 && + opcode <= Decoder::Opcode::DS_READ_ADDTID_B32; + case Decoder::Family::MIMG: + return opcode >= Decoder::Opcode::IMAGE_GET_RESINFO && + opcode <= Decoder::Opcode::IMAGE_GATHER4H; + default: return false; + } +} + +bool UsesDestinationAsMemorySource(Decoder::Opcode opcode) { + return (opcode >= Decoder::Opcode::BUFFER_STORE_FORMAT_X && + opcode <= Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW) || + (opcode >= Decoder::Opcode::BUFFER_STORE_BYTE && + opcode <= Decoder::Opcode::BUFFER_STORE_DWORDX4) || + (opcode >= Decoder::Opcode::TBUFFER_STORE_FORMAT_X && + opcode <= Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW) || + (opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP && + opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX) || + (opcode >= Decoder::Opcode::FLAT_STORE_BYTE && + opcode <= Decoder::Opcode::FLAT_STORE_DWORDX4) || + opcode == Decoder::Opcode::IMAGE_STORE || + opcode == Decoder::Opcode::IMAGE_STORE_MIP || + (opcode >= Decoder::Opcode::IMAGE_ATOMIC_ADD && + opcode <= Decoder::Opcode::IMAGE_ATOMIC_XOR); +} + +bool IsRegisterOperand(const Decoder::Operand& operand) { + switch (operand.kind) { + case Decoder::OperandKind::Sgpr: + case Decoder::OperandKind::Vgpr: + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: + case Decoder::OperandKind::VccZ: + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: + case Decoder::OperandKind::ExecZ: + case Decoder::OperandKind::Scc: + case Decoder::OperandKind::M0: return true; + default: return false; + } +} + +bool RequiresRegisterDestination(const Decoder::Instruction& inst) { + if (IsScalarAddressLoad(inst.opcode) || IsScalarBufferLoad(inst.opcode)) { + return true; + } + if (inst.family == Decoder::Family::MUBUF || inst.family == Decoder::Family::MTBUF) { + const bool load = (inst.opcode >= Decoder::Opcode::BUFFER_LOAD_FORMAT_X && + inst.opcode <= Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW) || + (inst.opcode >= Decoder::Opcode::BUFFER_LOAD_UBYTE && + inst.opcode <= Decoder::Opcode::BUFFER_LOAD_DWORDX4) || + (inst.opcode >= Decoder::Opcode::TBUFFER_LOAD_FORMAT_X && + inst.opcode <= Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZW); + const bool atomic = inst.opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP && + inst.opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX; + return load || (atomic && inst.glc); + } + if (inst.family == Decoder::Family::FLAT) { + return inst.opcode >= Decoder::Opcode::FLAT_LOAD_UBYTE && + inst.opcode <= Decoder::Opcode::FLAT_LOAD_DWORDX4; + } + if (inst.family == Decoder::Family::MIMG) { + const bool atomic = inst.opcode >= Decoder::Opcode::IMAGE_ATOMIC_ADD && + inst.opcode <= Decoder::Opcode::IMAGE_ATOMIC_XOR; + const bool load = inst.opcode == Decoder::Opcode::IMAGE_GET_RESINFO || + inst.opcode == Decoder::Opcode::IMAGE_GET_LOD || + inst.opcode == Decoder::Opcode::IMAGE_LOAD || + inst.opcode == Decoder::Opcode::IMAGE_LOAD_MIP || + inst.opcode == Decoder::Opcode::IMAGE_SAMPLE || + (inst.opcode >= Decoder::Opcode::IMAGE_GATHER4_LZ && + inst.opcode <= Decoder::Opcode::IMAGE_GATHER4H); + return load || (atomic && inst.glc); + } + if (inst.family == Decoder::Family::DS) { + switch (inst.opcode) { + case Decoder::Opcode::DS_READ2_B32: + case Decoder::Opcode::DS_READ2ST64_B32: + case Decoder::Opcode::DS_READ2_B64: + case Decoder::Opcode::DS_READ2ST64_B64: + case Decoder::Opcode::DS_READ_I8: + case Decoder::Opcode::DS_READ_U8: + case Decoder::Opcode::DS_READ_I16: + case Decoder::Opcode::DS_READ_U16: + case Decoder::Opcode::DS_READ_U16_D16: + case Decoder::Opcode::DS_READ_B32: + case Decoder::Opcode::DS_READ_B64: + case Decoder::Opcode::DS_READ_B96: + case Decoder::Opcode::DS_READ_B128: + case Decoder::Opcode::DS_SWIZZLE_B32: + case Decoder::Opcode::DS_CONSUME: + case Decoder::Opcode::DS_APPEND: + case Decoder::Opcode::DS_READ_ADDTID_B32: return true; + default: return IsDsReturnAtomic(inst.opcode); + } + } + return false; +} + +Decoder::Operand MakeM0Operand() { + Decoder::Operand operand; + operand.kind = Decoder::OperandKind::M0; + return operand; +} + +Decoder::Operand MakeImmediate(uint32_t value) { + Decoder::Operand operand; + operand.kind = Decoder::OperandKind::LiteralConstant; + operand.value = value; + return operand; +} + +Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t index) { + if (IsScalarAddressLoad(decoded.opcode) || IsScalarBufferLoad(decoded.opcode)) { + return decoded.src1; + } + if (decoded.family == Decoder::Family::MUBUF || decoded.family == Decoder::Family::MTBUF) { + const bool store_or_atomic = + (decoded.opcode >= Decoder::Opcode::BUFFER_STORE_FORMAT_X && + decoded.opcode <= Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW) || + (decoded.opcode >= Decoder::Opcode::BUFFER_STORE_BYTE && + decoded.opcode <= Decoder::Opcode::BUFFER_STORE_DWORDX4) || + (decoded.opcode >= Decoder::Opcode::TBUFFER_STORE_FORMAT_X && + decoded.opcode <= Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW) || + (decoded.opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP && + decoded.opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX); + uint32_t cursor = 0; + if (store_or_atomic) { + if (index == cursor++) return decoded.dst; + } + if (decoded.idxen) { + if (index == cursor++) return decoded.src0; + } + if (decoded.offen) { + if (index == cursor++) + return decoded.idxen ? OffsetDecodedRegister(decoded.src0, 1u) : decoded.src0; + } + if (index == cursor) return decoded.src2; + return MakeImmediate(0u); + } + if (decoded.family == Decoder::Family::FLAT) { + const bool store = decoded.opcode == Decoder::Opcode::FLAT_STORE_BYTE || + decoded.opcode == Decoder::Opcode::FLAT_STORE_SHORT || + decoded.opcode == Decoder::Opcode::FLAT_STORE_DWORD || + decoded.opcode == Decoder::Opcode::FLAT_STORE_DWORDX2 || + decoded.opcode == Decoder::Opcode::FLAT_STORE_DWORDX3 || + decoded.opcode == Decoder::Opcode::FLAT_STORE_DWORDX4; + if (store) { + return index == 0u ? decoded.dst : DecodedSourceAt(decoded, index - 1u); + } + return DecodedSourceAt(decoded, index); + } + if (decoded.family == Decoder::Family::MIMG) { + const bool store_or_atomic = decoded.opcode == Decoder::Opcode::IMAGE_STORE || + decoded.opcode == Decoder::Opcode::IMAGE_STORE_MIP || + (decoded.opcode >= Decoder::Opcode::IMAGE_ATOMIC_ADD && + decoded.opcode <= Decoder::Opcode::IMAGE_ATOMIC_XOR); + return store_or_atomic ? (index == 0u ? decoded.dst : decoded.src0) : decoded.src0; + } + if (decoded.family == Decoder::Family::DS) { + switch (decoded.opcode) { + case Decoder::Opcode::DS_SWIZZLE_B32: + return index == 0u ? decoded.src0 : MakeImmediate(decoded.offset & 0xffffu); + case Decoder::Opcode::DS_CONSUME: + case Decoder::Opcode::DS_APPEND: + case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand(); + case Decoder::Opcode::DS_WRITE_ADDTID_B32: + return index == 0u ? decoded.src1 : MakeM0Operand(); + case Decoder::Opcode::DS_MIN_F32: + case Decoder::Opcode::DS_MAX_F32: + return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2; + case Decoder::Opcode::DS_WRITE_B8: + case Decoder::Opcode::DS_WRITE_B16: + case Decoder::Opcode::DS_WRITE_B32: + case Decoder::Opcode::DS_WRITE_B64: + case Decoder::Opcode::DS_WRITE_B96: + case Decoder::Opcode::DS_WRITE_B128: + case Decoder::Opcode::DS_WRITE2_B32: + case Decoder::Opcode::DS_WRITE2ST64_B32: + case Decoder::Opcode::DS_WRITE2_B64: + case Decoder::Opcode::DS_WRITE2ST64_B64: + return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2; + default: + if (decoded.opcode >= Decoder::Opcode::DS_ADD_U32 && + decoded.opcode <= Decoder::Opcode::DS_WRXCHG_RTN_B32) { + return index == 0u ? decoded.src1 : decoded.src0; + } + return decoded.src0; + } + } + return DecodedSourceAt(decoded, index); +} + +} // namespace + +IR::MemoryFlags Translator::AddMemoryInfo(const IR::MemoryInfo& memory, uint32_t pc) { + const auto index = static_cast(program.memory_info.size()); + program.memory_info.push_back(memory); + return {.index = index, .pc = pc}; } IR::U32 Translator::GetResourceDword(uint32_t index, uint32_t dword) { @@ -25,20 +442,18 @@ IR::Value Translator::GetAddressResource(IR::Value low, IR::Value high) { return ir.Emit(IR::ValueOpcode::GetAddressResource, {low, high}); } -Translator::AddressOperands Translator::ReadAddressOperands(const IR::Instruction& inst, - uint32_t first_source) { - const auto low = ReadU32(inst.src[first_source]); - const auto& high_or_base = inst.src[first_source + 1u]; - if (inst.memory.kind == IR::ResourceKind::Scratch) { - const auto offset = high_or_base.kind == IR::OperandKind::Register && - high_or_base.reg.file != IR::RegisterFile::Vector - ? ReadU32(high_or_base) - : low; +Translator::AddressOperands Translator::ReadAddressOperands(const Decoder::Instruction& inst, + uint32_t first_source) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto low = ReadU32(MemorySourceAt(inst, first_source)); + const auto high_or_base = MemorySourceAt(inst, first_source + 1u); + if (memory.kind == IR::ResourceKind::Scratch) { + const auto offset = + high_or_base.kind != Decoder::OperandKind::Vgpr ? ReadU32(high_or_base) : low; return {ir.Emit(IR::ValueOpcode::GetScratchResource), offset, IR::Value(0u)}; } - if (inst.memory.kind == IR::ResourceKind::Global && - high_or_base.kind == IR::OperandKind::Register && - high_or_base.reg.file != IR::RegisterFile::Vector) { + if (memory.kind == IR::ResourceKind::Global && + high_or_base.kind != Decoder::OperandKind::Vgpr) { const auto base_low = ReadU32(high_or_base); const auto base_high = ReadU32(OffsetOperand(high_or_base, 1u)); return {GetAddressResource(base_low, base_high), low, IR::Value(0u)}; @@ -54,11 +469,10 @@ IR::Value Translator::GetScalarAddressResource(uint32_t base) { IR::Value Translator::GetImageResource(const IR::MemoryInfo& memory) { const auto dword = [&](uint32_t index) { return memory.image_r128 && index >= 4u ? IR::U32(IR::Value(0u)) - : GetResourceDword(memory.resource, index); + : GetResourceDword(memory.resource, index); }; - return ir.Emit(IR::ValueOpcode::GetImageResource, - {dword(0), dword(1), dword(2), dword(3), dword(4), dword(5), dword(6), - dword(7)}); + return ir.Emit(IR::ValueOpcode::GetImageResource, {dword(0), dword(1), dword(2), dword(3), + dword(4), dword(5), dword(6), dword(7)}); } IR::Value Translator::GetSamplerResource(const IR::MemoryInfo& memory) { @@ -67,15 +481,17 @@ IR::Value Translator::GetSamplerResource(const IR::MemoryInfo& memory) { GetResourceDword(memory.sampler, 2), GetResourceDword(memory.sampler, 3)}); } -IR::Value Translator::MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base) { +IR::Value Translator::MakeImageAddress(const Decoder::Instruction& inst, + const Decoder::Operand& base) { + const auto memory = MemoryInfoFromDecoded(inst); std::array components {}; components[0] = ReadRawU32(PlainOperand(base)); const auto nsa_components = - std::min(inst.memory.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents); + std::min(memory.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents); for (uint32_t index = 1; index < components.size(); index++) { if (index - 1u < nsa_components) { components[index] = - ir.GetVectorReg(static_cast(inst.memory.image_nsa_addr[index - 1u])); + ir.GetVectorReg(static_cast(memory.image_nsa_addr[index - 1u])); } else { components[index] = ReadRawU32(OffsetOperand(PlainOperand(base), index)); } @@ -86,7 +502,7 @@ IR::Value Translator::MakeImageAddress(const IR::Instruction& inst, const IR::Op components[10], components[11], components[12]}); } -IR::Value Translator::ConstructU32x4(const IR::Operand& base, uint32_t count) { +IR::Value Translator::ConstructU32x4(const Decoder::Operand& base, uint32_t count) { std::array components {IR::Value(0u), IR::Value(0u), IR::Value(0u), IR::Value(0u)}; for (uint32_t index = 0; index < std::min(count, 4u); index++) { @@ -96,13 +512,12 @@ IR::Value Translator::ConstructU32x4(const IR::Operand& base, uint32_t count) { {components[0], components[1], components[2], components[3]}); } -void Translator::WriteImageComponents(const IR::Operand& dst, IR::Value value, +void Translator::WriteImageComponents(const Decoder::Operand& dst, IR::Value value, const IR::MemoryInfo& memory, uint32_t component_limit) { if (memory.data_bits == 16u) { for (uint32_t index = 0; index < memory.data_dwords; index++) { - WriteOperand(OffsetOperand(dst, index), - ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, - {value, IR::Value(index)})); + WriteOperand(OffsetOperand(dst, index), ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, + {value, IR::Value(index)})); } return; } @@ -118,26 +533,25 @@ void Translator::WriteImageComponents(const IR::Operand& dst, IR::Value value, } } -IR::ValueOpcode Translator::ImageAtomicOpcode(IR::Opcode opcode) { +IR::ValueOpcode Translator::ImageAtomicOpcode(Decoder::Opcode opcode) { switch (opcode) { - case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::ImageAtomicIAdd32; - case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::ImageAtomicUMin32; - case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::ImageAtomicUMax32; - case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::ImageAtomicAnd32; - case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::ImageAtomicOr32; - case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::ImageAtomicXor32; + case Decoder::Opcode::IMAGE_ATOMIC_ADD: return IR::ValueOpcode::ImageAtomicIAdd32; + case Decoder::Opcode::IMAGE_ATOMIC_UMIN: return IR::ValueOpcode::ImageAtomicUMin32; + case Decoder::Opcode::IMAGE_ATOMIC_UMAX: return IR::ValueOpcode::ImageAtomicUMax32; + case Decoder::Opcode::IMAGE_ATOMIC_AND: return IR::ValueOpcode::ImageAtomicAnd32; + case Decoder::Opcode::IMAGE_ATOMIC_OR: return IR::ValueOpcode::ImageAtomicOr32; + case Decoder::Opcode::IMAGE_ATOMIC_XOR: return IR::ValueOpcode::ImageAtomicXor32; default: EXIT("invalid image atomic opcode"); } } -Translator::BufferAddress Translator::ReadBufferAddress(const IR::Instruction& inst, - uint32_t first_source) { - uint32_t cursor = first_source; - const auto next = [&]() { - return cursor < inst.src_count ? ReadU32(inst.src[cursor++]) : IR::U32(IR::Value(0u)); - }; - const auto index = inst.memory.idxen ? next() : IR::U32(IR::Value(0u)); - const auto offset = inst.memory.offen ? next() : IR::U32(IR::Value(0u)); +Translator::BufferAddress Translator::ReadBufferAddress(const Decoder::Instruction& inst, + uint32_t first_source) { + const auto memory = MemoryInfoFromDecoded(inst); + uint32_t cursor = first_source; + const auto next = [&]() { return ReadU32(MemorySourceAt(inst, cursor++)); }; + const auto index = memory.idxen ? next() : IR::U32(IR::Value(0u)); + const auto offset = memory.offen ? next() : IR::U32(IR::Value(0u)); const auto soffset = next(); return {index, offset, soffset}; } @@ -157,119 +571,128 @@ IR::Value Translator::NarrowSubdword(IR::U32 value, uint32_t bits) { : ir.Emit(IR::ValueOpcode::ConvertU16U32, {value}); } -IR::ValueOpcode Translator::BufferAtomicOpcode(IR::Opcode opcode) { +IR::ValueOpcode Translator::BufferAtomicOpcode(Decoder::Opcode opcode) { switch (opcode) { - case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::BufferAtomicSwap32; - case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::BufferAtomicIAdd32; - case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::BufferAtomicISub32; - case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::BufferAtomicSMin32; - case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::BufferAtomicUMin32; - case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::BufferAtomicSMax32; - case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::BufferAtomicUMax32; - case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::BufferAtomicAnd32; - case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::BufferAtomicOr32; - case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::BufferAtomicXor32; - case IR::Opcode::AtomicFMinF32: return IR::ValueOpcode::BufferAtomicFMin32; - case IR::Opcode::AtomicFMaxF32: return IR::ValueOpcode::BufferAtomicFMax32; + case Decoder::Opcode::BUFFER_ATOMIC_SWAP: return IR::ValueOpcode::BufferAtomicSwap32; + case Decoder::Opcode::BUFFER_ATOMIC_ADD: return IR::ValueOpcode::BufferAtomicIAdd32; + case Decoder::Opcode::BUFFER_ATOMIC_SUB: return IR::ValueOpcode::BufferAtomicISub32; + case Decoder::Opcode::BUFFER_ATOMIC_SMIN: return IR::ValueOpcode::BufferAtomicSMin32; + case Decoder::Opcode::BUFFER_ATOMIC_UMIN: return IR::ValueOpcode::BufferAtomicUMin32; + case Decoder::Opcode::BUFFER_ATOMIC_SMAX: return IR::ValueOpcode::BufferAtomicSMax32; + case Decoder::Opcode::BUFFER_ATOMIC_UMAX: return IR::ValueOpcode::BufferAtomicUMax32; + case Decoder::Opcode::BUFFER_ATOMIC_AND: return IR::ValueOpcode::BufferAtomicAnd32; + case Decoder::Opcode::BUFFER_ATOMIC_OR: return IR::ValueOpcode::BufferAtomicOr32; + case Decoder::Opcode::BUFFER_ATOMIC_XOR: return IR::ValueOpcode::BufferAtomicXor32; + case Decoder::Opcode::BUFFER_ATOMIC_FMIN: return IR::ValueOpcode::BufferAtomicFMin32; + case Decoder::Opcode::BUFFER_ATOMIC_FMAX: return IR::ValueOpcode::BufferAtomicFMax32; default: EXIT("invalid buffer atomic opcode"); } } -IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode) { +IR::ValueOpcode Translator::SharedAtomicOpcode(Decoder::Opcode opcode) { switch (opcode) { - case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::SharedAtomicSwap32; - case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::SharedAtomicIAdd32; - case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::SharedAtomicISub32; - case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::SharedAtomicSMin32; - case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::SharedAtomicUMin32; - case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::SharedAtomicSMax32; - case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::SharedAtomicUMax32; - case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::SharedAtomicAnd32; - case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::SharedAtomicOr32; - case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::SharedAtomicXor32; + case Decoder::Opcode::DS_WRXCHG_RTN_B32: return IR::ValueOpcode::SharedAtomicSwap32; + case Decoder::Opcode::DS_ADD_U32: + case Decoder::Opcode::DS_ADD_RTN_U32: return IR::ValueOpcode::SharedAtomicIAdd32; + case Decoder::Opcode::DS_SUB_U32: + case Decoder::Opcode::DS_SUB_RTN_U32: return IR::ValueOpcode::SharedAtomicISub32; + case Decoder::Opcode::DS_MIN_I32: + case Decoder::Opcode::DS_MIN_RTN_I32: return IR::ValueOpcode::SharedAtomicSMin32; + case Decoder::Opcode::DS_MIN_U32: + case Decoder::Opcode::DS_MIN_RTN_U32: return IR::ValueOpcode::SharedAtomicUMin32; + case Decoder::Opcode::DS_MAX_I32: + case Decoder::Opcode::DS_MAX_RTN_I32: return IR::ValueOpcode::SharedAtomicSMax32; + case Decoder::Opcode::DS_MAX_U32: + case Decoder::Opcode::DS_MAX_RTN_U32: return IR::ValueOpcode::SharedAtomicUMax32; + case Decoder::Opcode::DS_AND_B32: + case Decoder::Opcode::DS_AND_RTN_B32: return IR::ValueOpcode::SharedAtomicAnd32; + case Decoder::Opcode::DS_OR_B32: + case Decoder::Opcode::DS_OR_RTN_B32: return IR::ValueOpcode::SharedAtomicOr32; + case Decoder::Opcode::DS_XOR_B32: + case Decoder::Opcode::DS_XOR_RTN_B32: return IR::ValueOpcode::SharedAtomicXor32; default: EXIT("invalid shared atomic opcode"); } } -bool Translator::TranslateScalarMemory(const IR::Instruction& inst) { - switch (inst.op) { - case IR::Opcode::LoadSrtDword: { - const auto resource = ir.Emit(IR::ValueOpcode::GetSrtResource); - WriteOperand(inst.dst, - ir.Emit(IR::ValueOpcode::ReadConst, {resource, ReadU32(inst.src[0])})); - return true; - } - case IR::Opcode::SBufferLoadDword: - case IR::Opcode::SLoadDword: { - const auto resource = inst.op == IR::Opcode::SLoadDword - ? GetScalarAddressResource(inst.memory.resource) - : GetBufferResource(inst.memory); - const auto offset = ReadU32(inst.src[0]); - std::array loaded {}; - for (uint32_t component = 0; component < inst.memory.data_dwords; component++) { - auto scalar = inst; - scalar.memory.offset += component * sizeof(uint32_t); - scalar.memory.data_dwords = 1u; - scalar.memory.component_index = component; - if (inst.op == IR::Opcode::SBufferLoadDword) { - loaded[component] = ir.Emit(IR::ValueOpcode::ReadConstBuffer, - {resource, offset}, AddMemoryInfo(scalar)); - } else { - loaded[component] = ir.Emit( - IR::ValueOpcode::LoadAddressU32, - {resource, offset, IR::Value(0u), IR::Value(true)}, AddMemoryInfo(scalar)); - } - } - for (uint32_t component = 0; component < inst.memory.data_dwords; component++) { - WriteOperand(ScalarDestinationOperand(inst.dst, component), loaded[component]); - } - return true; - } - default: return false; +bool Translator::TranslateScalarMemory(const Decoder::Instruction& inst, std::string* error) { + if (!IsScalarAddressLoad(inst.opcode) && !IsScalarBufferLoad(inst.opcode)) { + return false; } + const auto memory = MemoryInfoFromDecoded(inst); + uint32_t dst_code = 0; + const auto alignment = memory.data_dwords == 1u ? 1u : memory.data_dwords == 2u ? 2u : 4u; + if (!TryGetEncodedScalarCode(inst.dst, dst_code) || dst_code % alignment != 0u || + dst_code + memory.data_dwords > 108u) { + if (error != nullptr) { + *error = "scalar-memory destination has an invalid aligned SGPR span"; + } + return false; + } + uint32_t base_code = 0; + const bool raw = IsScalarAddressLoad(inst.opcode); + const auto base_alignment = raw ? 2u : 4u; + const auto base_width = raw ? 2u : 4u; + if (!TryGetEncodedScalarCode(inst.src0, base_code) || base_code % base_alignment != 0u || + base_code + base_width > 108u) { + if (error != nullptr) { + *error = "scalar-memory base has an invalid aligned SGPR span"; + } + return false; + } + const auto resource = + raw ? GetScalarAddressResource(memory.resource) : GetBufferResource(memory); + const auto offset = ReadU32(inst.src1); + std::array loaded {}; + for (uint32_t component = 0; component < memory.data_dwords; component++) { + auto scalar = memory; + scalar.offset += component * sizeof(uint32_t); + scalar.data_dwords = 1u; + scalar.component_index = component; + if (!raw) { + loaded[component] = ir.Emit(IR::ValueOpcode::ReadConstBuffer, {resource, offset}, + AddMemoryInfo(scalar, inst.pc)); + } else { + loaded[component] = ir.Emit(IR::ValueOpcode::LoadAddressU32, + {resource, offset, IR::Value(0u), IR::Value(true)}, + AddMemoryInfo(scalar, inst.pc)); + } + } + for (uint32_t component = 0; component < memory.data_dwords; component++) { + WriteOperand(ScalarDestinationOperand(inst.dst, component), loaded[component]); + } + return true; } -bool Translator::TranslateBufferLoad(const IR::Instruction& inst) { +bool Translator::TranslateBufferLoad(const Decoder::Instruction& inst, std::string*) { + const auto memory = MemoryInfoFromDecoded(inst); IR::ValueOpcode opcode; - uint32_t bits; - bool sign; - switch (inst.op) { - case IR::Opcode::BufferLoadUbyte: - case IR::Opcode::BufferLoadSbyte: - opcode = IR::ValueOpcode::LoadBufferU8; - bits = 8u; - sign = inst.op == IR::Opcode::BufferLoadSbyte; - break; - case IR::Opcode::BufferLoadUshort: - case IR::Opcode::BufferLoadSshort: - opcode = IR::ValueOpcode::LoadBufferU16; - bits = 16u; - sign = inst.op == IR::Opcode::BufferLoadSshort; - break; - case IR::Opcode::BufferLoadDword: - switch (inst.memory.data_dwords) { + const auto bits = memory.data_bits; + const auto sign = memory.data_signed; + switch (bits) { + case 8u: opcode = IR::ValueOpcode::LoadBufferU8; break; + case 16u: opcode = IR::ValueOpcode::LoadBufferU16; break; + case 32u: + switch (memory.data_dwords) { case 1u: opcode = IR::ValueOpcode::LoadBufferU32; break; case 2u: opcode = IR::ValueOpcode::LoadBufferU32x2; break; case 3u: opcode = IR::ValueOpcode::LoadBufferU32x3; break; case 4u: opcode = IR::ValueOpcode::LoadBufferU32x4; break; default: return false; } - bits = 32u; - sign = false; break; default: return false; } - const auto resource = GetBufferResource(inst.memory); + const auto resource = GetBufferResource(memory); const auto address = ReadBufferAddress(inst, 0); const auto loaded = ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, ir.GetExec()}, - AddMemoryInfo(inst)); + AddMemoryInfo(memory, inst.pc)); if (bits != 32u) { WriteOperand(inst.dst, WidenSubdword(loaded, bits, sign)); - } else if (inst.memory.data_dwords == 1u) { + } else if (memory.data_dwords == 1u) { WriteOperand(inst.dst, loaded); } else { - for (uint32_t component = 0; component < inst.memory.data_dwords; component++) { + for (uint32_t component = 0; component < memory.data_dwords; component++) { WriteOperand(OffsetOperand(inst.dst, component), ir.CompositeExtract(loaded, component)); } @@ -277,23 +700,25 @@ bool Translator::TranslateBufferLoad(const IR::Instruction& inst) { return true; } -bool Translator::TranslateBufferStore(const IR::Instruction& inst) { - const auto resource = GetBufferResource(inst.memory); +bool Translator::TranslateBufferStore(const Decoder::Instruction& inst, std::string*) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto resource = GetBufferResource(memory); const auto address = ReadBufferAddress(inst, 1); - const auto data = ReadU32(inst.src[0]); + const auto data_src = MemorySourceAt(inst, 0); + const auto data = ReadU32(data_src); IR::ValueOpcode opcode; IR::Value value; - switch (inst.op) { - case IR::Opcode::BufferStoreByte: + switch (memory.data_bits) { + case 8u: opcode = IR::ValueOpcode::StoreBufferU8; value = NarrowSubdword(data, 8u); break; - case IR::Opcode::BufferStoreShort: + case 16u: opcode = IR::ValueOpcode::StoreBufferU16; value = NarrowSubdword(data, 16u); break; - case IR::Opcode::BufferStoreDword: - switch (inst.memory.data_dwords) { + case 32u: + switch (memory.data_dwords) { case 1u: opcode = IR::ValueOpcode::StoreBufferU32; value = data; @@ -301,20 +726,20 @@ bool Translator::TranslateBufferStore(const IR::Instruction& inst) { case 2u: opcode = IR::ValueOpcode::StoreBufferU32x2; value = ir.Emit(IR::ValueOpcode::CompositeConstructU32x2, - {data, ReadU32(OffsetOperand(inst.src[0], 1u))}); + {data, ReadU32(OffsetOperand(data_src, 1u))}); break; case 3u: opcode = IR::ValueOpcode::StoreBufferU32x3; value = ir.Emit(IR::ValueOpcode::CompositeConstructU32x3, - {data, ReadU32(OffsetOperand(inst.src[0], 1u)), - ReadU32(OffsetOperand(inst.src[0], 2u))}); + {data, ReadU32(OffsetOperand(data_src, 1u)), + ReadU32(OffsetOperand(data_src, 2u))}); break; case 4u: opcode = IR::ValueOpcode::StoreBufferU32x4; value = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4, - {data, ReadU32(OffsetOperand(inst.src[0], 1u)), - ReadU32(OffsetOperand(inst.src[0], 2u)), - ReadU32(OffsetOperand(inst.src[0], 3u))}); + {data, ReadU32(OffsetOperand(data_src, 1u)), + ReadU32(OffsetOperand(data_src, 2u)), + ReadU32(OffsetOperand(data_src, 3u))}); break; default: return false; } @@ -322,173 +747,174 @@ bool Translator::TranslateBufferStore(const IR::Instruction& inst) { default: return false; } ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, value, ir.GetExec()}, - AddMemoryInfo(inst)); + AddMemoryInfo(memory, inst.pc)); return true; } -bool Translator::TranslateAtomicMemory(const IR::Instruction& inst) { - IR::Value result; - switch (inst.memory.kind) { +bool Translator::TranslateAtomicMemory(const Decoder::Instruction& inst, std::string*) { + const auto memory = MemoryInfoFromDecoded(inst); + IR::Value result; + switch (memory.kind) { case IR::ResourceKind::Buffer: { - const auto resource = GetBufferResource(inst.memory); + const auto resource = GetBufferResource(memory); const auto address = ReadBufferAddress(inst, 1); - result = ir.Emit(BufferAtomicOpcode(inst.op), + result = ir.Emit(BufferAtomicOpcode(inst.opcode), {resource, address.index, address.offset, address.soffset, - ReadU32(inst.src[0]), ir.GetExec()}, - AddMemoryInfo(inst)); + ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); break; } case IR::ResourceKind::Image: case IR::ResourceKind::ImageUint: case IR::ResourceKind::StorageImage: case IR::ResourceKind::StorageImageUint: { - const auto resource = GetImageResource(inst.memory); - const auto address = MakeImageAddress(inst, inst.src[1]); - const auto flags = AddMemoryInfo(inst); - result = ir.Emit(ImageAtomicOpcode(inst.op), - {resource, address, ReadU32(inst.src[0]), ir.GetExec()}, flags); + const auto resource = GetImageResource(memory); + const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 1)); + const auto flags = AddMemoryInfo(memory, inst.pc); + result = + ir.Emit(ImageAtomicOpcode(inst.opcode), + {resource, address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, flags); break; } case IR::ResourceKind::Lds: case IR::ResourceKind::Gds: { - const auto address = ReadU32(inst.src[1]); - result = ir.Emit(SharedAtomicOpcode(inst.op), - {address, ReadU32(inst.src[0]), ir.GetExec()}, AddMemoryInfo(inst)); + const auto address = ReadU32(MemorySourceAt(inst, 1)); + result = ir.Emit(SharedAtomicOpcode(inst.opcode), + {address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); break; } default: return false; } - if (inst.dst.kind != IR::OperandKind::Null) { + if (inst.glc || IsDsReturnAtomic(inst.opcode)) { WriteOperand(inst.dst, result); } return true; } -bool Translator::TranslateFlatLoad(const IR::Instruction& inst) { +bool Translator::TranslateFlatLoad(const Decoder::Instruction& inst, std::string*) { + const auto memory = MemoryInfoFromDecoded(inst); IR::ValueOpcode opcode; - uint32_t bits; - bool sign; - switch (inst.op) { - case IR::Opcode::FlatLoadUbyte: - case IR::Opcode::FlatLoadSbyte: - opcode = IR::ValueOpcode::LoadAddressU8; - bits = 8u; - sign = inst.op == IR::Opcode::FlatLoadSbyte; - break; - case IR::Opcode::FlatLoadUshort: - case IR::Opcode::FlatLoadSshort: - opcode = IR::ValueOpcode::LoadAddressU16; - bits = 16u; - sign = inst.op == IR::Opcode::FlatLoadSshort; - break; - case IR::Opcode::FlatLoadDword: - opcode = IR::ValueOpcode::LoadAddressU32; - bits = 32u; - sign = false; - break; + const auto bits = memory.data_bits; + const auto sign = memory.data_signed; + switch (bits) { + case 8u: opcode = IR::ValueOpcode::LoadAddressU8; break; + case 16u: opcode = IR::ValueOpcode::LoadAddressU16; break; + case 32u: opcode = IR::ValueOpcode::LoadAddressU32; break; default: return false; } const auto address = ReadAddressOperands(inst, 0); const auto active = ir.GetExec(); - const auto count = bits == 32u ? std::min(inst.memory.data_dwords, 4u) : 1u; + const auto count = bits == 32u ? std::min(memory.data_dwords, 4u) : 1u; for (uint32_t index = 0; index < count; index++) { - auto component = inst; - component.memory.offset += index * 4u; - component.memory.data_dwords = 1u; - component.memory.component_index = index; + auto component = memory; + component.offset += index * 4u; + component.data_dwords = 1u; + component.component_index = index; const auto loaded = ir.Emit(opcode, {address.resource, address.low, address.high, active}, - AddMemoryInfo(component)); + AddMemoryInfo(component, inst.pc)); WriteOperand(OffsetOperand(inst.dst, index), bits == 32u ? loaded : WidenSubdword(loaded, bits, sign)); } return true; } -bool Translator::TranslateFlatStore(const IR::Instruction& inst) { - const auto data = ReadU32(inst.src[0]); +bool Translator::TranslateFlatStore(const Decoder::Instruction& inst, std::string*) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto data_op = MemorySourceAt(inst, 0); const auto address = ReadAddressOperands(inst, 1); IR::ValueOpcode opcode; - IR::Value value; - switch (inst.op) { - case IR::Opcode::FlatStoreByte: - opcode = IR::ValueOpcode::StoreAddressU8; - value = NarrowSubdword(data, 8u); - break; - case IR::Opcode::FlatStoreShort: - opcode = IR::ValueOpcode::StoreAddressU16; - value = NarrowSubdword(data, 16u); - break; - case IR::Opcode::FlatStoreDword: - opcode = IR::ValueOpcode::StoreAddressU32; - value = data; - break; + switch (memory.data_bits) { + case 8u: opcode = IR::ValueOpcode::StoreAddressU8; break; + case 16u: opcode = IR::ValueOpcode::StoreAddressU16; break; + case 32u: opcode = IR::ValueOpcode::StoreAddressU32; break; default: return false; } - ir.Emit(opcode, {address.resource, address.low, address.high, value, ir.GetExec()}, - AddMemoryInfo(inst)); + const auto count = memory.data_bits == 32u ? memory.data_dwords : 1u; + for (uint32_t index = 0; index < count; index++) { + auto component = memory; + component.offset += index * 4u; + component.data_dwords = 1u; + component.component_index = index; + auto value = IR::Value(ReadU32(OffsetOperand(data_op, index))); + if (memory.data_bits != 32u) { + value = NarrowSubdword(IR::U32(value), memory.data_bits); + } + ir.Emit(opcode, {address.resource, address.low, address.high, value, ir.GetExec()}, + AddMemoryInfo(component, inst.pc)); + } return true; } -bool Translator::TranslateImageMemory(const IR::Instruction& inst) { - const bool image = inst.memory.kind == IR::ResourceKind::Image || - inst.memory.kind == IR::ResourceKind::ImageUint || - inst.memory.kind == IR::ResourceKind::StorageImage || - inst.memory.kind == IR::ResourceKind::StorageImageUint; +bool Translator::TranslateImageMemory(const Decoder::Instruction& inst, std::string*) { + const auto memory = MemoryInfoFromDecoded(inst); + const bool image = memory.kind == IR::ResourceKind::Image || + memory.kind == IR::ResourceKind::ImageUint || + memory.kind == IR::ResourceKind::StorageImage || + memory.kind == IR::ResourceKind::StorageImageUint; if (!image) { return false; } - const auto resource = GetImageResource(inst.memory); - const auto address = - MakeImageAddress(inst, inst.op == IR::Opcode::ImageStore ? inst.src[1] : inst.src[0]); - const auto flags = AddMemoryInfo(inst); - switch (inst.op) { - case IR::Opcode::ImageGetResinfo: { + const auto resource = GetImageResource(memory); + const bool store = inst.opcode == Decoder::Opcode::IMAGE_STORE || + inst.opcode == Decoder::Opcode::IMAGE_STORE_MIP; + const auto address = MakeImageAddress(inst, MemorySourceAt(inst, store ? 1u : 0u)); + const auto flags = AddMemoryInfo(memory, inst.pc); + switch (inst.opcode) { + case Decoder::Opcode::IMAGE_GET_RESINFO: { const auto result = ir.Emit(IR::ValueOpcode::ImageQueryDimensions, {resource, address}, flags); - WriteImageComponents(inst.dst, result, inst.memory, 4u); + WriteImageComponents(inst.dst, result, memory, 4u); return true; } - case IR::Opcode::ImageGetLod: { - const auto sampler = GetSamplerResource(inst.memory); + case Decoder::Opcode::IMAGE_GET_LOD: { + const auto sampler = GetSamplerResource(memory); const auto result = ir.Emit(IR::ValueOpcode::ImageQueryLod, {resource, sampler, address}, flags); - WriteImageComponents(inst.dst, result, inst.memory, 2u); + WriteImageComponents(inst.dst, result, memory, 2u); return true; } - case IR::Opcode::ImageLoad: { + case Decoder::Opcode::IMAGE_LOAD: + case Decoder::Opcode::IMAGE_LOAD_MIP: { const auto result = ir.Emit(IR::ValueOpcode::ImageRead, {resource, address, ir.GetExec()}, flags); - WriteImageComponents(inst.dst, result, inst.memory, 4u); + WriteImageComponents(inst.dst, result, memory, 4u); return true; } - case IR::Opcode::ImageStore: { - const auto data = ConstructU32x4(inst.src[0], inst.memory.data_dwords); + case Decoder::Opcode::IMAGE_STORE: + case Decoder::Opcode::IMAGE_STORE_MIP: { + const auto data = ConstructU32x4(MemorySourceAt(inst, 0), memory.data_dwords); ir.Emit(IR::ValueOpcode::ImageWrite, {resource, address, data, ir.GetExec()}, flags); return true; } - case IR::Opcode::ImageSample: - case IR::Opcode::ImageGather4: { - const auto sampler = GetSamplerResource(inst.memory); - const auto opcode = inst.op == IR::Opcode::ImageSample - ? IR::ValueOpcode::ImageSampleRaw - : IR::ValueOpcode::ImageGatherRaw; - const auto result = ir.Emit(opcode, {resource, sampler, address}, flags); - const bool dref = - (inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u; - if (inst.op == IR::Opcode::ImageSample && dref && inst.memory.data_bits != 16u) { + case Decoder::Opcode::IMAGE_SAMPLE: + case Decoder::Opcode::IMAGE_GATHER4_LZ: + case Decoder::Opcode::IMAGE_GATHER4_C: + case Decoder::Opcode::IMAGE_GATHER4_C_LZ: + case Decoder::Opcode::IMAGE_GATHER4_LZ_O: + case Decoder::Opcode::IMAGE_GATHER4_C_O: + case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O: + case Decoder::Opcode::IMAGE_GATHER4H: { + const auto sampler = GetSamplerResource(memory); + const bool sample = inst.opcode == Decoder::Opcode::IMAGE_SAMPLE; + const auto opcode = + sample ? IR::ValueOpcode::ImageSampleRaw : IR::ValueOpcode::ImageGatherRaw; + const auto result = ir.Emit(opcode, {resource, sampler, address}, flags); + const bool dref = (memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u; + if (sample && dref && memory.data_bits != 16u) { const auto component = ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(0u)}); - for (uint32_t index = 0; index < inst.memory.data_dwords; index++) { + for (uint32_t index = 0; index < memory.data_dwords; index++) { WriteOperand(OffsetOperand(inst.dst, index), component); } - } else if (inst.op == IR::Opcode::ImageGather4) { - for (uint32_t index = 0; index < inst.memory.data_dwords; index++) { + } else if (!sample) { + for (uint32_t index = 0; index < memory.data_dwords; index++) { WriteOperand(OffsetOperand(inst.dst, index), ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(index)})); } } else { - WriteImageComponents(inst.dst, result, inst.memory, 4u); + WriteImageComponents(inst.dst, result, memory, 4u); } return true; } @@ -496,14 +922,17 @@ bool Translator::TranslateImageMemory(const IR::Instruction& inst) { } } -bool Translator::TranslateSharedMemory(const IR::Instruction& inst) { +bool Translator::TranslateSharedMemory(const Decoder::Instruction& inst, std::string*) { + const auto memory = MemoryInfoFromDecoded(inst); const bool shared = - inst.memory.kind == IR::ResourceKind::Lds || inst.memory.kind == IR::ResourceKind::Gds; - if (!shared && inst.op != IR::Opcode::DsSwizzleB32) { + memory.kind == IR::ResourceKind::Lds || memory.kind == IR::ResourceKind::Gds; + if (!shared && inst.opcode != Decoder::Opcode::DS_SWIZZLE_B32) { return false; } - const auto shared_address = [&](uint32_t source) { return ReadU32(inst.src[source]); }; - const auto load_u32 = [&](uint32_t width, IR::U32 address, const IR::Instruction& source) { + const auto shared_address = [&](uint32_t source) { + return ReadU32(MemorySourceAt(inst, source)); + }; + const auto load_u32 = [&](uint32_t width, IR::U32 address, const IR::MemoryInfo& source) { IR::ValueOpcode opcode; switch (width) { case 1u: opcode = IR::ValueOpcode::LoadSharedU32; break; @@ -512,7 +941,7 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) { case 4u: opcode = IR::ValueOpcode::LoadSharedU32x4; break; default: EXIT("invalid shared load width"); } - return ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(source)); + return ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(source, inst.pc)); }; const auto extract_u32 = [&](IR::Value value, uint32_t width, uint32_t index) { if (width == 1u) { @@ -525,71 +954,79 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) { }; const auto write_u32 = [&](uint32_t width, IR::U32 address, const std::array& values, - const IR::Instruction& source) { + const IR::MemoryInfo& source) { switch (width) { case 1u: ir.Emit(IR::ValueOpcode::WriteSharedU32, {address, values[0], ir.GetExec()}, - AddMemoryInfo(source)); + AddMemoryInfo(source, inst.pc)); break; case 2u: ir.Emit(IR::ValueOpcode::WriteSharedU32x2, - {address, values[0], values[1], ir.GetExec()}, AddMemoryInfo(source)); + {address, values[0], values[1], ir.GetExec()}, + AddMemoryInfo(source, inst.pc)); break; case 3u: ir.Emit(IR::ValueOpcode::WriteSharedU32x3, {address, values[0], values[1], values[2], ir.GetExec()}, - AddMemoryInfo(source)); + AddMemoryInfo(source, inst.pc)); break; case 4u: ir.Emit(IR::ValueOpcode::WriteSharedU32x4, {address, values[0], values[1], values[2], values[3], ir.GetExec()}, - AddMemoryInfo(source)); + AddMemoryInfo(source, inst.pc)); break; default: EXIT("invalid shared store width"); } }; - switch (inst.op) { - case IR::Opcode::DsReadUbyte: - case IR::Opcode::DsReadSbyte: - case IR::Opcode::DsReadUshort: - case IR::Opcode::DsReadSshort: - case IR::Opcode::DsReadB32: { + switch (inst.opcode) { + case Decoder::Opcode::DS_READ_I8: + case Decoder::Opcode::DS_READ_U8: + case Decoder::Opcode::DS_READ_I16: + case Decoder::Opcode::DS_READ_U16: + case Decoder::Opcode::DS_READ_U16_D16: + case Decoder::Opcode::DS_READ_B32: + case Decoder::Opcode::DS_READ_B64: + case Decoder::Opcode::DS_READ_B96: + case Decoder::Opcode::DS_READ_B128: { IR::ValueOpcode opcode; uint32_t bits; bool sign; - if (inst.op == IR::Opcode::DsReadUbyte || inst.op == IR::Opcode::DsReadSbyte) { + if (memory.data_bits == 8u) { opcode = IR::ValueOpcode::LoadSharedU8; bits = 8u; - sign = inst.op == IR::Opcode::DsReadSbyte; - } else if (inst.op == IR::Opcode::DsReadUshort || inst.op == IR::Opcode::DsReadSshort) { + sign = memory.data_signed; + } else if (memory.data_bits == 16u) { opcode = IR::ValueOpcode::LoadSharedU16; bits = 16u; - sign = inst.op == IR::Opcode::DsReadSshort; + sign = memory.data_signed; } else { - const auto width = inst.memory.data_dwords; - const auto loaded = load_u32(width, shared_address(0), inst); + const auto width = memory.data_dwords; + const auto loaded = load_u32(width, shared_address(0), memory); for (uint32_t index = 0; index < width; index++) { WriteOperand(OffsetOperand(inst.dst, index), extract_u32(loaded, width, index)); } return true; } const auto loaded = - ir.Emit(opcode, {shared_address(0), ir.GetExec()}, AddMemoryInfo(inst)); + ir.Emit(opcode, {shared_address(0), ir.GetExec()}, AddMemoryInfo(memory, inst.pc)); WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign)); return true; } - case IR::Opcode::DsRead2B32: { - const auto width = inst.memory.data_dwords / 2u; - const auto address = shared_address(0); - auto first = inst; - first.memory.data_dwords = width; - first.memory.component_count = width; - const auto first_value = load_u32(width, address, first); - IR::Value second_value = first_value; - if (inst.memory.secondary_offset != inst.memory.offset) { - auto second = first; - second.memory.offset = inst.memory.secondary_offset; - second_value = load_u32(width, address, second); + case Decoder::Opcode::DS_READ2_B32: + case Decoder::Opcode::DS_READ2ST64_B32: + case Decoder::Opcode::DS_READ2_B64: + case Decoder::Opcode::DS_READ2ST64_B64: { + const auto width = memory.data_dwords / 2u; + const auto address = shared_address(0); + auto first = memory; + first.data_dwords = width; + first.component_count = width; + const auto first_value = load_u32(width, address, first); + IR::Value second_value = first_value; + if (memory.secondary_offset != memory.offset) { + auto second = first; + second.offset = memory.secondary_offset; + second_value = load_u32(width, address, second); } for (uint32_t index = 0; index < width; index++) { WriteOperand(OffsetOperand(inst.dst, index), @@ -599,162 +1036,287 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) { } return true; } - case IR::Opcode::DsWriteByte: - case IR::Opcode::DsWriteShort: - case IR::Opcode::DsWriteB32: { - const auto width = inst.memory.data_dwords; + case Decoder::Opcode::DS_WRITE_B8: + case Decoder::Opcode::DS_WRITE_B16: + case Decoder::Opcode::DS_WRITE_B32: + case Decoder::Opcode::DS_WRITE_B64: + case Decoder::Opcode::DS_WRITE_B96: + case Decoder::Opcode::DS_WRITE_B128: { + const auto width = memory.data_dwords; + const auto data_operand = MemorySourceAt(inst, 0); std::array values {}; for (uint32_t index = 0; index < width; index++) { - values[index] = ReadU32(OffsetOperand(inst.src[0], index)); + values[index] = ReadU32(OffsetOperand(data_operand, index)); } const auto data = IR::U32(values[0]); const auto address = shared_address(1); IR::ValueOpcode opcode; IR::Value value; - if (inst.op == IR::Opcode::DsWriteByte) { + if (memory.data_bits == 8u) { opcode = IR::ValueOpcode::WriteSharedU8; value = NarrowSubdword(data, 8u); - } else if (inst.op == IR::Opcode::DsWriteShort) { + } else if (memory.data_bits == 16u) { opcode = IR::ValueOpcode::WriteSharedU16; value = NarrowSubdword(data, 16u); } else { - write_u32(width, address, values, inst); + write_u32(width, address, values, memory); return true; } - ir.Emit(opcode, {address, value, ir.GetExec()}, AddMemoryInfo(inst)); + ir.Emit(opcode, {address, value, ir.GetExec()}, AddMemoryInfo(memory, inst.pc)); return true; } - case IR::Opcode::DsWrite2B32: { - const auto width = inst.memory.data_dwords / 2u; - const auto address = shared_address(1); + case Decoder::Opcode::DS_WRITE2_B32: + case Decoder::Opcode::DS_WRITE2ST64_B32: + case Decoder::Opcode::DS_WRITE2_B64: + case Decoder::Opcode::DS_WRITE2ST64_B64: { + const auto width = memory.data_dwords / 2u; + const auto address = shared_address(1); + const auto first_data = MemorySourceAt(inst, 0); + const auto second_data = MemorySourceAt(inst, 2); std::array first_values {}; std::array second_values {}; for (uint32_t index = 0; index < width; index++) { - first_values[index] = ReadU32(OffsetOperand(inst.src[0], index)); - second_values[index] = ReadU32(OffsetOperand(inst.src[2], index)); + first_values[index] = ReadU32(OffsetOperand(first_data, index)); + second_values[index] = ReadU32(OffsetOperand(second_data, index)); } - auto first = inst; - first.memory.data_dwords = width; - first.memory.component_count = width; + auto first = memory; + first.data_dwords = width; + first.component_count = width; write_u32(width, address, first_values, first); - if (inst.memory.secondary_offset != inst.memory.offset) { - auto second = first; - second.memory.offset = inst.memory.secondary_offset; + if (memory.secondary_offset != memory.offset) { + auto second = first; + second.offset = memory.secondary_offset; write_u32(width, address, second_values, second); } return true; } - case IR::Opcode::DsMinF32: - case IR::Opcode::DsMaxF32: { - const auto opcode = inst.op == IR::Opcode::DsMinF32 + case Decoder::Opcode::DS_MIN_F32: + case Decoder::Opcode::DS_MAX_F32: { + const auto opcode = inst.opcode == Decoder::Opcode::DS_MIN_F32 ? IR::ValueOpcode::SharedAtomicFMin32 : IR::ValueOpcode::SharedAtomicFMax32; ir.Emit(opcode, - {shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]), ir.GetExec()}, - AddMemoryInfo(inst)); + {shared_address(1), ReadU32(MemorySourceAt(inst, 0)), + ReadU32(MemorySourceAt(inst, 2)), ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); return true; } - case IR::Opcode::DsAppend: - case IR::Opcode::DsConsume: { - const bool append = inst.op == IR::Opcode::DsAppend; + case Decoder::Opcode::DS_APPEND: + case Decoder::Opcode::DS_CONSUME: { + const bool append = inst.opcode == Decoder::Opcode::DS_APPEND; const auto opcode = append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume; WriteOperand(inst.dst, ir.Emit(opcode, - {ReadU32(inst.src[0]), ir.GetExec(), ir.GetExecLo(), - ir.GetExecHi()}, - AddMemoryInfo(inst))); + {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec(), + ir.GetExecLo(), ir.GetExecHi()}, + AddMemoryInfo(memory, inst.pc))); return true; } - case IR::Opcode::DsWriteAddtidB32: - case IR::Opcode::DsReadAddtidB32: { - const auto m0_index = inst.op == IR::Opcode::DsWriteAddtidB32 ? 1u : 0u; + case Decoder::Opcode::DS_WRITE_ADDTID_B32: + case Decoder::Opcode::DS_READ_ADDTID_B32: { + const auto m0_index = inst.opcode == Decoder::Opcode::DS_WRITE_ADDTID_B32 ? 1u : 0u; const auto base = - ir.BitwiseAnd(ReadU32(inst.src[m0_index]), IR::U32(IR::Value(0xffffu))); + ir.BitwiseAnd(ReadU32(MemorySourceAt(inst, m0_index)), IR::U32(IR::Value(0xffffu))); const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); const auto address = ir.IAdd(base, ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u)))); - if (inst.op == IR::Opcode::DsWriteAddtidB32) { + if (inst.opcode == Decoder::Opcode::DS_WRITE_ADDTID_B32) { ir.Emit(IR::ValueOpcode::WriteSharedU32, - {address, ReadU32(inst.src[0]), ir.GetExec()}, AddMemoryInfo(inst)); + {address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); } else { - WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::LoadSharedU32, - {address, ir.GetExec()}, AddMemoryInfo(inst))); + WriteOperand(inst.dst, + ir.Emit(IR::ValueOpcode::LoadSharedU32, {address, ir.GetExec()}, + AddMemoryInfo(memory, inst.pc))); } return true; } - case IR::Opcode::DsSwizzleB32: - WriteOperand(inst.dst, - ir.Emit(IR::ValueOpcode::SwizzleU32, - {ReadU32(inst.src[0]), ReadU32(inst.src[1]), ir.GetExec()})); + case Decoder::Opcode::DS_SWIZZLE_B32: + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::SwizzleU32, + {ReadU32(MemorySourceAt(inst, 0)), + ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()})); return true; default: return false; } } -bool Translator::TranslateMemoryOperation(const IR::Instruction& inst) { - switch (inst.op) { - case IR::Opcode::LoadSrtDword: - case IR::Opcode::SLoadDword: - case IR::Opcode::SBufferLoadDword: - return TranslateScalarMemory(inst); +bool Translator::TranslateMemoryOperation(const Decoder::Instruction& inst, std::string* error) { + if (!MemoryOpcodeMatchesFamily(inst.opcode, inst.family)) { + if (error != nullptr) { + *error = fmt::format("memory opcode {} does not belong to decoded family {}", + magic_enum::enum_name(inst.opcode), magic_enum::enum_name(inst.family)); + } + return false; + } + if (RequiresRegisterDestination(inst) && !IsRegisterOperand(inst.dst)) { + if (error != nullptr) { + *error = "decoded operand cannot be used as an IR register"; + } + return false; + } + if (UsesDestinationAsMemorySource(inst.opcode) && + inst.dst.kind == Decoder::OperandKind::Unknown) { + if (error != nullptr) { + *error = "decoded operand cannot be used as an IR register"; + } + return false; + } + for (uint32_t index = 0; index < inst.src_count; index++) { + if (DecodedSourceAt(inst, index).kind == Decoder::OperandKind::Unknown) { + if (error != nullptr) { + *error = "decoded operand cannot be used as an IR register"; + } + return false; + } + } + switch (inst.opcode) { + case Decoder::Opcode::S_LOAD_DWORD: + case Decoder::Opcode::S_LOAD_DWORDX2: + case Decoder::Opcode::S_LOAD_DWORDX4: + case Decoder::Opcode::S_LOAD_DWORDX8: + case Decoder::Opcode::S_LOAD_DWORDX16: + case Decoder::Opcode::S_BUFFER_LOAD_DWORD: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX2: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16: return TranslateScalarMemory(inst, error); - case IR::Opcode::BufferLoadUbyte: - case IR::Opcode::BufferLoadSbyte: - case IR::Opcode::BufferLoadUshort: - case IR::Opcode::BufferLoadSshort: - case IR::Opcode::BufferLoadDword: return TranslateBufferLoad(inst); + case Decoder::Opcode::BUFFER_LOAD_UBYTE: + case Decoder::Opcode::BUFFER_LOAD_SBYTE: + case Decoder::Opcode::BUFFER_LOAD_USHORT: + case Decoder::Opcode::BUFFER_LOAD_SSHORT: + case Decoder::Opcode::BUFFER_LOAD_DWORD: + case Decoder::Opcode::BUFFER_LOAD_DWORDX2: + case Decoder::Opcode::BUFFER_LOAD_DWORDX3: + case Decoder::Opcode::BUFFER_LOAD_DWORDX4: + case Decoder::Opcode::BUFFER_LOAD_FORMAT_X: + case Decoder::Opcode::BUFFER_LOAD_FORMAT_XY: + case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ: + case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW: + case Decoder::Opcode::TBUFFER_LOAD_FORMAT_X: + case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XY: + case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZ: + case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZW: return TranslateBufferLoad(inst, error); - case IR::Opcode::BufferStoreByte: - case IR::Opcode::BufferStoreShort: - case IR::Opcode::BufferStoreDword: return TranslateBufferStore(inst); + case Decoder::Opcode::BUFFER_STORE_DWORD: + case Decoder::Opcode::BUFFER_STORE_DWORDX2: + case Decoder::Opcode::BUFFER_STORE_DWORDX3: + case Decoder::Opcode::BUFFER_STORE_DWORDX4: + case Decoder::Opcode::BUFFER_STORE_BYTE: + case Decoder::Opcode::BUFFER_STORE_SHORT: + case Decoder::Opcode::BUFFER_STORE_FORMAT_X: + case Decoder::Opcode::BUFFER_STORE_FORMAT_XY: + case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZ: + case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW: + case Decoder::Opcode::TBUFFER_STORE_FORMAT_X: + case Decoder::Opcode::TBUFFER_STORE_FORMAT_XY: + case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZ: + case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW: return TranslateBufferStore(inst, error); - case IR::Opcode::AtomicSwapU32: - case IR::Opcode::AtomicAddU32: - case IR::Opcode::AtomicSubU32: - case IR::Opcode::AtomicSMinI32: - case IR::Opcode::AtomicUMinU32: - case IR::Opcode::AtomicSMaxI32: - case IR::Opcode::AtomicUMaxU32: - case IR::Opcode::AtomicAndU32: - case IR::Opcode::AtomicOrU32: - case IR::Opcode::AtomicXorU32: - case IR::Opcode::AtomicFMinF32: - case IR::Opcode::AtomicFMaxF32: return TranslateAtomicMemory(inst); + case Decoder::Opcode::BUFFER_ATOMIC_SWAP: + case Decoder::Opcode::BUFFER_ATOMIC_ADD: + case Decoder::Opcode::BUFFER_ATOMIC_SUB: + case Decoder::Opcode::BUFFER_ATOMIC_SMIN: + case Decoder::Opcode::BUFFER_ATOMIC_UMIN: + case Decoder::Opcode::BUFFER_ATOMIC_SMAX: + case Decoder::Opcode::BUFFER_ATOMIC_UMAX: + case Decoder::Opcode::BUFFER_ATOMIC_AND: + case Decoder::Opcode::BUFFER_ATOMIC_OR: + case Decoder::Opcode::BUFFER_ATOMIC_XOR: + case Decoder::Opcode::BUFFER_ATOMIC_FMIN: + case Decoder::Opcode::BUFFER_ATOMIC_FMAX: + case Decoder::Opcode::DS_ADD_U32: + case Decoder::Opcode::DS_ADD_RTN_U32: + case Decoder::Opcode::DS_SUB_U32: + case Decoder::Opcode::DS_SUB_RTN_U32: + case Decoder::Opcode::DS_MIN_I32: + case Decoder::Opcode::DS_MIN_RTN_I32: + case Decoder::Opcode::DS_MAX_I32: + case Decoder::Opcode::DS_MAX_RTN_I32: + case Decoder::Opcode::DS_MIN_U32: + case Decoder::Opcode::DS_MIN_RTN_U32: + case Decoder::Opcode::DS_MAX_U32: + case Decoder::Opcode::DS_MAX_RTN_U32: + case Decoder::Opcode::DS_AND_B32: + case Decoder::Opcode::DS_AND_RTN_B32: + case Decoder::Opcode::DS_OR_B32: + case Decoder::Opcode::DS_OR_RTN_B32: + case Decoder::Opcode::DS_XOR_B32: + case Decoder::Opcode::DS_XOR_RTN_B32: + case Decoder::Opcode::DS_WRXCHG_RTN_B32: + case Decoder::Opcode::IMAGE_ATOMIC_ADD: + case Decoder::Opcode::IMAGE_ATOMIC_UMIN: + case Decoder::Opcode::IMAGE_ATOMIC_UMAX: + case Decoder::Opcode::IMAGE_ATOMIC_AND: + case Decoder::Opcode::IMAGE_ATOMIC_OR: + case Decoder::Opcode::IMAGE_ATOMIC_XOR: return TranslateAtomicMemory(inst, error); - case IR::Opcode::FlatLoadUbyte: - case IR::Opcode::FlatLoadSbyte: - case IR::Opcode::FlatLoadUshort: - case IR::Opcode::FlatLoadSshort: - case IR::Opcode::FlatLoadDword: return TranslateFlatLoad(inst); + case Decoder::Opcode::FLAT_LOAD_UBYTE: + case Decoder::Opcode::FLAT_LOAD_SBYTE: + case Decoder::Opcode::FLAT_LOAD_USHORT: + case Decoder::Opcode::FLAT_LOAD_SSHORT: + case Decoder::Opcode::FLAT_LOAD_DWORD: + case Decoder::Opcode::FLAT_LOAD_DWORDX2: + case Decoder::Opcode::FLAT_LOAD_DWORDX3: + case Decoder::Opcode::FLAT_LOAD_DWORDX4: return TranslateFlatLoad(inst, error); - case IR::Opcode::FlatStoreByte: - case IR::Opcode::FlatStoreShort: - case IR::Opcode::FlatStoreDword: return TranslateFlatStore(inst); + case Decoder::Opcode::FLAT_STORE_BYTE: + case Decoder::Opcode::FLAT_STORE_SHORT: + case Decoder::Opcode::FLAT_STORE_DWORD: + case Decoder::Opcode::FLAT_STORE_DWORDX2: + case Decoder::Opcode::FLAT_STORE_DWORDX3: + case Decoder::Opcode::FLAT_STORE_DWORDX4: return TranslateFlatStore(inst, error); - case IR::Opcode::ImageGetResinfo: - case IR::Opcode::ImageGetLod: - case IR::Opcode::ImageLoad: - case IR::Opcode::ImageStore: - case IR::Opcode::ImageSample: - case IR::Opcode::ImageGather4: return TranslateImageMemory(inst); + case Decoder::Opcode::IMAGE_GET_RESINFO: + case Decoder::Opcode::IMAGE_GET_LOD: + case Decoder::Opcode::IMAGE_LOAD: + case Decoder::Opcode::IMAGE_LOAD_MIP: + case Decoder::Opcode::IMAGE_STORE: + case Decoder::Opcode::IMAGE_STORE_MIP: + case Decoder::Opcode::IMAGE_GATHER4_LZ: + case Decoder::Opcode::IMAGE_GATHER4_C: + case Decoder::Opcode::IMAGE_GATHER4_C_LZ: + case Decoder::Opcode::IMAGE_GATHER4_LZ_O: + case Decoder::Opcode::IMAGE_GATHER4_C_O: + case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O: + case Decoder::Opcode::IMAGE_GATHER4H: + case Decoder::Opcode::IMAGE_SAMPLE: return TranslateImageMemory(inst, error); - case IR::Opcode::DsReadUbyte: - case IR::Opcode::DsReadSbyte: - case IR::Opcode::DsReadUshort: - case IR::Opcode::DsReadSshort: - case IR::Opcode::DsReadB32: - case IR::Opcode::DsRead2B32: - case IR::Opcode::DsWriteByte: - case IR::Opcode::DsWriteShort: - case IR::Opcode::DsWriteB32: - case IR::Opcode::DsWrite2B32: - case IR::Opcode::DsMinF32: - case IR::Opcode::DsMaxF32: - case IR::Opcode::DsSwizzleB32: - case IR::Opcode::DsConsume: - case IR::Opcode::DsAppend: - case IR::Opcode::DsWriteAddtidB32: - case IR::Opcode::DsReadAddtidB32: return TranslateSharedMemory(inst); - default: return false; + case Decoder::Opcode::DS_MIN_F32: + case Decoder::Opcode::DS_MAX_F32: + case Decoder::Opcode::DS_SWIZZLE_B32: + case Decoder::Opcode::DS_CONSUME: + case Decoder::Opcode::DS_APPEND: + case Decoder::Opcode::DS_WRITE_ADDTID_B32: + case Decoder::Opcode::DS_READ_ADDTID_B32: + case Decoder::Opcode::DS_READ2_B32: + case Decoder::Opcode::DS_READ2ST64_B32: + case Decoder::Opcode::DS_READ2_B64: + case Decoder::Opcode::DS_READ2ST64_B64: + case Decoder::Opcode::DS_READ_I8: + case Decoder::Opcode::DS_READ_U8: + case Decoder::Opcode::DS_READ_I16: + case Decoder::Opcode::DS_READ_U16: + case Decoder::Opcode::DS_READ_U16_D16: + case Decoder::Opcode::DS_READ_B32: + case Decoder::Opcode::DS_READ_B64: + case Decoder::Opcode::DS_READ_B96: + case Decoder::Opcode::DS_READ_B128: + case Decoder::Opcode::DS_WRITE2_B32: + case Decoder::Opcode::DS_WRITE2ST64_B32: + case Decoder::Opcode::DS_WRITE2_B64: + case Decoder::Opcode::DS_WRITE2ST64_B64: + case Decoder::Opcode::DS_WRITE_B8: + case Decoder::Opcode::DS_WRITE_B16: + case Decoder::Opcode::DS_WRITE_B32: + case Decoder::Opcode::DS_WRITE_B64: + case Decoder::Opcode::DS_WRITE_B96: + case Decoder::Opcode::DS_WRITE_B128: return TranslateSharedMemory(inst, error); + default: + if (error != nullptr) { + *error = fmt::format("memory-family opcode has no specialized IR translation: {}", + magic_enum::enum_name(inst.opcode)); + } + return false; } } -} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index a7eb15c9c..5f3fbddec 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -4,63 +4,92 @@ #include #include #include +#include #include namespace Libs::Graphics::ShaderRecompiler::Frontend { using ShaderError::Fail; -namespace Detail { +Decoder::Operand Translator::SourceAt(const Decoder::Instruction& inst, uint32_t index) { + switch (index) { + case 0: return inst.src0; + case 1: return inst.src1; + case 2: return inst.src2; + case 3: return inst.src3; + default: EXIT("decoded source operand index is out of range"); + } +} -IR::Operand Translator::OffsetOperand(const IR::Operand& operand, uint32_t offset) { - if (offset == 0 || operand.kind != IR::OperandKind::Register) { +Decoder::Operand Translator::DestinationOperand(const Decoder::Instruction& inst) { + auto destination = inst.dst; + if (destination.kind != Decoder::OperandKind::Vgpr) { + return destination; + } + for (uint32_t index = 0; index < std::min(inst.src_count, 3u); index++) { + const auto source = SourceAt(inst, index); + if (!source.dpp) { + continue; + } + destination.dpp = true; + destination.dpp_ctrl = source.dpp_ctrl; + destination.dpp_row_mask = source.dpp_row_mask; + destination.dpp_bank_mask = source.dpp_bank_mask; + destination.dpp_fetch_inactive = source.dpp_fetch_inactive; + destination.dpp_bound_ctrl = source.dpp_bound_ctrl; + break; + } + return destination; +} + +Decoder::Operand Translator::OffsetOperand(const Decoder::Operand& operand, uint32_t offset) { + if (offset == 0) { return operand; } - auto result = operand; - result.reg.index += offset; - result.sdwa_sel = 6; - result.sdwa_dst_unused = 2; - result.omod = 0; - result.sdwa_sext = false; - result.op_sel = false; - result.op_sel_hi = false; - result.negate = false; - result.negate_hi = false; - result.absolute = false; - result.clamp = false; - result.dpp_ctrl = 0; - result.dpp_row_mask = 0xf; - result.dpp_bank_mask = 0xf; - result.dpp_fetch_inactive = false; - result.dpp_bound_ctrl = false; - result.dpp = false; + auto result = PlainOperand(operand); + switch (result.kind) { + case Decoder::OperandKind::Sgpr: + case Decoder::OperandKind::Vgpr: result.reg += offset; break; + case Decoder::OperandKind::VccLo: + EXIT_IF(offset != 1u); + result.kind = Decoder::OperandKind::VccHi; + break; + case Decoder::OperandKind::ExecLo: + EXIT_IF(offset != 1u); + result.kind = Decoder::OperandKind::ExecHi; + break; + case Decoder::OperandKind::VccHi: + case Decoder::OperandKind::ExecHi: EXIT("special-register operand offset is out of range"); + default: return operand; + } return result; } -IR::Operand Translator::ScalarDestinationOperand(const IR::Operand& operand, uint32_t offset) { - EXIT_IF(operand.kind != IR::OperandKind::Register); +Decoder::Operand Translator::ScalarDestinationOperand(const Decoder::Operand& operand, + uint32_t offset) { uint32_t code = 0; - switch (operand.reg.file) { - case IR::RegisterFile::Scalar: code = operand.reg.index; break; - case IR::RegisterFile::Vcc: code = 106u + operand.reg.index; break; + switch (operand.kind) { + case Decoder::OperandKind::Sgpr: code = operand.reg; break; + case Decoder::OperandKind::VccLo: code = 106u; break; + case Decoder::OperandKind::VccHi: code = 107u; break; default: EXIT("invalid scalar-memory destination"); } code += offset; - IR::Operand result; - result.kind = IR::OperandKind::Register; + Decoder::Operand result {}; if (code < 106u) { - result.reg = {IR::RegisterFile::Scalar, code}; + result.kind = Decoder::OperandKind::Sgpr; + result.reg = code; } else { switch (code) { - case 106u: result.reg = {IR::RegisterFile::Vcc, 0u}; break; - case 107u: result.reg = {IR::RegisterFile::Vcc, 1u}; break; + case 106u: result.kind = Decoder::OperandKind::VccLo; break; + case 107u: result.kind = Decoder::OperandKind::VccHi; break; default: EXIT("scalar-memory destination crosses an invalid register"); } } return result; } -IR::Operand Translator::PlainOperand(const IR::Operand& operand) { +Decoder::Operand Translator::PlainOperand(const Decoder::Operand& operand) { auto result = operand; result.sdwa_sel = 6; result.sdwa_dst_unused = 2; @@ -81,45 +110,37 @@ IR::Operand Translator::PlainOperand(const IR::Operand& operand) { return result; } -IR::U32 Translator::ReadPcRelativeU32(const IR::Operand& operand) { - EXIT_IF(operand.kind != IR::OperandKind::PcRelativeU32 && - operand.kind != IR::OperandKind::PcRelativeHighU32); - const auto address = - ir.Emit(IR::ValueOpcode::IAdd64, {ir.Emit(IR::ValueOpcode::GetShaderBase), - IR::Value(static_cast(operand.imm))}); - return ir.CompositeExtract(address, operand.kind == IR::OperandKind::PcRelativeU32 ? 0u : 1u); -} - std::array Translator::BallotMask(IR::U1 value) { return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))), IR::U32(IR::Value(0u))}; } -IR::U32 Translator::ReadRawU32(const IR::Operand& operand) { - if (operand.kind == IR::OperandKind::ImmediateU32) { - return IR::U32(IR::Value(operand.imm)); - } - if (operand.kind == IR::OperandKind::PcRelativeU32 || - operand.kind == IR::OperandKind::PcRelativeHighU32) { - return ReadPcRelativeU32(operand); - } - if (operand.kind == IR::OperandKind::Null) { - return IR::U32(IR::Value(0u)); - } - EXIT_IF(operand.kind != IR::OperandKind::Register); - switch (operand.reg.file) { - case IR::RegisterFile::Scalar: - return ir.GetScalarReg(static_cast(operand.reg.index)); - case IR::RegisterFile::Vector: - return ir.GetVectorReg(static_cast(operand.reg.index)); - case IR::RegisterFile::Vcc: return operand.reg.index == 0 ? ir.GetVccLo() : ir.GetVccHi(); - case IR::RegisterFile::M0: return ir.GetM0(); - case IR::RegisterFile::Exec: - return operand.reg.index == 0 ? ir.GetExecLo() : ir.GetExecHi(); - case IR::RegisterFile::Scc: +IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) { + switch (operand.kind) { + case Decoder::OperandKind::LiteralConstant: + case Decoder::OperandKind::IntegerInlineConstant: + case Decoder::OperandKind::FloatInlineConstant: return IR::U32(IR::Value(operand.value)); + case Decoder::OperandKind::Null: + case Decoder::OperandKind::PopsExitingWaveId: return IR::U32(IR::Value(0u)); + case Decoder::OperandKind::Sgpr: + return ir.GetScalarReg(static_cast(operand.reg)); + case Decoder::OperandKind::Vgpr: + return ir.GetVectorReg(static_cast(operand.reg)); + case Decoder::OperandKind::VccLo: return ir.GetVccLo(); + case Decoder::OperandKind::VccHi: return ir.GetVccHi(); + case Decoder::OperandKind::M0: return ir.GetM0(); + case Decoder::OperandKind::ExecLo: return ir.GetExecLo(); + case Decoder::OperandKind::ExecHi: return ir.GetExecHi(); + case Decoder::OperandKind::Scc: return ir.Select(ir.GetScc(), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))); + case Decoder::OperandKind::VccZ: + return ir.Select(ir.LogicalNot(ir.GetVcc()), IR::U32(IR::Value(1u)), + IR::U32(IR::Value(0u))); + case Decoder::OperandKind::ExecZ: + return ir.Select(ir.LogicalNot(ir.GetExec()), IR::U32(IR::Value(1u)), + IR::U32(IR::Value(0u))); + default: EXIT("invalid decoded operand used as a raw U32 source"); } - EXIT("invalid register file used as a raw U32 source"); } // Scalar operands share one encoded namespace with VCC, M0, and EXEC aliases. @@ -140,7 +161,7 @@ IR::U32 Translator::ReadScalarCode(uint32_t code) { } } -IR::U32 Translator::ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value) { +IR::U32 Translator::ApplyBitSourceModifiers(const Decoder::Operand& operand, IR::U32 value) { if (operand.dpp) { const IR::DppMoveFlags flags { .control = static_cast(operand.dpp_ctrl), @@ -170,7 +191,7 @@ IR::U32 Translator::ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 return value; } -IR::Value Translator::ReadOperand(const IR::Operand& operand, IR::Type type) { +IR::Value Translator::ReadOperand(const Decoder::Operand& operand, IR::Type type) { if (type == IR::Type::U16) { return ir.Emit(IR::ValueOpcode::ConvertU16U32, {ApplyBitSourceModifiers(operand, ReadRawU32(operand))}); @@ -181,11 +202,14 @@ IR::Value Translator::ReadOperand(const IR::Operand& operand, IR::Type type) { return ir.Emit(IR::ValueOpcode::BitCastF16U16, {bits}); } if (type == IR::Type::U1) { - EXIT_IF(operand.kind != IR::OperandKind::Register); - switch (operand.reg.file) { - case IR::RegisterFile::Scc: return ir.GetScc(); - case IR::RegisterFile::Exec: return ir.GetExec(); - case IR::RegisterFile::Vcc: return ir.GetVcc(); + switch (operand.kind) { + case Decoder::OperandKind::Scc: return ir.GetScc(); + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: return ir.GetExec(); + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: return ir.GetVcc(); + case Decoder::OperandKind::VccZ: return ir.LogicalNot(ir.GetVcc()); + case Decoder::OperandKind::ExecZ: return ir.LogicalNot(ir.GetExec()); default: break; } return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u))); @@ -212,17 +236,16 @@ IR::Value Translator::ReadOperand(const IR::Operand& operand, IR::Type type) { bits = ir.BitwiseXor(bits, IR::U32(IR::Value(0x80000000u))); } if (!TypesOverlap(type, IR::Type::U32)) { - EXIT("opcode %s at 0x%08x requested unsupported operand type %s", - IR::OpcodeName(current_opcode).data(), current_pc, IR::TypeName(type).c_str()); + EXIT("opcode %u at 0x%08x requested unsupported operand type %s", + static_cast(current_opcode), current_pc, IR::TypeName(type).c_str()); } return bits; } -void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) { - if (operand.kind == IR::OperandKind::Null) { +void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) { + if (operand.kind == Decoder::OperandKind::Null) { return; } - EXIT_IF(operand.kind != IR::OperandKind::Register); if (operand.sdwa_sel != 6u) { uint32_t offset = 0; uint32_t width = 0; @@ -259,9 +282,9 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) { default: EXIT("reserved SDWA DST_U mode"); } } - switch (operand.reg.file) { - case IR::RegisterFile::Scalar: { - const auto reg = static_cast(operand.reg.index); + switch (operand.kind) { + case Decoder::OperandKind::Sgpr: { + const auto reg = static_cast(operand.reg); ir.SetScalarReg(reg, value); ir.SetThreadBitScalarReg(reg, ThreadBit(value)); ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false))); @@ -271,8 +294,8 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) { } break; } - case IR::RegisterFile::Vector: { - const auto reg = static_cast(operand.reg.index); + case Decoder::OperandKind::Vgpr: { + const auto reg = static_cast(operand.reg); const auto old = ir.GetVectorReg(reg); if (operand.dpp) { const IR::DppMoveFlags flags { @@ -290,30 +313,31 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) { ir.SetVectorReg(reg, value); break; } - case IR::RegisterFile::Vcc: - if (operand.reg.index == 0) { - ir.SetVccLo(IR::U32(value)); - ir.SetVcc(ThreadBit(IR::U32(value))); - } else { - ir.SetVccHi(IR::U32(value)); - ir.SetVcc(ThreadBit(ir.GetVccLo())); - } + case Decoder::OperandKind::VccLo: + ir.SetVccLo(IR::U32(value)); + ir.SetVcc(ThreadBit(IR::U32(value))); break; - case IR::RegisterFile::M0: ir.SetM0(IR::U32(value)); break; - case IR::RegisterFile::Exec: - if (operand.reg.index == 0) { - ir.SetExecLo(IR::U32(value)); - ir.SetExec(ThreadBit(IR::U32(value))); - } else { - ir.SetExecHi(IR::U32(value)); - ir.SetExec(ThreadBit(ir.GetExecLo())); - } + case Decoder::OperandKind::VccHi: + ir.SetVccHi(IR::U32(value)); + ir.SetVcc(ThreadBit(ir.GetVccLo())); break; - case IR::RegisterFile::Scc: ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); break; + case Decoder::OperandKind::M0: ir.SetM0(IR::U32(value)); break; + case Decoder::OperandKind::ExecLo: + ir.SetExecLo(IR::U32(value)); + ir.SetExec(ThreadBit(IR::U32(value))); + break; + case Decoder::OperandKind::ExecHi: + ir.SetExecHi(IR::U32(value)); + ir.SetExec(ThreadBit(ir.GetExecLo())); + break; + case Decoder::OperandKind::Scc: + ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); + break; + default: EXIT("invalid decoded operand used as a destination"); } } -IR::F32 Translator::ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value) { +IR::F32 Translator::ApplyF32ResultModifiers(const Decoder::Operand& operand, IR::F32 value) { if (operand.omod != 0u) { float multiplier = 0.5f; switch (operand.omod) { @@ -329,8 +353,8 @@ IR::F32 Translator::ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 return value; } -void Translator::WriteOperand(const IR::Operand& operand, IR::Value value) { - if (operand.kind == IR::OperandKind::Null) { +void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value) { + if (operand.kind == Decoder::OperandKind::Null) { return; } auto type = value.GetType(); @@ -339,21 +363,22 @@ void Translator::WriteOperand(const IR::Operand& operand, IR::Value value) { type = IR::Type::F32; } if (type == IR::Type::Opaque) { - EXIT("opcode %s at 0x%08x produced an untyped value", IR::OpcodeName(current_opcode).data(), + EXIT("opcode %u at 0x%08x produced an untyped value", static_cast(current_opcode), current_pc); } if (type == IR::Type::U1) { - EXIT_IF(operand.kind != IR::OperandKind::Register); - switch (operand.reg.file) { - case IR::RegisterFile::Scc: ir.SetScc(IR::U1(value)); return; - case IR::RegisterFile::Exec: { + switch (operand.kind) { + case Decoder::OperandKind::Scc: ir.SetScc(IR::U1(value)); return; + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: { const auto mask = BallotMask(IR::U1(value)); ir.SetExec(IR::U1(value)); ir.SetExecLo(mask[0]); ir.SetExecHi(mask[1]); return; } - case IR::RegisterFile::Vcc: { + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: { const auto mask = BallotMask(IR::U1(value)); ir.SetVcc(IR::U1(value)); ir.SetVccLo(mask[0]); @@ -392,7 +417,7 @@ IR::U32 Translator::PackHalf2x16(IR::F32 low, IR::F32 high) { return IR::U32(ir.Emit(IR::ValueOpcode::PackHalf2x16, {pair})); } -void Translator::WriteF16(const IR::Operand& operand, IR::F32 value) { +void Translator::WriteF16(const Decoder::Operand& operand, IR::F32 value) { value = ApplyF32ResultModifiers(operand, value); const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value})); const auto bits = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, @@ -411,7 +436,7 @@ void Translator::WriteF16(const IR::Operand& operand, IR::F32 value) { WriteRawU32(raw, merged); } -void Translator::WriteU16(const IR::Operand& operand, IR::U32 value) { +void Translator::WriteU16(const Decoder::Operand& operand, IR::U32 value) { auto merged = ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu))); if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) { merged = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert, @@ -426,38 +451,37 @@ void Translator::WriteU16(const IR::Operand& operand, IR::U32 value) { WriteRawU32(raw, merged); } -IR::U32 Translator::ReadU32(const IR::Operand& operand) { +IR::U32 Translator::ReadU32(const Decoder::Operand& operand) { return IR::U32(ReadOperand(operand, IR::Type::U32)); } -std::array Translator::ReadU32Pair(const IR::Operand& operand) { - if (operand.kind == IR::OperandKind::Register) { - if (operand.reg.file == IR::RegisterFile::Exec) { - return {ir.GetExecLo(), ir.GetExecHi()}; - } - if (operand.reg.file == IR::RegisterFile::Vcc) { - return {ir.GetVccLo(), ir.GetVccHi()}; - } +std::array Translator::ReadU32Pair(const Decoder::Operand& operand) { + if (operand.kind == Decoder::OperandKind::ExecLo) { + return {ir.GetExecLo(), ir.GetExecHi()}; + } + if (operand.kind == Decoder::OperandKind::VccLo) { + return {ir.GetVccLo(), ir.GetVccHi()}; } const auto low = ApplyBitSourceModifiers(operand, ReadRawU32(operand)); IR::U32 high(IR::Value(0u)); - if (operand.kind == IR::OperandKind::Register) { + if (operand.kind == Decoder::OperandKind::Sgpr || operand.kind == Decoder::OperandKind::Vgpr) { high = ReadRawU32(OffsetOperand(operand, 1)); - } else if (operand.kind == IR::OperandKind::ImmediateU32 && operand.sext_64) { + } else if (operand.kind == Decoder::OperandKind::IntegerInlineConstant && + operand.signed_val < 0) { high = IR::U32(IR::Value(0xffffffffu)); } return {low, high}; } -IR::U64 Translator::ReadU64(const IR::Operand& operand) { +IR::U64 Translator::ReadU64(const Decoder::Operand& operand) { return IR::U64(ReadOperand(operand, IR::Type::U64)); } -IR::F32 Translator::ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed) { - if (operand.float_inline) { +IR::F32 Translator::ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane, bool packed) { + if (operand.kind == Decoder::OperandKind::FloatInlineConstant) { const bool use_zero = packed && (high_lane ? operand.op_sel_hi : operand.op_sel); auto value = use_zero ? IR::F32(IR::Value::F32(0.0f)) - : ir.BitCastF32(IR::U32(IR::Value(operand.imm))); + : ir.BitCastF32(IR::U32(IR::Value(operand.value))); const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value})); value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half})); if (operand.absolute) { @@ -490,11 +514,11 @@ IR::F32 Translator::ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, return value; } -IR::F32 Translator::ReadF16AsF32(const IR::Operand& operand) { +IR::F32 Translator::ReadF16AsF32(const Decoder::Operand& operand) { return ReadF16LaneAsF32(operand, false); } -IR::F32 Translator::ReadMixF32(const IR::Operand& operand) { +IR::F32 Translator::ReadMixF32(const Decoder::Operand& operand) { if (operand.op_sel_hi) { return ReadF16AsF32(operand); } @@ -505,7 +529,7 @@ IR::F32 Translator::ReadMixF32(const IR::Operand& operand) { return IR::F32(ReadOperand(value_operand, IR::Type::F32)); } -IR::U32 Translator::ReadU16LaneRaw(const IR::Operand& operand, bool high_lane) { +IR::U32 Translator::ReadU16LaneRaw(const Decoder::Operand& operand, bool high_lane) { auto raw_operand = operand; raw_operand.sdwa_sel = 6; raw_operand.sdwa_sext = false; @@ -522,7 +546,8 @@ IR::U32 Translator::ReadU16LaneRaw(const IR::Operand& operand, bool high_lane) { ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(width)})); } -IR::U32 Translator::ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend) { +IR::U32 Translator::ReadU16LaneAsU32(const Decoder::Operand& operand, bool high_lane, + bool sign_extend) { auto value = ReadU16LaneRaw(operand, high_lane); if (high_lane ? operand.negate_hi : operand.negate) { value = ir.BitwiseAnd(ir.ISub(IR::U32(IR::Value(0u)), value), IR::U32(IR::Value(0xffffu))); @@ -534,11 +559,11 @@ IR::U32 Translator::ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, return value; } -IR::U32 Translator::ReadU16AsU32(const IR::Operand& operand, bool sign_extend) { +IR::U32 Translator::ReadU16AsU32(const Decoder::Operand& operand, bool sign_extend) { return ReadU16LaneAsU32(operand, false, sign_extend); } -IR::U32 Translator::ReadF16LaneBits(const IR::Operand& operand, bool high_lane) { +IR::U32 Translator::ReadF16LaneBits(const Decoder::Operand& operand, bool high_lane) { auto value = ReadU16LaneRaw(operand, high_lane); if (operand.absolute) { value = ir.BitwiseAnd(value, IR::U32(IR::Value(0x7fffu))); @@ -553,26 +578,25 @@ std::array Translator::ExtractU64(IR::U64 value) { return {ir.CompositeExtract(value, 0), ir.CompositeExtract(value, 1)}; } -void Translator::WriteU32Pair(const IR::Operand& operand, const std::array& value) { - if (operand.kind == IR::OperandKind::Null) { +void Translator::WriteU32Pair(const Decoder::Operand& operand, + const std::array& value) { + if (operand.kind == Decoder::OperandKind::Null) { return; } - if (operand.kind == IR::OperandKind::Register) { - const auto thread_bit = ThreadBit(value[0]); - switch (operand.reg.file) { - case IR::RegisterFile::Exec: - ir.SetExec(thread_bit); - ir.SetExecLo(value[0]); - ir.SetExecHi(value[1]); - return; - case IR::RegisterFile::Vcc: - ir.SetVcc(thread_bit); - ir.SetVccLo(value[0]); - ir.SetVccHi(value[1]); - return; - case IR::RegisterFile::Scalar: break; - default: break; - } + const auto thread_bit = ThreadBit(value[0]); + switch (operand.kind) { + case Decoder::OperandKind::ExecLo: + ir.SetExec(thread_bit); + ir.SetExecLo(value[0]); + ir.SetExecHi(value[1]); + return; + case Decoder::OperandKind::VccLo: + ir.SetVcc(thread_bit); + ir.SetVccLo(value[0]); + ir.SetVccHi(value[1]); + return; + case Decoder::OperandKind::Sgpr: break; + default: break; } WriteRawU32(operand, value[0]); WriteRawU32(OffsetOperand(operand, 1), value[1]); @@ -582,58 +606,71 @@ IR::U1 Translator::ThreadBit(IR::U32 low) { return ir.INotEqual(low, IR::U32(IR::Value(0u))); } -IR::U1 Translator::ReadCondition(const IR::Operand& operand) { +IR::U1 Translator::ReadCondition(const Decoder::Operand& operand) { return IR::U1(ReadOperand(operand, IR::Type::U1)); } -IR::U32 Translator::ConditionBit(const IR::Operand& operand) { - if (operand.kind == IR::OperandKind::ImmediateU32) { - return IR::U32(IR::Value(operand.imm & 1u)); +IR::U32 Translator::ConditionBit(const Decoder::Operand& operand) { + if (operand.kind == Decoder::OperandKind::LiteralConstant || + operand.kind == Decoder::OperandKind::IntegerInlineConstant || + operand.kind == Decoder::OperandKind::FloatInlineConstant) { + return IR::U32(IR::Value(operand.value & 1u)); } return ir.Select(ReadMask(operand), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))); } -IR::U1 Translator::ReadMask(const IR::Operand& operand) { - if (operand.kind == IR::OperandKind::ImmediateU32) { - return IR::U1(IR::Value(operand.imm != 0u || operand.sext_64)); +IR::U1 Translator::ReadMask(const Decoder::Operand& operand) { + if (operand.kind == Decoder::OperandKind::LiteralConstant || + operand.kind == Decoder::OperandKind::IntegerInlineConstant || + operand.kind == Decoder::OperandKind::FloatInlineConstant) { + return IR::U1(IR::Value(operand.value != 0u)); } - EXIT_IF(operand.kind != IR::OperandKind::Register); - switch (operand.reg.file) { - case IR::RegisterFile::Scalar: { - const auto reg = static_cast(operand.reg.index); + switch (operand.kind) { + case Decoder::OperandKind::Sgpr: { + const auto reg = static_cast(operand.reg); return ir.GetThreadBitScalarReg(reg); } - case IR::RegisterFile::Exec: return ir.GetExec(); - case IR::RegisterFile::Vcc: return ir.GetVcc(); - case IR::RegisterFile::Scc: return ir.GetScc(); + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: return ir.GetExec(); + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: return ir.GetVcc(); + case Decoder::OperandKind::Scc: return ir.GetScc(); + case Decoder::OperandKind::VccZ: return ir.LogicalNot(ir.GetVcc()); + case Decoder::OperandKind::ExecZ: return ir.LogicalNot(ir.GetExec()); default: return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u))); } } -IR::U1 Translator::ReadMaskValid(const IR::Operand& operand) { - if (operand.kind == IR::OperandKind::ImmediateU32) { - const bool zero = operand.imm == 0u && !operand.sext_64; - const bool ones = operand.imm == 0xffffffffu && operand.sext_64; +IR::U1 Translator::ReadMaskValid(const Decoder::Operand& operand) { + if (operand.kind == Decoder::OperandKind::LiteralConstant || + operand.kind == Decoder::OperandKind::IntegerInlineConstant || + operand.kind == Decoder::OperandKind::FloatInlineConstant) { + const bool zero = operand.value == 0u; + const bool ones = operand.value == 0xffffffffu && + operand.kind == Decoder::OperandKind::IntegerInlineConstant && + operand.signed_val < 0; return IR::U1(IR::Value(zero || ones)); } - if (operand.kind != IR::OperandKind::Register) { - return IR::U1(IR::Value(false)); - } - switch (operand.reg.file) { - case IR::RegisterFile::Scalar: - return ir.GetScalarMaskTag(static_cast(operand.reg.index)); - case IR::RegisterFile::Exec: - case IR::RegisterFile::Vcc: - case IR::RegisterFile::Scc: return IR::U1(IR::Value(true)); + switch (operand.kind) { + case Decoder::OperandKind::Null: + case Decoder::OperandKind::PopsExitingWaveId: return IR::U1(IR::Value(true)); + case Decoder::OperandKind::Sgpr: + return ir.GetScalarMaskTag(static_cast(operand.reg)); + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: + case Decoder::OperandKind::VccZ: + case Decoder::OperandKind::ExecZ: + case Decoder::OperandKind::Scc: return IR::U1(IR::Value(true)); default: return IR::U1(IR::Value(false)); } } -void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) { - EXIT_IF(operand.kind != IR::OperandKind::Register); - switch (operand.reg.file) { - case IR::RegisterFile::Scalar: { - const auto reg = static_cast(operand.reg.index); +void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) { + switch (operand.kind) { + case Decoder::OperandKind::Sgpr: { + const auto reg = static_cast(operand.reg); const auto mask = BallotMask(value); ir.SetThreadBitScalarReg(reg, value); ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true))); @@ -651,21 +688,23 @@ void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) { } return; } - case IR::RegisterFile::Exec: { + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: { const auto mask = BallotMask(value); ir.SetExec(value); ir.SetExecLo(mask[0]); ir.SetExecHi(mask[1]); return; } - case IR::RegisterFile::Vcc: { + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: { const auto mask = BallotMask(value); ir.SetVcc(value); ir.SetVccLo(mask[0]); ir.SetVccHi(mask[1]); return; } - case IR::RegisterFile::Scc: ir.SetScc(value); return; + case Decoder::OperandKind::Scc: ir.SetScc(value); return; default: WriteRawU32(operand, ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)))); WriteRawU32(OffsetOperand(operand, 1), IR::U32(IR::Value(0u))); @@ -673,12 +712,12 @@ void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) { } } -void Translator::WriteMask64(const IR::Operand& operand, IR::U1 value) { - if (operand.kind != IR::OperandKind::Register || operand.reg.file != IR::RegisterFile::Scalar) { +void Translator::WriteMask64(const Decoder::Operand& operand, IR::U1 value) { + if (operand.kind != Decoder::OperandKind::Sgpr) { WriteMask(operand, value); return; } - const auto reg = static_cast(operand.reg.index); + const auto reg = static_cast(operand.reg); const auto mask = BallotMask(value); ir.SetThreadBitScalarReg(reg, value); ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true))); @@ -695,8 +734,8 @@ void Translator::WriteMask64(const IR::Operand& operand, IR::U1 value) { } } -void Translator::WriteCompareResult(const IR::Operand& operand, IR::U1 value) { - if (operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc) { +void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 value) { + if (operand.kind == Decoder::OperandKind::Scc) { WriteOperand(operand, value); return; } @@ -706,23 +745,7 @@ void Translator::WriteCompareResult(const IR::Operand& operand, IR::U1 value) { WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value)); } -bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error) { - for (const auto& source_inst: source.instructions) { - current_opcode = source_inst.op; - current_pc = source_inst.pc; - if (static_cast(source_inst.op) >= static_cast(IR::Opcode::Count)) { - return Fail(error, "value IR input opcode is out of range"); - } - if (TranslateInstruction(source_inst)) { - continue; - } - return Fail(error, fmt::format("opcode {} at 0x{:08x} has no typed Value IR lowering", - IR::OpcodeName(source_inst.op), source_inst.pc)); - } - return true; -} - -bool Translator::AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info, +bool Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info, std::string* error) { if (source.terminator.goto_value >= 0) { if (source.terminator.goto_variable == UINT32_MAX) { @@ -772,89 +795,243 @@ bool Translator::AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlock return true; } -} // namespace Detail +namespace { -bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, - const ShaderVertexInputInfo* vertex_input_info, - const ShaderPixelInputInfo* pixel_input_info, - const ShaderComputeInputInfo* compute_input_info, std::string* error) { - result = {}; - const uint32_t wave_size = source.wave_size; - uint32_t vector_limit = 1u; - const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) { - if (operand.kind == IR::OperandKind::Register && - operand.reg.file == IR::RegisterFile::Vector) { - vector_limit = - std::min(IR::NumVectorRegs, std::max(vector_limit, operand.reg.index + count)); +bool IsCodeTableLoad(const CFG::Graph& cfg, uint32_t pc) { + return std::ranges::find(cfg.code_table_load_pcs, pc) != cfg.code_table_load_pcs.end(); +} + +const EmbeddedFetchLoad* FindEmbeddedFetchLoad(const EmbeddedFetchPlan* plan, uint32_t pc) { + if (plan == nullptr) { + return nullptr; + } + const auto found = std::ranges::find(plan->loads, pc, &EmbeddedFetchLoad::pc); + return found != plan->loads.end() ? &*found : nullptr; +} + +bool IsEmbeddedFetchPrologLoad(const EmbeddedFetchPlan* plan, uint32_t pc) { + if (plan == nullptr) { + return false; + } + return std::ranges::any_of(plan->loads, [pc](const auto& load) { + return std::ranges::find(load.prolog_loads, pc) != load.prolog_loads.end(); + }); +} + +int ResolveEmbeddedFetchResource(const ShaderVertexInputInfo& input, + const EmbeddedFetchLoad& load) { + if (load.attrib_id >= 0 && load.attrib_id < input.resources_num && + input.resources_dst[load.attrib_id].attr_id == load.attrib_id) { + return load.attrib_id; + } + for (int index = 0; index < input.resources_num; index++) { + const auto& destination = input.resources_dst[index]; + if (destination.attr_id == load.attrib_id && + load.components <= static_cast(std::max(destination.registers_num, 1))) { + return index; + } + } + for (int index = 0; index < input.resources_num; index++) { + if (input.resources_dst[index].attr_id == load.attrib_id) { + return index; + } + } + return -1; +} + +bool IsScalarMemoryLoad(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::S_LOAD_DWORD: + case Decoder::Opcode::S_LOAD_DWORDX2: + case Decoder::Opcode::S_LOAD_DWORDX4: + case Decoder::Opcode::S_LOAD_DWORDX8: + case Decoder::Opcode::S_LOAD_DWORDX16: + case Decoder::Opcode::S_BUFFER_LOAD_DWORD: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX2: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8: + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16: return true; + default: return false; + } +} + +bool IsBufferDwordLoad(Decoder::Opcode opcode) { + switch (opcode) { + case Decoder::Opcode::BUFFER_LOAD_DWORD: + case Decoder::Opcode::BUFFER_LOAD_DWORDX2: + case Decoder::Opcode::BUFFER_LOAD_DWORDX3: + case Decoder::Opcode::BUFFER_LOAD_DWORDX4: + case Decoder::Opcode::BUFFER_LOAD_FORMAT_X: + case Decoder::Opcode::BUFFER_LOAD_FORMAT_XY: + case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ: + case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW: + case Decoder::Opcode::TBUFFER_LOAD_FORMAT_X: + case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XY: + case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZ: + case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZW: return true; + default: return false; + } +} + +void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_t& vector_limit) { + const auto include_vector = [&](const Decoder::Operand& operand, uint32_t count = 1u) { + if (operand.kind == Decoder::OperandKind::Vgpr) { + vector_limit = std::min(IR::NumVectorRegs, std::max(vector_limit, operand.reg + count)); } }; - for (const auto& block: source.blocks) { - for (const auto& inst: block.instructions) { - include_vector(inst.dst); - include_vector(inst.dst2); - for (uint32_t index = 0; index < inst.src_count; index++) { - include_vector(inst.src[index]); - } - if (inst.op == IR::Opcode::BufferLoadDword) { - include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u)); - } else if (inst.op == IR::Opcode::BufferStoreDword) { - include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u)); - } else if (inst.op == IR::Opcode::DsReadB32 || inst.op == IR::Opcode::DsRead2B32) { - include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u)); - } else if (inst.op == IR::Opcode::DsWriteB32) { - include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u)); - } else if (inst.op == IR::Opcode::DsWrite2B32) { - const auto width = std::max(inst.memory.data_dwords / 2u, 1u); - include_vector(inst.src[0], width); - include_vector(inst.src[2], width); - } else if (inst.op == IR::Opcode::LoadInputF32) { - include_vector(inst.dst, std::max(inst.input_info.component_count, 1u)); + const bool memory_family = + inst.family == Decoder::Family::MUBUF || inst.family == Decoder::Family::MTBUF || + inst.family == Decoder::Family::FLAT || inst.family == Decoder::Family::DS || + inst.family == Decoder::Family::MIMG; + include_vector(inst.dst, memory_family ? std::max(inst.data_dwords, 1u) : 1u); + include_vector(inst.dst2); + include_vector(inst.src0); + include_vector(inst.src1); + include_vector(inst.src2); + include_vector(inst.src3); + if (inst.family == Decoder::Family::DS) { + switch (inst.opcode) { + case Decoder::Opcode::DS_WRITE_B64: + case Decoder::Opcode::DS_WRITE_B96: + case Decoder::Opcode::DS_WRITE_B128: include_vector(inst.src1, inst.data_dwords); break; + case Decoder::Opcode::DS_WRITE2_B32: + case Decoder::Opcode::DS_WRITE2ST64_B32: + case Decoder::Opcode::DS_WRITE2_B64: + case Decoder::Opcode::DS_WRITE2ST64_B64: { + const auto width = std::max(inst.data_dwords / 2u, 1u); + include_vector(inst.src1, width); + include_vector(inst.src2, width); + break; } + default: break; } } - result.block_storage.reserve(source.blocks.size() + 1u); - result.blocks.reserve(source.blocks.size() + 1u); - result.block_info.reserve(source.blocks.size() + 1u); - if (!source.blocks.empty()) { - const auto max_id = - std::ranges::max_element(source.blocks, {}, [](const IR::BasicBlock& block) { - return block.id; - })->id; - if (max_id == UINT32_MAX) { - return Fail(error, "cannot allocate a typed entry block id"); - } - CFG::Terminator terminator; - terminator.kind = CFG::TerminatorKind::Branch; - terminator.true_block = source.blocks.front().id; - result.block_storage.push_back(std::make_unique()); - result.blocks.push_back(result.block_storage.back().get()); - result.block_info.push_back({max_id + 1u, source.blocks.front().start_pc, - source.blocks.front().start_pc, std::move(terminator)}); + for (uint32_t index = 0; index + 1u < inst.image_address_components && + index < Decoder::MaxImageNsaAddressComponents; + index++) { + vector_limit = + std::min(IR::NumVectorRegs, std::max(vector_limit, inst.image_nsa_addr[index] + 1u)); } - for (const auto& source_block: source.blocks) { +} + +bool ValidateTranslateOptions(const TranslateOptions& options, std::string* error) { + if (options.wave_size != 32u && options.wave_size != 64u) { + return Fail(error, "shader translation requires wave32 or wave64"); + } + switch (options.stage) { + case ShaderType::Vertex: + return options.vertex != nullptr || + Fail(error, "vertex shader translation has no vertex input metadata"); + case ShaderType::Pixel: + return options.pixel != nullptr || + Fail(error, "pixel shader translation has no pixel input metadata"); + case ShaderType::Compute: + return options.compute != nullptr || + Fail(error, "compute shader translation has no compute input metadata"); + default: return Fail(error, "shader translation has an unsupported stage"); + } +} + +} // namespace + +bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, + const TranslateOptions& options, IR::Program& result, std::string* error) { + result = {}; + struct ResetOnFailure { + IR::Program& program; + bool complete = false; + + ~ResetOnFailure() { + if (!complete) { + program = {}; + } + } + } reset_on_failure {result}; + + if (!ValidateTranslateOptions(options, error)) { + return false; + } + if (cfg.blocks.empty()) { + return Fail(error, "cannot translate an empty CFG"); + } + + result.stage = options.stage; + result.wave_size = options.wave_size; + result.shader_hash = options.shader_hash; + result.user_data_base = options.user_data_base; + result.user_data_count = options.user_data_count; + result.scratch_dwords = options.scratch_dwords; + result.dispatcher_fallback = options.dispatcher_fallback; + result.cfg_failure_kind = options.cfg_failure_kind; + result.fallback_reason = options.fallback_reason; + if (options.embedded_fetch != nullptr) { + result.info.vertex_offset_sgpr = options.embedded_fetch->vertex_offset_sgpr; + } + + uint32_t vector_limit = 1u; + for (const auto& cfg_block: cfg.blocks) { + for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) { + if (index >= decoded.instructions.size()) { + return Fail(error, + "CFG block references an instruction outside the decoded program"); + } + const auto& instruction = decoded.instructions[index]; + if (IsCodeTableLoad(cfg, instruction.pc)) { + continue; + } + IncludeInstructionVectorRegisters(instruction, vector_limit); + } + } + + result.block_storage.reserve(cfg.blocks.size() + 1u); + result.blocks.reserve(cfg.blocks.size() + 1u); + result.block_info.reserve(cfg.blocks.size() + 1u); + const auto max_id = std::ranges::max_element(cfg.blocks, {}, [](const CFG::BasicBlock& block) { + return block.id; + })->id; + if (max_id == UINT32_MAX) { + return Fail(error, "cannot allocate a typed entry block id"); + } + CFG::Terminator terminator; + terminator.kind = CFG::TerminatorKind::Branch; + terminator.true_block = cfg.blocks.front().id; + result.block_storage.push_back(std::make_unique()); + result.blocks.push_back(result.block_storage.back().get()); + result.block_info.push_back({max_id + 1u, cfg.blocks.front().start_pc, + cfg.blocks.front().start_pc, std::move(terminator)}); + + std::unordered_map block_indices; + block_indices.reserve(cfg.blocks.size()); + for (const auto& source_block: cfg.blocks) { + if (!block_indices.emplace(source_block.id, result.blocks.size()).second) { + return Fail(error, fmt::format("CFG contains duplicate block id {}", source_block.id)); + } result.block_storage.push_back(std::make_unique()); result.blocks.push_back(result.block_storage.back().get()); result.block_info.push_back( {source_block.id, source_block.start_pc, source_block.end_pc, source_block.terminator}); } - for (size_t index = 0; index < source.blocks.size(); index++) { - for (const auto successor: source.blocks[index].successors) { - if (successor >= source.blocks.size()) { - return Fail(error, "value IR block successor is out of range"); + for (const auto& source_block: cfg.blocks) { + const auto source_index = block_indices.at(source_block.id); + for (const auto successor: source_block.successors) { + const auto target = block_indices.find(successor); + if (target == block_indices.end()) { + return Fail(error, fmt::format("CFG block {} has unknown successor {}", + source_block.id, successor)); } - result.blocks[index + 1u]->AddBranch(result.blocks[successor + 1u]); + result.blocks[source_index]->AddBranch(result.blocks[target->second]); } } - if (!result.blocks.empty()) { - result.blocks.front()->AddBranch(result.blocks[1]); + { + result.blocks.front()->AddBranch(result.blocks.at(block_indices.at(cfg.blocks.front().id))); IR::IREmitter entry_ir(result.blocks.front()); const auto builtin = [&](IR::StageInputKind kind, uint32_t component = 0u) { return IR::U32( entry_ir.Emit(IR::ValueOpcode::GetBuiltin, {IR::Value(static_cast(kind)), IR::Value(component)})); }; - for (uint32_t index = 0; index < source.user_data_count; index++) { - const auto reg = static_cast(source.user_data_base + index); + for (uint32_t index = 0; index < options.user_data_count; index++) { + const auto reg = static_cast(options.user_data_base + index); if (IR::RegIndex(reg) >= IR::NumScalarRegs) { break; } @@ -866,8 +1043,8 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, entry_ir.SetExec(IR::U1(IR::Value(true))); entry_ir.SetExecLo(IR::U32(IR::Value(1u))); entry_ir.SetExecHi(IR::U32(IR::Value(0u))); - if (source.stage == ShaderType::Compute) { - const auto* cs = compute_input_info; + if (options.stage == ShaderType::Compute) { + const auto* cs = options.compute; const auto thread_ids = cs->thread_ids_num > 0 ? std::min(cs->thread_ids_num, 3u) : 0u; for (uint32_t index = 0; index < thread_ids; index++) { @@ -900,8 +1077,8 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, entry_ir.BitwiseOr(entry_ir.BitwiseOr(wave_bits, IR::U32(IR::Value(waves))), first_bit)); } - } else if (source.stage == ShaderType::Pixel) { - const auto* ps = pixel_input_info; + } else if (options.stage == ShaderType::Pixel) { + const auto* ps = options.pixel; if (ps->ps_perspective_center_vgpr != UINT32_MAX) { entry_ir.SetVectorReg(static_cast(ps->ps_perspective_center_vgpr), builtin(IR::StageInputKind::BaryCoordSmooth, 0)); @@ -930,22 +1107,68 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, entry_ir.SetVectorReg(static_cast(reg), builtin(IR::StageInputKind::FrontFacing)); } - } else if (source.stage == ShaderType::Vertex) { + } else if (options.stage == ShaderType::Vertex) { entry_ir.SetVectorReg(static_cast(5), builtin(IR::StageInputKind::VertexIndex)); entry_ir.SetVectorReg(static_cast(8), builtin(IR::StageInputKind::InstanceIndex)); } } - for (size_t index = 0; index < source.blocks.size(); index++) { - Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size); - if (!translator.TranslateBlock(source.blocks[index], error) || - !translator.AddBranchCondition(source.blocks[index], result.block_info[index + 1u], - error)) { + for (const auto& cfg_block: cfg.blocks) { + const auto typed_index = block_indices.at(cfg_block.id); + Translator translator(result, result.blocks[typed_index], vector_limit, options.wave_size); + for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) { + const auto& instruction = decoded.instructions[index]; + if (IsCodeTableLoad(cfg, instruction.pc)) { + continue; + } + if (IsScalarMemoryLoad(instruction.opcode) && + IsEmbeddedFetchPrologLoad(options.embedded_fetch, instruction.pc)) { + continue; + } + const auto* embedded = FindEmbeddedFetchLoad(options.embedded_fetch, instruction.pc); + if (embedded != nullptr && IsBufferDwordLoad(instruction.opcode) && + instruction.data_dwords == embedded->components && + instruction.dst.kind == Decoder::OperandKind::Vgpr) { + if (options.vertex == nullptr) { + return Fail(error, "embedded vertex fetch plan has no vertex input metadata"); + } + const auto resource = ResolveEmbeddedFetchResource(*options.vertex, *embedded); + if (resource < 0 || resource >= options.vertex->resources_num) { + return Fail( + error, + fmt::format( + "embedded vertex fetch at 0x{:08x} has no resource for attribute {}", + instruction.pc, embedded->attrib_id)); + } + result.info.vertex_fetch_components[static_cast(resource)] = + static_cast(std::max( + result.info.vertex_fetch_components[static_cast(resource)], + embedded->components)); + if (!translator.TranslateEmbeddedFetch(instruction, static_cast(resource), + embedded->components)) { + return false; + } + continue; + } + if (!translator.TranslateInstruction(instruction, error)) { + if (error == nullptr || error->empty()) { + return Fail(error, fmt::format("opcode {} at 0x{:08x} has no IR translation", + magic_enum::enum_name(instruction.opcode), + instruction.pc)); + } + return false; + } + } + if (!translator.AddBranchCondition(cfg_block, result.block_info[typed_index], error)) { return false; } } - return IR::ValidateValueProgram(result, false, error); + if (!IR::ValidateProgram(result, false, error)) { + return false; + } + reset_on_failure.complete = true; + return true; } } // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.h b/src/graphics/shader/recompiler/frontend/translate/Translate.h index 3c3efb7d0..af54d178f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.h @@ -1,12 +1,44 @@ #pragma once -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" + +#include +#include +#include namespace Libs::Graphics::ShaderRecompiler::Frontend { -bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, - const ShaderVertexInputInfo* vertex_input_info, - const ShaderPixelInputInfo* pixel_input_info, - const ShaderComputeInputInfo* compute_input_info, std::string* error); +struct EmbeddedFetchLoad { + uint32_t pc = 0; + int attrib_id = -1; + uint32_t components = 0; + std::vector prolog_loads; +}; + +struct EmbeddedFetchPlan { + std::vector loads; + int32_t vertex_offset_sgpr = -1; +}; + +struct TranslateOptions { + ShaderType stage = ShaderType::Unknown; + uint32_t wave_size = 64; + uint64_t shader_hash = 0; + uint32_t user_data_base = 0; + uint32_t user_data_count = 64; + uint32_t scratch_dwords = 0; + bool dispatcher_fallback = false; + CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None; + std::string_view fallback_reason; + const ShaderVertexInputInfo* vertex = nullptr; + const ShaderPixelInputInfo* pixel = nullptr; + const ShaderComputeInputInfo* compute = nullptr; + const EmbeddedFetchPlan* embedded_fetch = nullptr; +}; + +bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, + const TranslateOptions& options, IR::Program& result, std::string* error); } // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 76d630bba..0fdc2756b 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -5,58 +5,59 @@ #include -namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { +namespace Libs::Graphics::ShaderRecompiler::Frontend { class Translator { public: - Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit, - uint32_t wave_size) - : value_program(program), ir(block), current_vector_limit(vector_limit), + Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit, uint32_t wave_size) + : program(program), ir(block), current_vector_limit(vector_limit), current_wave_size(wave_size) {} - bool TranslateBlock(const IR::BasicBlock& source, std::string* error); - bool AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info, - std::string* error); + bool TranslateInstruction(const Decoder::Instruction& inst, std::string* error); + bool TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute, + uint32_t component_count); + bool AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info, std::string* error); private: - IR::Operand OffsetOperand(const IR::Operand& operand, uint32_t offset); - IR::Operand ScalarDestinationOperand(const IR::Operand& operand, uint32_t offset); - IR::Operand PlainOperand(const IR::Operand& operand); - IR::U32 ReadPcRelativeU32(const IR::Operand& operand); + Decoder::Operand SourceAt(const Decoder::Instruction& inst, uint32_t index); + Decoder::Operand DestinationOperand(const Decoder::Instruction& inst); + Decoder::Operand OffsetOperand(const Decoder::Operand& operand, uint32_t offset); + Decoder::Operand ScalarDestinationOperand(const Decoder::Operand& operand, uint32_t offset); + Decoder::Operand PlainOperand(const Decoder::Operand& operand); std::array BallotMask(IR::U1 value); - IR::U32 ReadRawU32(const IR::Operand& operand); + IR::U32 ReadRawU32(const Decoder::Operand& operand); IR::U32 ReadScalarCode(uint32_t code); - IR::U32 ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value); - IR::Value ReadOperand(const IR::Operand& operand, IR::Type type); + IR::U32 ApplyBitSourceModifiers(const Decoder::Operand& operand, IR::U32 value); + IR::Value ReadOperand(const Decoder::Operand& operand, IR::Type type); IR::U1 ThreadBit(IR::U32 low); - void WriteRawU32(const IR::Operand& operand, IR::U32 value); - IR::F32 ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value); - void WriteOperand(const IR::Operand& operand, IR::Value value); + void WriteRawU32(const Decoder::Operand& operand, IR::U32 value); + IR::F32 ApplyF32ResultModifiers(const Decoder::Operand& operand, IR::F32 value); + void WriteOperand(const Decoder::Operand& operand, IR::Value value); IR::U32 PackHalf2x16(IR::F32 low, IR::F32 high); - void WriteF16(const IR::Operand& operand, IR::F32 value); - void WriteU16(const IR::Operand& operand, IR::U32 value); - IR::U32 ReadU32(const IR::Operand& operand); - std::array ReadU32Pair(const IR::Operand& operand); - IR::U64 ReadU64(const IR::Operand& operand); - IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed = false); - IR::F32 ReadF16AsF32(const IR::Operand& operand); - IR::F32 ReadMixF32(const IR::Operand& operand); - IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane); - IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend); - IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend); - IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane); + void WriteF16(const Decoder::Operand& operand, IR::F32 value); + void WriteU16(const Decoder::Operand& operand, IR::U32 value); + IR::U32 ReadU32(const Decoder::Operand& operand); + std::array ReadU32Pair(const Decoder::Operand& operand); + IR::U64 ReadU64(const Decoder::Operand& operand); + IR::F32 ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane, bool packed = false); + IR::F32 ReadF16AsF32(const Decoder::Operand& operand); + IR::F32 ReadMixF32(const Decoder::Operand& operand); + IR::U32 ReadU16LaneRaw(const Decoder::Operand& operand, bool high_lane); + IR::U32 ReadU16LaneAsU32(const Decoder::Operand& operand, bool high_lane, bool sign_extend); + IR::U32 ReadU16AsU32(const Decoder::Operand& operand, bool sign_extend); + IR::U32 ReadF16LaneBits(const Decoder::Operand& operand, bool high_lane); std::array ExtractU64(IR::U64 value); - void WriteU32Pair(const IR::Operand& operand, const std::array& value); - IR::U1 ReadCondition(const IR::Operand& operand); - IR::U32 ConditionBit(const IR::Operand& operand); - IR::U1 ReadMask(const IR::Operand& operand); - IR::U1 ReadMaskValid(const IR::Operand& operand); - void WriteMask(const IR::Operand& operand, IR::U1 value); - void WriteMask64(const IR::Operand& operand, IR::U1 value); - void WriteCompareResult(const IR::Operand& operand, IR::U1 value); + void WriteU32Pair(const Decoder::Operand& operand, const std::array& value); + IR::U1 ReadCondition(const Decoder::Operand& operand); + IR::U32 ConditionBit(const Decoder::Operand& operand); + IR::U1 ReadMask(const Decoder::Operand& operand); + IR::U1 ReadMaskValid(const Decoder::Operand& operand); + void WriteMask(const Decoder::Operand& operand, IR::U1 value); + void WriteMask64(const Decoder::Operand& operand, IR::U1 value); + void WriteCompareResult(const Decoder::Operand& operand, IR::U1 value); - IR::MemoryFlags AddMemoryInfo(const IR::Instruction& inst); - IR::ExportFlags AddExportInfo(const IR::Instruction& inst); + IR::MemoryFlags AddMemoryInfo(const IR::MemoryInfo& memory, uint32_t pc); + IR::ExportFlags AddExportInfo(const Decoder::Instruction& inst); struct AddressOperands { IR::Value resource; IR::Value low; @@ -67,31 +68,31 @@ private: IR::U32 offset; IR::U32 soffset; }; - AddressOperands ReadAddressOperands(const IR::Instruction& inst, uint32_t first_source); + AddressOperands ReadAddressOperands(const Decoder::Instruction& inst, uint32_t first_source); IR::U32 GetResourceDword(uint32_t index, uint32_t dword); IR::Value GetBufferResource(const IR::MemoryInfo& memory); IR::Value GetAddressResource(IR::Value low, IR::Value high); IR::Value GetScalarAddressResource(uint32_t base); IR::Value GetImageResource(const IR::MemoryInfo& memory); IR::Value GetSamplerResource(const IR::MemoryInfo& memory); - IR::Value MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base); - IR::Value ConstructU32x4(const IR::Operand& base, uint32_t count); - void WriteImageComponents(const IR::Operand& dst, IR::Value value, const IR::MemoryInfo& memory, - uint32_t component_limit); - IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode); - BufferAddress ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset); + IR::Value MakeImageAddress(const Decoder::Instruction& inst, const Decoder::Operand& base); + IR::Value ConstructU32x4(const Decoder::Operand& base, uint32_t count); + void WriteImageComponents(const Decoder::Operand& dst, IR::Value value, + const IR::MemoryInfo& memory, uint32_t component_limit); + IR::ValueOpcode ImageAtomicOpcode(Decoder::Opcode opcode); + BufferAddress ReadBufferAddress(const Decoder::Instruction& inst, uint32_t source_offset); IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign); IR::Value NarrowSubdword(IR::U32 value, uint32_t bits); - IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode); - IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode); - bool TranslateScalarMemory(const IR::Instruction& inst); - bool TranslateBufferLoad(const IR::Instruction& inst); - bool TranslateBufferStore(const IR::Instruction& inst); - bool TranslateAtomicMemory(const IR::Instruction& inst); - bool TranslateFlatLoad(const IR::Instruction& inst); - bool TranslateFlatStore(const IR::Instruction& inst); - bool TranslateImageMemory(const IR::Instruction& inst); - bool TranslateSharedMemory(const IR::Instruction& inst); + IR::ValueOpcode BufferAtomicOpcode(Decoder::Opcode opcode); + IR::ValueOpcode SharedAtomicOpcode(Decoder::Opcode opcode); + bool TranslateScalarMemory(const Decoder::Instruction& inst, std::string* error); + bool TranslateBufferLoad(const Decoder::Instruction& inst, std::string* error); + bool TranslateBufferStore(const Decoder::Instruction& inst, std::string* error); + bool TranslateAtomicMemory(const Decoder::Instruction& inst, std::string* error); + bool TranslateFlatLoad(const Decoder::Instruction& inst, std::string* error); + bool TranslateFlatStore(const Decoder::Instruction& inst, std::string* error); + bool TranslateImageMemory(const Decoder::Instruction& inst, std::string* error); + bool TranslateSharedMemory(const Decoder::Instruction& inst, std::string* error); IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value); IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper, @@ -101,34 +102,33 @@ private: uint32_t upper_result); IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high); - bool TranslateInstruction(const IR::Instruction& inst); - bool TranslateStateOperation(const IR::Instruction& inst); - bool TranslateControlOperation(const IR::Instruction& inst); - bool TranslateMove(const IR::Instruction& inst); - bool TranslateLaneOperation(const IR::Instruction& inst); - bool TranslateAttributeOperation(const IR::Instruction& inst); - bool TranslateMemoryOperation(const IR::Instruction& inst); - bool TranslateIntegerCompare(const IR::Instruction& inst); - bool TranslateInteger16Compare(const IR::Instruction& inst); - bool TranslateFloatCompare(const IR::Instruction& inst); - bool TranslateConversion(const IR::Instruction& inst); - bool TranslateInteger16Operation(const IR::Instruction& inst); - bool TranslatePackedInteger16(const IR::Instruction& inst); - bool TranslatePackedFloat16(const IR::Instruction& inst); - bool TranslateFloat16Operation(const IR::Instruction& inst); - bool TranslateFloatOperation(const IR::Instruction& inst); - IR::U1 EvaluateU64Mask(const IR::Instruction& inst); - bool TranslateU64MaskOperation(const IR::Instruction& inst); - bool TranslateSimpleInteger(const IR::Instruction& inst); - bool TranslateComposedInteger(const IR::Instruction& inst); - bool TranslateExtendedInteger(const IR::Instruction& inst); + bool TranslateStateOperation(const Decoder::Instruction& inst); + bool TranslateControlOperation(const Decoder::Instruction& inst); + bool TranslateMove(const Decoder::Instruction& inst, std::string* error); + bool TranslateLaneOperation(const Decoder::Instruction& inst, std::string* error); + bool TranslateAttributeOperation(const Decoder::Instruction& inst, std::string* error); + bool TranslateMemoryOperation(const Decoder::Instruction& inst, std::string* error); + bool TranslateIntegerCompare(const Decoder::Instruction& inst); + bool TranslateInteger16Compare(const Decoder::Instruction& inst); + bool TranslateFloatCompare(const Decoder::Instruction& inst); + bool TranslateConversion(const Decoder::Instruction& inst); + bool TranslateInteger16Operation(const Decoder::Instruction& inst); + bool TranslatePackedInteger16(const Decoder::Instruction& inst); + bool TranslatePackedFloat16(const Decoder::Instruction& inst); + bool TranslateFloat16Operation(const Decoder::Instruction& inst); + bool TranslateFloatOperation(const Decoder::Instruction& inst); + IR::U1 EvaluateU64Mask(const Decoder::Instruction& inst); + bool TranslateU64MaskOperation(const Decoder::Instruction& inst); + bool TranslateSimpleInteger(const Decoder::Instruction& inst); + bool TranslateComposedInteger(const Decoder::Instruction& inst); + bool TranslateExtendedInteger(const Decoder::Instruction& inst); - IR::ValueProgram& value_program; - IR::IREmitter ir; - IR::Opcode current_opcode = IR::Opcode::ControlNop; - uint32_t current_pc = 0; - uint32_t current_vector_limit = 1; - uint32_t current_wave_size = 64; + IR::Program& program; + IR::IREmitter ir; + Decoder::Opcode current_opcode = Decoder::Opcode::UNKNOWN; + uint32_t current_pc = 0; + uint32_t current_vector_limit = 1; + uint32_t current_wave_size = 64; }; -} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.cpp b/src/graphics/shader/recompiler/ir/Program.cpp similarity index 97% rename from src/graphics/shader/recompiler/ir/ValueProgram.cpp rename to src/graphics/shader/recompiler/ir/Program.cpp index 7586172fc..c7a4758f0 100644 --- a/src/graphics/shader/recompiler/ir/ValueProgram.cpp +++ b/src/graphics/shader/recompiler/ir/Program.cpp @@ -1,11 +1,13 @@ -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/shader.h" #include #include +#include #include #include +#include namespace Libs::Graphics::ShaderRecompiler::IR { @@ -50,7 +52,7 @@ bool IsRuntimeRead(ValueOpcode opcode) { AddressOpcodeInfoOf(opcode).access == AddressAccess::Read; } -bool EquivalentValue(const ValueProgram& program, Value left, Value right, +bool EquivalentValue(const Program& program, Value left, Value right, std::vector>& visited) { left = left.Resolve(); right = right.Resolve(); @@ -93,7 +95,7 @@ bool EquivalentValue(const ValueProgram& program, Value left, Value right, } // namespace -ValueProgram::~ValueProgram() { +Program::~Program() { // Values may cross block boundaries. Detach all arguments before any block starts destroying // its instruction storage so reverse-use links always point to live definitions. for (auto* block: blocks) { @@ -103,12 +105,20 @@ ValueProgram::~ValueProgram() { } } -bool EquivalentValue(const ValueProgram& program, Value left, Value right) { +Program& Program::operator=(Program&& other) noexcept { + if (this != &other) { + this->~Program(); + new (this) Program(std::move(other)); + } + return *this; +} + +bool EquivalentValue(const Program& program, Value left, Value right) { std::vector> visited; return EquivalentValue(program, left, right, visited); } -Value ResolveInvariantPhi(const ValueProgram& program, Value value) { +Value ResolveInvariantPhi(const Program& program, Value value) { value = value.Resolve(); const auto* root = value.TryInstruction(); if (root == nullptr || root->GetOpcode() != ValueOpcode::Phi) { @@ -139,7 +149,7 @@ Value ResolveInvariantPhi(const ValueProgram& program, Value value) { return invariant; } -bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::string* error) { +bool ValidateProgram(const Program& program, bool require_ssa, std::string* error) { if (program.blocks.size() != program.block_info.size() || program.blocks.size() != program.block_storage.size()) { return Fail(error, "value IR block storage is inconsistent"); @@ -591,14 +601,14 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st return true; } -void ResolveControlFlowIdentities(ValueProgram& program) { +void ResolveControlFlowIdentities(Program& program) { for (auto& info: program.block_info) { info.condition = info.condition.Resolve(); info.indirect_target = info.indirect_target.Resolve(); } } -std::string ValueProgramToString(const ValueProgram& program) { +std::string ProgramToString(const Program& program) { std::map ids; size_t next_id = 1; for (const auto* block: program.blocks) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.cpp b/src/graphics/shader/recompiler/ir/ShaderIR.cpp deleted file mode 100644 index 7a655e4f2..000000000 --- a/src/graphics/shader/recompiler/ir/ShaderIR.cpp +++ /dev/null @@ -1,1250 +0,0 @@ -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -#include "common/assert.h" -#include "graphics/shader/recompiler/BufferFormat.h" -#include "graphics/shader/recompiler/ir/ShaderIRInternal.h" - -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { - -Operand MakeSccOperand() { - Operand operand; - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Scc; - operand.reg.index = static_cast(Decoder::OperandKind::Scc); - return operand; -} - -Operand MakeImmediateU32(uint32_t value) { - Operand operand; - operand.kind = OperandKind::ImmediateU32; - operand.imm = value; - return operand; -} - -Operand MakePcRelativeU32(uint32_t value) { - Operand operand; - operand.kind = OperandKind::PcRelativeU32; - operand.imm = value; - return operand; -} - -Operand MakePcRelativeHighU32(uint32_t value) { - Operand operand; - operand.kind = OperandKind::PcRelativeHighU32; - operand.imm = value; - return operand; -} - -bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - if (!ScalarResultWritesSccNonZero(decoded.opcode)) { - return true; - } - switch (decoded.opcode) { - case Decoder::Opcode::S_AND_B64: - case Decoder::Opcode::S_ANDN2_B64: - case Decoder::Opcode::S_NOT_B64: - case Decoder::Opcode::S_OR_B64: - case Decoder::Opcode::S_ORN2_B64: - case Decoder::Opcode::S_XOR_B64: - case Decoder::Opcode::S_NAND_B64: - case Decoder::Opcode::S_NOR_B64: - case Decoder::Opcode::S_XNOR_B64: - case Decoder::Opcode::S_WQM_B64: - // These instructions maintain an invocation-local mask shadow and update SCC - // while lowering the operation. A raw dword comparison would disagree after - // complementing a mask temporary. - return true; - default: break; - } - if (block.instructions.empty()) { - SetError(error, "internal IR error: missing scalar result for SCC update"); - return false; - } - - const auto result = block.instructions.back().dst; - if (result.kind != OperandKind::Register) { - SetError(error, "internal IR error: scalar SCC update without register result"); - return false; - } - - // Scalar SOP result instructions update SCC from result != 0. Text pixel shaders commonly - // branch on SCC immediately after masking EXEC with VCC via s_andn2_b64. - Instruction scc_update; - scc_update.pc = decoded.pc; - scc_update.op = - ScalarResultIs64Bit(decoded.opcode) ? Opcode::CompareNeU64 : Opcode::CompareNeU32; - scc_update.dst = MakeSccOperand(); - scc_update.src[0] = result; - scc_update.src[1] = MakeImmediateU32(0); - scc_update.src_count = 2; - block.instructions.push_back(scc_update); - return true; -} - -uint32_t VectorByteConvertIndex(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::V_CVT_F32_UBYTE0: return 0; - case Decoder::Opcode::V_CVT_F32_UBYTE1: return 1; - case Decoder::Opcode::V_CVT_F32_UBYTE2: return 2; - case Decoder::Opcode::V_CVT_F32_UBYTE3: return 3; - default: return 0xffffffffu; - } -} - -void CopyOperandModifiers(const Decoder::Operand& decoded, Operand& operand) { - operand.sdwa_sel = decoded.sdwa_sel; - operand.sdwa_dst_unused = decoded.sdwa_dst_unused; - operand.omod = decoded.omod; - operand.sdwa_sext = decoded.sdwa_sext; - operand.op_sel = decoded.op_sel; - operand.op_sel_hi = decoded.op_sel_hi; - operand.negate = decoded.negate; - operand.negate_hi = decoded.negate_hi; - operand.absolute = decoded.absolute; - operand.clamp = decoded.clamp; - operand.dpp_ctrl = decoded.dpp_ctrl; - operand.dpp_row_mask = decoded.dpp_row_mask; - operand.dpp_bank_mask = decoded.dpp_bank_mask; - operand.dpp_fetch_inactive = decoded.dpp_fetch_inactive; - operand.dpp_bound_ctrl = decoded.dpp_bound_ctrl; - operand.dpp = decoded.dpp; -} - -bool LowerRegisterOperand(const Decoder::Operand& decoded, Operand& operand, std::string* error) { - operand = {}; - CopyOperandModifiers(decoded, operand); - - switch (decoded.kind) { - case Decoder::OperandKind::Sgpr: - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Scalar; - operand.reg.index = decoded.reg; - return true; - case Decoder::OperandKind::Vgpr: - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Vector; - operand.reg.index = decoded.reg; - return true; - case Decoder::OperandKind::VccLo: - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Vcc; - operand.reg.index = 0; - return true; - case Decoder::OperandKind::VccHi: - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Vcc; - operand.reg.index = 1; - return true; - case Decoder::OperandKind::ExecLo: - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Exec; - operand.reg.index = 0; - return true; - case Decoder::OperandKind::ExecHi: - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Exec; - operand.reg.index = 1; - return true; - case Decoder::OperandKind::Scc: - case Decoder::OperandKind::VccZ: - case Decoder::OperandKind::ExecZ: - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Scc; - operand.reg.index = static_cast(decoded.kind); - return true; - case Decoder::OperandKind::M0: - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::M0; - operand.reg.index = 0; - return true; - case Decoder::OperandKind::Null: operand.kind = OperandKind::Null; return true; - default: SetError(error, "decoded operand cannot be used as an IR register"); return false; - } -} - -bool LowerSourceOperand(const Decoder::Operand& decoded, Operand& operand, std::string* error) { - switch (decoded.kind) { - case Decoder::OperandKind::LiteralConstant: - case Decoder::OperandKind::IntegerInlineConstant: - case Decoder::OperandKind::FloatInlineConstant: - operand = {}; - CopyOperandModifiers(decoded, operand); - operand.kind = OperandKind::ImmediateU32; - operand.imm = decoded.value; - operand.float_inline = decoded.kind == Decoder::OperandKind::FloatInlineConstant; - operand.sext_64 = decoded.kind == Decoder::OperandKind::IntegerInlineConstant && - decoded.signed_val < 0; - return true; - case Decoder::OperandKind::Null: - case Decoder::OperandKind::PopsExitingWaveId: - operand = {}; - CopyOperandModifiers(decoded, operand); - operand.kind = OperandKind::ImmediateU32; - operand.imm = 0; - return true; - default: return LowerRegisterOperand(decoded, operand, error); - } -} - -void ApplyDppDestinationMask(const Decoder::Instruction& decoded, Operand& dst) { - if (dst.kind != OperandKind::Register || dst.reg.file != RegisterFile::Vector) { - return; - } - - const Decoder::Operand* sources[] = {&decoded.src0, &decoded.src1, &decoded.src2}; - for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) { - if (!sources[i]->dpp) { - continue; - } - dst.dpp = true; - dst.dpp_ctrl = sources[i]->dpp_ctrl; - dst.dpp_row_mask = sources[i]->dpp_row_mask; - dst.dpp_bank_mask = sources[i]->dpp_bank_mask; - dst.dpp_fetch_inactive = sources[i]->dpp_fetch_inactive; - dst.dpp_bound_ctrl = sources[i]->dpp_bound_ctrl; - return; - } -} - -uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) { - switch (operand.kind) { - case Decoder::OperandKind::Sgpr: return operand.reg / 4u; - case Decoder::OperandKind::Vgpr: return operand.reg; - case Decoder::OperandKind::IntegerInlineConstant: - case Decoder::OperandKind::LiteralConstant: return operand.value; - default: return 0; - } -} - -MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind) { - MemoryInfo mem; - mem.kind = kind; - mem.offset = decoded.offset; - mem.secondary_offset = decoded.secondary_offset; - mem.dmask = decoded.dmask; - mem.data_dwords = decoded.data_dwords; - mem.data_bits = decoded.data_bits; - mem.component_count = - decoded.data_components != 0u ? decoded.data_components : decoded.data_dwords; - mem.data_format = decoded.data_format; - mem.number_format = decoded.number_format; - mem.image_sample_flags = decoded.image_sample_flags; - mem.image_dimension = decoded.image_dimension; - mem.image_address_components = decoded.image_address_components; - mem.image_nsa_dwords = decoded.image_nsa_dwords; - for (uint32_t i = 0; i < Decoder::MaxImageNsaAddressComponents; i++) { - mem.image_nsa_addr[i] = decoded.image_nsa_addr[i]; - } - mem.memory_segment = decoded.memory_segment; - mem.address_is_full = - kind == ResourceKind::Flat || - (kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr); - mem.data_signed = decoded.data_signed; - mem.typed = decoded.typed; - mem.formatted = decoded.formatted; - mem.image_has_mip = decoded.opcode == Decoder::Opcode::IMAGE_LOAD_MIP || - decoded.opcode == Decoder::Opcode::IMAGE_STORE_MIP; - mem.image_r128 = decoded.image_r128; - mem.glc = decoded.glc; - mem.slc = decoded.slc; - mem.idxen = decoded.idxen; - mem.offen = decoded.offen; - mem.resource = ResourceIndexFromOperand(decoded.src1); - mem.sampler = ResourceIndexFromOperand(decoded.src2); - if (kind == ResourceKind::Lds || kind == ResourceKind::Gds || kind == ResourceKind::Scratch) { - mem.resource = 0; - mem.sampler = 0; - } - return mem; -} - -constexpr uint32_t SCALAR_DST_VCC_LO = 106u; -constexpr uint32_t SCALAR_DST_VCC_HI = 107u; -constexpr uint32_t SCALAR_DST_EXEC_LO = 126u; -constexpr uint32_t SCALAR_DST_EXEC_HI = 127u; - -uint32_t RawScalarLoadBase(const Decoder::Operand& operand) { - if (operand.kind == Decoder::OperandKind::Sgpr) { - return operand.reg; - } - return operand.kind == Decoder::OperandKind::VccLo ? SCALAR_DST_VCC_LO : 0u; -} - -void ClearRegisterOffsetModifiers(Decoder::Operand& operand) { - operand.sdwa_sel = 6; - operand.sdwa_dst_unused = 2; - operand.omod = 0; - operand.sdwa_sext = false; - operand.op_sel = false; - operand.op_sel_hi = false; - operand.negate = false; - operand.negate_hi = false; - operand.absolute = false; - operand.dpp_ctrl = 0; - operand.dpp_row_mask = 0xf; - operand.dpp_bank_mask = 0xf; - operand.dpp_fetch_inactive = false; - operand.dpp_bound_ctrl = false; - operand.dpp = false; -} - -bool TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code) { - switch (operand.kind) { - case Decoder::OperandKind::Sgpr: code = operand.reg; return true; - case Decoder::OperandKind::VccLo: code = SCALAR_DST_VCC_LO; return true; - case Decoder::OperandKind::VccHi: code = SCALAR_DST_VCC_HI; return true; - case Decoder::OperandKind::ExecLo: code = SCALAR_DST_EXEC_LO; return true; - case Decoder::OperandKind::ExecHi: code = SCALAR_DST_EXEC_HI; return true; - default: return false; - } -} - -bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index, - Decoder::Operand& decoded) { - // Wide scalar destinations advance through the encoded SGPR destination space, where - // aliases such as VCC and EXEC sit beside ordinary SGPRs. Re-decode after adding the - // dword index so s_buffer_load_dwordx2 vcc_lo writes vcc_lo, then vcc_hi. - uint32_t code = 0; - return TryGetEncodedScalarCode(operand, code) && - Decoder::DecodeScalarDestination(code + index, 0, decoded, nullptr); -} - -Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index) { - if (index == 0) { - return operand; - } - auto ret = operand; - ClearRegisterOffsetModifiers(ret); - switch (ret.kind) { - case Decoder::OperandKind::Vgpr: ret.reg += index; break; - case Decoder::OperandKind::Sgpr: - case Decoder::OperandKind::VccLo: - case Decoder::OperandKind::VccHi: - case Decoder::OperandKind::ExecLo: - case Decoder::OperandKind::ExecHi: { - Decoder::Operand decoded; - if (TryOffsetScalarDestination(ret, index, decoded)) { - ret = decoded; - } - break; - } - default: break; - } - return ret; -} - -MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, - uint32_t dword_index) { - auto mem = MemoryInfoFromDecoded(decoded, kind); - mem.offset += dword_index * 4u; - mem.data_dwords = 1; - mem.component_index = dword_index; - mem.component_count = decoded.data_dwords; - return mem; -} - -MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, - uint32_t byte_offset) { - auto mem = MemoryInfoFromDecoded(decoded, kind); - mem.offset = byte_offset; - mem.secondary_offset = 0; - mem.data_dwords = 1; - mem.component_index = 0; - mem.component_count = 1; - return mem; -} - -ResourceKind FlatSegmentResourceKind(uint32_t segment) { - switch (segment) { - case 1u: return ResourceKind::Scratch; - case 2u: return ResourceKind::Global; - default: return ResourceKind::Flat; - } -} - -const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index) { - switch (index) { - case 0: return decoded.src0; - case 1: return decoded.src1; - case 2: return decoded.src2; - default: return decoded.src3; - } -} - -Decoder::Operand M0Operand() { - Decoder::Operand operand; - operand.kind = Decoder::OperandKind::M0; - return operand; -} - -namespace { - -bool LowerScalarSelect(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - Decoder::Operand scc; - scc.kind = Decoder::OperandKind::Scc; - - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::SelectU32; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerRegisterOperand(scc, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error) || - !LowerSourceOperand(decoded.src1, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerScalarSelect64(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Decoder::Operand scc; - scc.kind = Decoder::OperandKind::Scc; - - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::SelectU64; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerRegisterOperand(scc, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error) || - !LowerSourceOperand(decoded.src1, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -uint32_t ScalarShiftLeftAddAmount(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::S_LSHL1_ADD_U32: return 1; - case Decoder::Opcode::S_LSHL2_ADD_U32: return 2; - case Decoder::Opcode::S_LSHL3_ADD_U32: return 3; - case Decoder::Opcode::S_LSHL4_ADD_U32: return 4; - default: return 0; - } -} - -bool LowerScalarShiftLeftAdd(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - const auto shift = ScalarShiftLeftAddAmount(decoded.opcode); - if (shift == 0) { - SetError(error, "internal IR error: unknown scalar shift-left-add opcode"); - return false; - } - - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::ScalarShiftLeftAddCarryU32; - inst.src_count = 3; - inst.src[1].kind = OperandKind::ImmediateU32; - inst.src[1].imm = shift; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -Opcode ScalarMinMaxSccCompareOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::S_MIN_I32: return Opcode::CompareLtI32; - case Decoder::Opcode::S_MAX_I32: return Opcode::CompareGtI32; - case Decoder::Opcode::S_MIN_U32: return Opcode::CompareLtU32; - case Decoder::Opcode::S_MAX_U32: return Opcode::CompareGtU32; - default: return Opcode::CompareFalse; - } -} - -bool IsScalarMinMaxOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::S_MIN_I32: - case Decoder::Opcode::S_MAX_I32: - case Decoder::Opcode::S_MIN_U32: - case Decoder::Opcode::S_MAX_U32: return true; - default: return false; - } -} - -bool ResolveIrOpcode(Decoder::Opcode decoded, Opcode& lowered, std::string* error) { - const auto opcode = LookupIrOpcode(decoded); - if (!opcode.has_value()) { - if (error != nullptr) { - *error = fmt::format("decoded opcode has no IR lowering: {}", - magic_enum::enum_name(decoded)); - } - return false; - } - lowered = *opcode; - return true; -} - -bool LowerScalarMinMaxWithScc(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction result; - result.pc = decoded.pc; - result.src_count = 2; - if (!ResolveIrOpcode(decoded.opcode, result.op, error) || - !LowerRegisterOperand(decoded.dst, result.dst, error) || - !LowerSourceOperand(decoded.src0, result.src[0], error) || - !LowerSourceOperand(decoded.src1, result.src[1], error)) { - return false; - } - block.instructions.push_back(result); - - Instruction scc; - scc.pc = decoded.pc; - scc.op = ScalarMinMaxSccCompareOpcode(decoded.opcode); - scc.dst = MakeSccOperand(); - scc.src_count = 2; - if (!LowerSourceOperand(decoded.src0, scc.src[0], error) || - !LowerSourceOperand(decoded.src1, scc.src[1], error)) { - return false; - } - block.instructions.push_back(scc); - return true; -} - -bool LowerScalarAddCarry(const Decoder::Instruction& decoded, BasicBlock& block, bool carry_in, - std::string* error) { - Decoder::Operand scc; - scc.kind = Decoder::OperandKind::Scc; - - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::ScalarAddCarryU32; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[1], error)) { - return false; - } - if (carry_in) { - if (!LowerRegisterOperand(scc, inst.src[2], error)) { - return false; - } - } else { - inst.src[2].kind = OperandKind::ImmediateU32; - inst.src[2].imm = 0; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerScalarSubBorrowCarry(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Decoder::Operand scc; - scc.kind = Decoder::OperandKind::Scc; - - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::ScalarSubBorrowCarryU32; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[1], error) || - !LowerRegisterOperand(scc, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerScalarBinaryScc(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = op; - inst.src_count = 2; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerScalarBitset0B32(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::BitClearU32; - inst.src_count = 2; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.dst, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerScalarBitset1B32(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::BitSetU32; - inst.src_count = 2; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.dst, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorCndmask(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Decoder::Operand vcc; - vcc.kind = Decoder::OperandKind::VccLo; - const auto& mask = decoded.src_count >= 3 ? decoded.src2 : vcc; - - Instruction inst; - inst.pc = decoded.pc; - inst.op = - decoded.src0.negate || decoded.src0.absolute || decoded.src1.negate || decoded.src1.absolute - ? Opcode::SelectMaskF32Bits - : Opcode::SelectMaskU32; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(mask, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[1], error) || - !LowerSourceOperand(decoded.src0, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorMoveB32(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = decoded.src0.negate || decoded.src0.absolute ? Opcode::MoveF32Bits : Opcode::MoveU32; - inst.src_count = 1; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error)) { - return false; - } - ApplyDppDestinationMask(decoded, inst.dst); - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorMoveRelSource(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - if (decoded.dst.kind != Decoder::OperandKind::Vgpr || - decoded.src0.kind != Decoder::OperandKind::Vgpr) { - SetError(error, "V_MOVRELS_B32 requires VGPR source and destination"); - return false; - } - if (decoded.dst.sdwa_sel != 6u || decoded.dst.omod != 0u || decoded.dst.clamp || - decoded.src0.sdwa_sel != 6u || decoded.src0.sdwa_sext || decoded.src0.negate || - decoded.src0.absolute || decoded.src0.dpp) { - SetError(error, "V_MOVRELS_B32 modifiers are not implemented"); - return false; - } - - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::MoveRelSourceU32; - inst.src_count = 2; - const auto m0 = M0Operand(); - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(m0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorMoveRelDestination(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - if (decoded.dst.kind != Decoder::OperandKind::Vgpr) { - SetError(error, "V_MOVRELD_B32 requires VGPR destination"); - return false; - } - if (decoded.dst.sdwa_sel != 6u || decoded.dst.omod != 0u || decoded.dst.clamp || - decoded.src0.sdwa_sel != 6u || decoded.src0.sdwa_sext || decoded.src0.negate || - decoded.src0.absolute || decoded.src0.dpp) { - SetError(error, "V_MOVRELD_B32 modifiers are not implemented"); - return false; - } - - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::MoveRelDestU32; - inst.src_count = 2; - const auto m0 = M0Operand(); - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(m0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorAddCarry(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::IAddCarryU32; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerRegisterOperand(decoded.dst2, inst.dst2, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[1], error) || - !LowerSourceOperand(decoded.src2, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorSubrevBorrowCarry(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::ISubBorrowCarryU32; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerRegisterOperand(decoded.dst2, inst.dst2, error) || - !LowerSourceOperand(decoded.src1, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error) || - !LowerSourceOperand(decoded.src2, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorCarryOut(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = - decoded.opcode == Decoder::Opcode::V_ADD_I32 ? Opcode::IAddCarryU32 : Opcode::ISubBorrowU32; - inst.src_count = decoded.opcode == Decoder::Opcode::V_ADD_I32 ? 3u : 2u; - const auto& src0 = - decoded.opcode == Decoder::Opcode::V_SUBREV_I32 ? decoded.src1 : decoded.src0; - const auto& src1 = - decoded.opcode == Decoder::Opcode::V_SUBREV_I32 ? decoded.src0 : decoded.src1; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerRegisterOperand(decoded.dst2, inst.dst2, error) || - !LowerSourceOperand(src0, inst.src[0], error) || - !LowerSourceOperand(src1, inst.src[1], error)) { - return false; - } - if (decoded.opcode == Decoder::Opcode::V_ADD_I32) { - inst.src[2] = MakeImmediateU32(0); - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorMadU64U32(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::UMadU64U32; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerRegisterOperand(decoded.dst2, inst.dst2, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[1], error) || - !LowerSourceOperand(decoded.src2, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorMacF32(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::FMadF32; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[1], error) || - !LowerSourceOperand(decoded.dst, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorPkFmacF16(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::PackedFmaF16; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[1], error) || - !LowerSourceOperand(decoded.dst, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorFmacF16(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::FmaF16; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[1], error) || - !LowerSourceOperand(decoded.dst, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorDot2cF32F16(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::Dot2AccF32F16; - inst.src_count = 3; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error) || - !LowerSourceOperand(decoded.src1, inst.src[1], error)) { - return false; - } - inst.src[2] = inst.dst; - block.instructions.push_back(inst); - return true; -} - -bool LowerVectorByteConvert(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - const auto byte_index = VectorByteConvertIndex(decoded.opcode); - if (byte_index > 3u) { - SetError(error, "internal IR error: unknown vector byte convert opcode"); - return false; - } - - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::ConvertByteU32ToF32; - inst.src_count = 2; - inst.src[1].kind = OperandKind::ImmediateU32; - inst.src[1].imm = byte_index; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerVInterpP1F32(const Decoder::Instruction& decoded, BasicBlock& block) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::ControlNop; - block.instructions.push_back(inst); - return true; -} - -bool LowerVInterpLoadF32(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - if (decoded.opcode == Decoder::Opcode::V_INTERP_MOV_F32 && decoded.src0.value >= 3u) { - if (error != nullptr) { - *error = fmt::format("v_interp_mov_f32 mode {} is reserved at pc 0x{:08x}", - decoded.src0.value, decoded.pc); - } - return false; - } - - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::LoadInputF32; - inst.input_info.attr = decoded.src1.value; - inst.input_info.chan = decoded.src2.value; - if (decoded.opcode == Decoder::Opcode::V_INTERP_MOV_F32) { - inst.input_info.interpolation_mode = decoded.src0.value; - } - if (!LowerRegisterOperand(decoded.dst, inst.dst, error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -ExportTargetKind ExportTargetKindFromTarget(uint32_t target, uint32_t& index) { - index = 0; - switch (target) { - case 0x08u: return ExportTargetKind::MrtZ; - case 0x09u: return ExportTargetKind::Null; - case 0x14u: return ExportTargetKind::Primitive; - default: break; - } - if (target <= 0x07u) { - index = target; - return ExportTargetKind::Mrt; - } - if (target >= 0x0cu && target <= 0x0fu) { - index = target - 0x0cu; - return ExportTargetKind::Position; - } - if (target >= 0x20u && target <= 0x3fu) { - index = target - 0x20u; - return ExportTargetKind::Parameter; - } - return ExportTargetKind::Unknown; -} - -bool LowerExportInstruction(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - uint32_t index = 0; - const auto kind = ExportTargetKindFromTarget(decoded.exp.target, index); - if (kind == ExportTargetKind::Unknown) { - if (error != nullptr) { - *error = fmt::format("unsupported EXP target 0x{:02x} at pc 0x{:08x}", - decoded.exp.target, decoded.pc); - } - return false; - } - - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::Export; - inst.dst.kind = OperandKind::Null; - inst.src_count = decoded.src_count; - inst.export_info.kind = kind; - inst.export_info.target = decoded.exp.target; - inst.export_info.index = index; - inst.export_info.en = decoded.exp.en; - inst.export_info.done = decoded.exp.done; - inst.export_info.compr = decoded.exp.compr; - inst.export_info.vm = decoded.exp.vm; - - const Decoder::Operand* sources[] = {&decoded.src0, &decoded.src1, &decoded.src2, - &decoded.src3}; - for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) { - if (!LowerSourceOperand(*sources[i], inst.src[i], error)) { - return false; - } - } - - block.instructions.push_back(inst); - return true; -} - -bool LowerControlMarker(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op, - bool has_imm, std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = op; - inst.dst.kind = OperandKind::Null; - inst.src_count = has_imm ? 1u : 0u; - if (has_imm && !LowerSourceOperand(decoded.src0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerControlInstruction(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - switch (decoded.opcode) { - case Decoder::Opcode::S_NOP: - return LowerControlMarker(decoded, block, Opcode::ControlNop, true, error); - case Decoder::Opcode::S_WAITCNT: - case Decoder::Opcode::S_WAITCNT_DEPCTR: - return LowerControlMarker(decoded, block, Opcode::Waitcnt, true, error); - case Decoder::Opcode::S_BARRIER: - return LowerControlMarker(decoded, block, Opcode::Barrier, false, error); - case Decoder::Opcode::S_SENDMSG: - return LowerControlMarker(decoded, block, Opcode::Sendmsg, true, error); - case Decoder::Opcode::S_SETREG_B32: - case Decoder::Opcode::S_SLEEP: - return LowerControlMarker(decoded, block, Opcode::ControlNop, true, error); - case Decoder::Opcode::S_TRAP: - // RDNA2 converts S_TRAP to a NOP when the unmodeled trap state is disabled. - return LowerControlMarker(decoded, block, Opcode::ControlNop, true, error); - case Decoder::Opcode::S_TTRACEDATA: - return LowerControlMarker(decoded, block, Opcode::TtraceData, true, error); - case Decoder::Opcode::S_INST_PREFETCH: - return LowerControlMarker(decoded, block, Opcode::InstPrefetch, true, error); - default: - if (error != nullptr) { - *error = fmt::format("control opcode has no specialized IR lowering: {}", - magic_enum::enum_name(decoded.opcode)); - } - return false; - } -} - -bool LowerScalarGetpcB64(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - for (uint32_t i = 0; i < 2u; i++) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::MoveU32; - inst.src[0] = - i == 0u ? MakePcRelativeU32(decoded.pc + 4u) : MakePcRelativeHighU32(decoded.pc + 4u); - inst.src_count = 1; - if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error)) { - return false; - } - block.instructions.push_back(inst); - } - return true; -} - -bool IsScalarSaveexecOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::S_AND_SAVEEXEC_B32: - case Decoder::Opcode::S_ANDN1_SAVEEXEC_B32: - case Decoder::Opcode::S_AND_SAVEEXEC_B64: - case Decoder::Opcode::S_ORN2_SAVEEXEC_B64: - case Decoder::Opcode::S_ANDN1_SAVEEXEC_B64: return true; - default: return false; - } -} - -SaveexecMode ScalarSaveexecMode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::S_ORN2_SAVEEXEC_B64: return SaveexecMode::Orn2; - case Decoder::Opcode::S_ANDN1_SAVEEXEC_B32: - case Decoder::Opcode::S_ANDN1_SAVEEXEC_B64: return SaveexecMode::Andn1; - default: return SaveexecMode::And; - } -} - -bool LowerScalarSaveexec(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = (decoded.opcode == Decoder::Opcode::S_AND_SAVEEXEC_B32 || - decoded.opcode == Decoder::Opcode::S_ANDN1_SAVEEXEC_B32) - ? Opcode::SaveexecB32 - : Opcode::SaveexecB64; - inst.saveexec_mode = ScalarSaveexecMode(decoded.opcode); - inst.src_count = 1; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool IsControlOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::S_NOP: - case Decoder::Opcode::S_WAITCNT: - case Decoder::Opcode::S_WAITCNT_DEPCTR: - case Decoder::Opcode::S_BARRIER: - case Decoder::Opcode::S_SENDMSG: - case Decoder::Opcode::S_SETREG_B32: - case Decoder::Opcode::S_SLEEP: - case Decoder::Opcode::S_TRAP: - case Decoder::Opcode::S_TTRACEDATA: - case Decoder::Opcode::S_INST_PREFETCH: return true; - default: return false; - } -} - -bool IsTerminatorOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::S_ENDPGM: - case Decoder::Opcode::S_BRANCH: - case Decoder::Opcode::S_CBRANCH_SCC0: - case Decoder::Opcode::S_CBRANCH_SCC1: - case Decoder::Opcode::S_CBRANCH_VCCZ: - case Decoder::Opcode::S_CBRANCH_VCCNZ: - case Decoder::Opcode::S_CBRANCH_EXECZ: - case Decoder::Opcode::S_CBRANCH_EXECNZ: - case Decoder::Opcode::S_SETPC_B64: return true; - default: return false; - } -} - -bool LowerImplemented(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error); - -bool IsMemoryFamily(Decoder::Family family) { - switch (family) { - case Decoder::Family::SMEM: - case Decoder::Family::MUBUF: - case Decoder::Family::MTBUF: - case Decoder::Family::FLAT: - case Decoder::Family::DS: - case Decoder::Family::MIMG: return true; - default: return false; - } -} - -bool LowerDecodedInstruction(const Decoder::Instruction& inst, BasicBlock& block, - std::string* error) { - switch (inst.opcode) { - case Decoder::Opcode::UNSUPPORTED: - if (error != nullptr) { - *error = fmt::format("unsupported decoded instruction: {}", - Decoder::InstructionToString(inst).c_str()); - } - return false; - case Decoder::Opcode::S_GETPC_B64: return LowerScalarGetpcB64(inst, block, error); - case Decoder::Opcode::V_NOP: return true; - case Decoder::Opcode::EXP: return LowerExportInstruction(inst, block, error); - case Decoder::Opcode::S_CSELECT_B32: return LowerScalarSelect(inst, block, error); - case Decoder::Opcode::S_CSELECT_B64: return LowerScalarSelect64(inst, block, error); - case Decoder::Opcode::S_ADD_U32: return LowerScalarAddCarry(inst, block, false, error); - case Decoder::Opcode::S_ADDC_U32: return LowerScalarAddCarry(inst, block, true, error); - case Decoder::Opcode::S_SUBB_U32: return LowerScalarSubBorrowCarry(inst, block, error); - case Decoder::Opcode::S_SUB_U32: - return LowerScalarBinaryScc(inst, block, Opcode::ScalarSubBorrowU32, error); - case Decoder::Opcode::S_BITSET0_B32: return LowerScalarBitset0B32(inst, block, error); - case Decoder::Opcode::S_BITSET1_B32: return LowerScalarBitset1B32(inst, block, error); - case Decoder::Opcode::S_ADD_I32: - return LowerScalarBinaryScc(inst, block, Opcode::ScalarSignedAddOverflowI32, error); - case Decoder::Opcode::S_SUB_I32: - return LowerScalarBinaryScc(inst, block, Opcode::ScalarSignedSubOverflowI32, error); - case Decoder::Opcode::V_CNDMASK_B32: return LowerVectorCndmask(inst, block, error); - case Decoder::Opcode::V_MOV_B32: return LowerVectorMoveB32(inst, block, error); - case Decoder::Opcode::V_MOVRELD_B32: - return LowerVectorMoveRelDestination(inst, block, error); - case Decoder::Opcode::V_MOVRELS_B32: return LowerVectorMoveRelSource(inst, block, error); - case Decoder::Opcode::V_ADDC_U32: return LowerVectorAddCarry(inst, block, error); - case Decoder::Opcode::V_SUBREV_CO_CI_U32: - return LowerVectorSubrevBorrowCarry(inst, block, error); - case Decoder::Opcode::V_MAD_U64_U32: return LowerVectorMadU64U32(inst, block, error); - case Decoder::Opcode::V_MAC_F32: return LowerVectorMacF32(inst, block, error); - case Decoder::Opcode::V_PK_FMAC_F16: return LowerVectorPkFmacF16(inst, block, error); - case Decoder::Opcode::V_FMAC_F16: return LowerVectorFmacF16(inst, block, error); - case Decoder::Opcode::V_DOT2C_F32_F16: return LowerVectorDot2cF32F16(inst, block, error); - case Decoder::Opcode::V_INTERP_P1_F32: return LowerVInterpP1F32(inst, block); - case Decoder::Opcode::V_INTERP_P2_F32: - case Decoder::Opcode::V_INTERP_MOV_F32: return LowerVInterpLoadF32(inst, block, error); - default: break; - } - - if (IsTerminatorOpcode(inst.opcode)) { - return true; - } - if (IsControlOpcode(inst.opcode)) { - return LowerControlInstruction(inst, block, error); - } - if (IsScalarSaveexecOpcode(inst.opcode)) { - return LowerScalarSaveexec(inst, block, error); - } - if (IsMemoryOpcode(inst.opcode)) { - return LowerMemoryInstruction(inst, block, error); - } - // Memory instructions require specialized lowering to populate MemoryInfo. Never let a - // newly decoded memory-family opcode fall through to generic register-only IR. - if (IsMemoryFamily(inst.family)) { - if (error != nullptr) { - *error = fmt::format("decoded memory-family opcode has no specialized IR lowering: {}", - magic_enum::enum_name(inst.opcode)); - } - return false; - } - if (ScalarShiftLeftAddAmount(inst.opcode) != 0) { - return LowerScalarShiftLeftAdd(inst, block, error); - } - if (IsScalarMinMaxOpcode(inst.opcode)) { - return LowerScalarMinMaxWithScc(inst, block, error); - } - if (IsVectorCarryOutOpcode(inst.opcode)) { - return LowerVectorCarryOut(inst, block, error); - } - if (VectorByteConvertIndex(inst.opcode) <= 3u) { - return LowerVectorByteConvert(inst, block, error); - } - return LowerImplemented(inst, block, error); -} - -bool LowerImplemented(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.src_count = decoded.src_count; - - if (!ResolveIrOpcode(decoded.opcode, inst.op, error) || - !LowerRegisterOperand(decoded.dst, inst.dst, error)) { - return false; - } - ApplyDppDestinationMask(decoded, inst.dst); - for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) { - const auto src_index = - (IsReversedBinary(decoded.opcode) && decoded.src_count == 2u) ? 1u - i : i; - if (!LowerSourceOperand(DecodedSourceAt(decoded, src_index), inst.src[i], error)) { - return false; - } - } - - block.instructions.push_back(inst); - return AppendScalarResultSccNonZero(decoded, block, error); -} - -} // namespace - -bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage, - uint32_t wave_size, Program& program, std::string* error) { - if (wave_size != 32u && wave_size != 64u) { - EXIT("invalid shader wave size: %u\n", wave_size); - } - - program = {}; - program.stage = stage; - program.wave_size = wave_size; - program.cfg_failure_kind = cfg.failure_kind; - program.fallback_reason = cfg.unsupported_reason; - program.blocks.clear(); - - if (cfg.blocks.empty()) { - SetError(error, "cannot lower empty CFG"); - return false; - } - - for (const auto& cfg_block: cfg.blocks) { - BasicBlock block; - block.id = cfg_block.id; - block.start_pc = cfg_block.start_pc; - block.end_pc = cfg_block.end_pc; - block.inst_begin = cfg_block.inst_begin; - block.inst_end = cfg_block.inst_end; - block.predecessors = cfg_block.predecessors; - block.successors = cfg_block.successors; - block.terminator = cfg_block.terminator; - - for (uint32_t i = cfg_block.inst_begin; i < cfg_block.inst_end; i++) { - if (i >= decoded.instructions.size()) { - SetError(error, "CFG block references instruction outside decoded program"); - return false; - } - if (std::find(cfg.code_table_load_pcs.begin(), cfg.code_table_load_pcs.end(), - decoded.instructions[i].pc) != cfg.code_table_load_pcs.end()) { - continue; - } - if (!LowerDecodedInstruction(decoded.instructions[i], block, error)) { - return false; - } - } - - program.blocks.push_back(std::move(block)); - } - - return true; -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 8f2312766..fb6f63968 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -6,7 +6,7 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h" #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" -#include "graphics/shader/recompiler/ir/opcodes/Opcodes.h" +#include "graphics/shader/recompiler/ir/Block.h" #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h" #include "graphics/shader/shader.h" @@ -19,47 +19,6 @@ namespace Libs::Graphics::ShaderRecompiler::IR { -struct ValueProgram; - -enum class RegisterFile { Scalar, Vector, Vcc, Exec, Scc, M0 }; - -struct Register { - RegisterFile file = RegisterFile::Scalar; - uint32_t index = 0; - - bool operator==(const Register& other) const { - return file == other.file && index == other.index; - } -}; - -enum class OperandKind { Register, ImmediateU32, PcRelativeU32, PcRelativeHighU32, Null }; - -struct Operand { - OperandKind kind = OperandKind::Null; - Register reg; - uint32_t imm = 0; - bool float_inline = false; - bool sext_64 = false; - uint32_t sdwa_sel = 6; - uint32_t sdwa_dst_unused = 2; - uint32_t omod = 0; - bool sdwa_sext = false; - bool op_sel = false; - bool op_sel_hi = false; - bool negate = false; - bool negate_hi = false; - bool absolute = false; - bool clamp = false; - uint32_t dpp_ctrl = 0; - uint32_t dpp_row_mask = 0xf; - uint32_t dpp_bank_mask = 0xf; - bool dpp_fetch_inactive = false; - bool dpp_bound_ctrl = false; - bool dpp = false; - - bool operator==(const Operand& other) const = default; -}; - enum class ResourceKind { None, ScalarBuffer, @@ -144,44 +103,6 @@ struct ExportInfo { bool operator==(const ExportInfo& other) const = default; }; -struct InputInfo { - uint32_t attr = 0; - uint32_t chan = 0; - uint32_t component_count = 1; - uint32_t interpolation_mode = 3; - - bool operator==(const InputInfo& other) const = default; -}; - -enum class SaveexecMode { And, Orn2, Andn1 }; - -struct Instruction { - uint32_t pc = 0; - Opcode op = Opcode::MoveU32; - Operand dst; - Operand dst2; - Operand src[4]; - uint32_t src_count = 0; - MemoryInfo memory; - ExportInfo export_info; - InputInfo input_info; - SaveexecMode saveexec_mode = SaveexecMode::And; - - bool operator==(const Instruction& other) const = default; -}; - -struct BasicBlock { - uint32_t id = 0; - uint32_t start_pc = 0; - uint32_t end_pc = 0; - uint32_t inst_begin = 0; - uint32_t inst_end = 0; - std::vector predecessors; - std::vector successors; - CFG::Terminator terminator; - std::vector instructions; -}; - struct DescriptorValue { std::array dwords = {}; uint32_t dword_count = 0; @@ -522,6 +443,7 @@ struct ShaderInfo { std::vector sampled_pairs; std::vector inputs; std::vector outputs; + std::array vertex_fetch_components {}; int32_t vertex_offset_sgpr = -1; bool has_bitwise_xor = false; @@ -539,7 +461,50 @@ struct SpirvRequirements { bool pixel_valid_mask = false; }; +struct BlockInfo { + uint32_t id = 0; + uint32_t start_pc = 0; + uint32_t end_pc = 0; + CFG::Terminator terminator; + Value condition; + Value indirect_target; +}; + +struct DescriptorSource { + struct IndirectImage { + uint32_t material_source = 0; + uint32_t heap_source = 0; + uint32_t selector_stride = 0; + uint32_t selector_offset = 0; + uint32_t key_arg = 0; + + bool operator==(const IndirectImage& other) const = default; + }; + + std::array dwords {}; + uint32_t dword_count = 0; + std::optional indirect_image; + + bool operator==(const DescriptorSource& other) const = default; +}; + +struct SrtRead { + Value value; + uint32_t flat_offset = 0; + uint32_t use_pc = 0; + + bool operator==(const SrtRead& other) const = default; +}; + struct Program { + Program() = default; + ~Program(); + + Program(const Program&) = delete; + Program& operator=(const Program&) = delete; + Program(Program&&) noexcept = default; + Program& operator=(Program&& other) noexcept; + ShaderType stage = ShaderType::Unknown; uint64_t shader_hash = 0; uint32_t wave_size = 64; @@ -549,8 +514,17 @@ struct Program { bool dispatcher_fallback = false; CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None; std::string fallback_reason; - std::vector blocks; - std::shared_ptr values; + std::vector> block_storage; + BlockList blocks; + std::vector block_info; + // Decoded MIMG/VMEM metadata carries details such as RDNA2 NSA address registers and + // storage-image swizzles. Typed memory instructions carry a dense index into these shader-local + // tables until those fields are consumed by emission. + std::vector memory_info; + std::vector export_info; + std::vector descriptor_sources; + std::vector srt_reads; + std::vector dynamic_reads; bool srt_plan_complete = false; ShaderInfo info; bool resource_tracking_complete = false; @@ -561,15 +535,13 @@ struct Program { std::optional spirv_requirements; }; -bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage, - uint32_t wave_size, Program& program, std::string* error); - -std::string RegisterToString(Register reg); -std::string OperandToString(const Operand& operand); -std::string ExportTargetKindToString(ExportTargetKind kind); -std::string InstructionToString(const Instruction& inst); std::string ProgramToString(const Program& program); +bool ValidateProgram(const Program& program, bool require_ssa, std::string* error); +void ResolveControlFlowIdentities(Program& program); +bool EquivalentValue(const Program& program, Value left, Value right); +Value ResolveInvariantPhi(const Program& program, Value value); + } // namespace Libs::Graphics::ShaderRecompiler::IR #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERIR_H_ */ diff --git a/src/graphics/shader/recompiler/ir/ShaderIRInternal.h b/src/graphics/shader/recompiler/ir/ShaderIRInternal.h deleted file mode 100644 index 0db895f0c..000000000 --- a/src/graphics/shader/recompiler/ir/ShaderIRInternal.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_ -#define EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_ - -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { - -void SetError(std::string* error, const char* message); - -std::optional LookupIrOpcode(Decoder::Opcode opcode); -bool IsReversedBinary(Decoder::Opcode opcode); -bool IsVectorCarryOutOpcode(Decoder::Opcode opcode); -bool ScalarResultWritesSccNonZero(Decoder::Opcode opcode); -bool ScalarResultIs64Bit(Decoder::Opcode opcode); - -Operand MakeSccOperand(); -Operand MakeImmediateU32(uint32_t value); -Operand MakePcRelativeU32(uint32_t value); -Operand MakePcRelativeHighU32(uint32_t value); -bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error); -uint32_t VectorByteConvertIndex(Decoder::Opcode opcode); -void CopyOperandModifiers(const Decoder::Operand& decoded, Operand& operand); -bool LowerRegisterOperand(const Decoder::Operand& decoded, Operand& operand, std::string* error); -bool LowerSourceOperand(const Decoder::Operand& decoded, Operand& operand, std::string* error); -void ApplyDppDestinationMask(const Decoder::Instruction& decoded, Operand& dst); -uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand); -MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind); -uint32_t RawScalarLoadBase(const Decoder::Operand& operand); -void ClearRegisterOffsetModifiers(Decoder::Operand& operand); -bool TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code); -bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index, - Decoder::Operand& result); -Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index); -MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, - uint32_t dword_index); -MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, - uint32_t byte_offset); -ResourceKind FlatSegmentResourceKind(uint32_t segment); -const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index); -Decoder::Operand M0Operand(); - -bool IsMemoryOpcode(Decoder::Opcode opcode); -bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error); - -} // namespace Libs::Graphics::ShaderRecompiler::IR - -#endif // EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_ diff --git a/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp b/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp deleted file mode 100644 index 6e4afe058..000000000 --- a/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp +++ /dev/null @@ -1,214 +0,0 @@ -#include "common/assert.h" -#include "graphics/shader/recompiler/BufferFormat.h" -#include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/ShaderIRInternal.h" - -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { - -std::string RegisterToString(Register reg) { - switch (reg.file) { - case RegisterFile::Scalar: return fmt::format("s{}", reg.index); - case RegisterFile::Vector: return fmt::format("v{}", reg.index); - case RegisterFile::Vcc: - return reg.index == 0 ? std::string("vcc_lo") : std::string("vcc_hi"); - case RegisterFile::Exec: - return reg.index == 0 ? std::string("exec_lo") : std::string("exec_hi"); - case RegisterFile::Scc: - if (reg.index == static_cast(Decoder::OperandKind::Scc)) { - return "scc"; - } - if (reg.index == static_cast(Decoder::OperandKind::VccZ)) { - return "vccz"; - } - if (reg.index == static_cast(Decoder::OperandKind::ExecZ)) { - return "execz"; - } - return fmt::format("scc_like_{}", reg.index); - case RegisterFile::M0: return "m0"; - default: return "unknown_reg"; - } -} - -std::string OperandToString(const Operand& operand) { - std::string text; - switch (operand.kind) { - case OperandKind::Register: text = RegisterToString(operand.reg); break; - case OperandKind::ImmediateU32: text = fmt::format("0x{:08x}", operand.imm); break; - case OperandKind::PcRelativeU32: text = fmt::format("pc_rel(0x{:08x})", operand.imm); break; - case OperandKind::PcRelativeHighU32: - text = fmt::format("pc_rel_hi(0x{:08x})", operand.imm); - break; - case OperandKind::Null: text = "null"; break; - default: text = "unknown"; break; - } - if (operand.sdwa_sel != 6 || operand.sdwa_sext) { - text += fmt::format(".sdwa(sel={},sext={})", operand.sdwa_sel, operand.sdwa_sext ? 1u : 0u); - } - if (operand.absolute) { - text += ".abs"; - } - if (operand.negate) { - text += ".neg"; - } - if (operand.op_sel || operand.op_sel_hi || operand.negate_hi) { - text += fmt::format(".opsel(lo={},hi={},neghi={})", operand.op_sel ? 1u : 0u, - operand.op_sel_hi ? 1u : 0u, operand.negate_hi ? 1u : 0u); - } - if (operand.omod != 0) { - text += fmt::format(".omod({})", operand.omod); - } - if (operand.clamp) { - text += ".clamp"; - } - if (operand.dpp) { - text += fmt::format(".dpp(ctrl=0x{:x},fi={},bc={})", operand.dpp_ctrl, - operand.dpp_fetch_inactive ? 1u : 0u, operand.dpp_bound_ctrl ? 1u : 0u); - } - return text; -} - -std::string ResourceKindToString(ResourceKind kind) { - switch (kind) { - case ResourceKind::ScalarBuffer: return "scalar_buffer"; - case ResourceKind::ScalarAddress: return "scalar_address"; - case ResourceKind::Buffer: return "buffer"; - case ResourceKind::Flat: return "flat"; - case ResourceKind::Global: return "global"; - case ResourceKind::Scratch: return "scratch"; - case ResourceKind::Lds: return "lds"; - case ResourceKind::Gds: return "gds"; - case ResourceKind::Image: return "image"; - case ResourceKind::ImageUint: return "image_uint"; - case ResourceKind::StorageImage: return "storage_image"; - case ResourceKind::StorageImageUint: return "storage_image_uint"; - case ResourceKind::Sampler: return "sampler"; - default: return "none"; - } -} - -std::string MemoryInfoToString(const MemoryInfo& mem) { - if (mem.kind == ResourceKind::None) { - return ""; - } - std::string text = fmt::format( - " ; {} resource={} sampler={} offset={} offset2={} dmask=0x{:x} data_dwords={} " - "data_bits={} component={}/{} dfmt={} nfmt={} signed={} typed={} formatted={} segment={} " - "glc={} slc={} idxen={} offen={} image_flags=0x{:x} image_dim={} image_addr={} " - "image_mip={} image_r128={}", - ResourceKindToString(mem.kind).c_str(), mem.resource, mem.sampler, mem.offset, - mem.secondary_offset, mem.dmask, mem.data_dwords, mem.data_bits, mem.component_index, - mem.component_count, mem.data_format, mem.number_format, mem.data_signed ? 1u : 0u, - mem.typed ? 1u : 0u, mem.formatted ? 1u : 0u, mem.memory_segment, mem.glc ? 1u : 0u, - mem.slc ? 1u : 0u, mem.idxen ? 1u : 0u, mem.offen ? 1u : 0u, mem.image_sample_flags, - Decoder::ImageDimensionToString(mem.image_dimension), mem.image_address_components, - mem.image_has_mip ? 1u : 0u, mem.image_r128 ? 1u : 0u); - if (mem.image_nsa_dwords != 0) { - text += fmt::format(" image_nsa_dwords={} image_nsa_addr=", mem.image_nsa_dwords); - const auto count = - std::min(mem.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents); - for (uint32_t i = 0; i < count; i++) { - if (i != 0) { - text += ","; - } - text += fmt::format("{}", mem.image_nsa_addr[i]); - } - } - return text; -} - -std::string ExportInfoToString(const ExportInfo& exp) { - return fmt::format(" ; target={} index={} raw=0x{:02x} en=0x{:x} done={} compr={} vm={}", - ExportTargetKindToString(exp.kind).c_str(), exp.index, exp.target, exp.en, - exp.done ? 1u : 0u, exp.compr ? 1u : 0u, exp.vm ? 1u : 0u); -} - -std::string InputInfoToString(Opcode op, const InputInfo& input) { - if (op != Opcode::LoadInputF32) { - return ""; - } - return fmt::format(" ; input_attr={} input_chan={} input_components={}", input.attr, input.chan, - input.component_count); -} - -std::string TerminatorToString(const CFG::Terminator& term) { - return fmt::format( - "term={} cond={} true={} false={} merge={} continue={} loop={} indirect_sgpr={} " - "indirect_selector={} indirect_targets={} selector_values={}", - static_cast(term.kind), CFG::BranchConditionToString(term.condition).c_str(), - term.true_block, term.false_block, term.merge_block, term.continue_block, - term.loop_header ? 1u : 0u, term.indirect_pc_sgpr, term.indirect_selector_code, - static_cast(term.indirect_targets.size()), - static_cast(term.indirect_selector_values.size())); -} - -std::string ExportSourcesToString(const Instruction& inst) { - std::string text; - for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) { - text += i == 0 ? " " : ", "; - text += OperandToString(inst.src[i]); - } - return text; -} - -std::string InstructionToString(const Instruction& inst) { - std::string text = fmt::format("0x{:08x}: ", inst.pc); - if (inst.op == Opcode::Export) { - text += "Export"; - text += ExportSourcesToString(inst); - text += ExportInfoToString(inst.export_info); - return text; - } - text += OpcodeName(inst.op); - text += " "; - text += OperandToString(inst.dst); - if (inst.dst2.kind != OperandKind::Null) { - text += ", "; - text += OperandToString(inst.dst2); - } - for (uint32_t i = 0; i < inst.src_count; i++) { - text += i == 0 ? ", " : ", "; - text += OperandToString(inst.src[i]); - } - text += MemoryInfoToString(inst.memory); - text += InputInfoToString(inst.op, inst.input_info); - return text; -} - -std::string ExportTargetKindToString(ExportTargetKind kind) { - switch (kind) { - case ExportTargetKind::Null: return "null"; - case ExportTargetKind::Position: return "position"; - case ExportTargetKind::Primitive: return "primitive"; - case ExportTargetKind::Parameter: return "parameter"; - case ExportTargetKind::Mrt: return "mrt"; - case ExportTargetKind::MrtZ: return "mrtz"; - default: return "unknown"; - } -} - -std::string ProgramToString(const Program& program) { - std::string text; - text += fmt::format("mode={} fallback={} reason={}\n", - program.dispatcher_fallback ? "dispatcher" : "structured", - CFG::FailureKindToString(program.cfg_failure_kind).c_str(), - program.fallback_reason.c_str()); - for (const auto& block: program.blocks) { - text += fmt::format("block_{} pc=0x{:08x} end=0x{:08x} preds={} succs={} {}\n", block.id, - block.start_pc, block.end_pc, - static_cast(block.predecessors.size()), - static_cast(block.successors.size()), - TerminatorToString(block.terminator).c_str()); - for (const auto& inst: block.instructions) { - text += " "; - text += InstructionToString(inst); - text += "\n"; - } - } - return text; -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp b/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp deleted file mode 100644 index 90f28cc80..000000000 --- a/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp +++ /dev/null @@ -1,812 +0,0 @@ -#include "common/assert.h" -#include "graphics/shader/recompiler/BufferFormat.h" -#include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/ShaderIRInternal.h" - -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { -namespace { - -Opcode BufferLoadOpcode(const Decoder::Instruction& decoded) { - switch (decoded.data_bits) { - case 8u: return decoded.data_signed ? Opcode::BufferLoadSbyte : Opcode::BufferLoadUbyte; - case 16u: return decoded.data_signed ? Opcode::BufferLoadSshort : Opcode::BufferLoadUshort; - default: return Opcode::BufferLoadDword; - } -} - -Opcode BufferStoreOpcode(uint32_t data_bits) { - switch (data_bits) { - case 8u: return Opcode::BufferStoreByte; - case 16u: return Opcode::BufferStoreShort; - default: return Opcode::BufferStoreDword; - } -} - -Opcode DsReadOpcode(const Decoder::Instruction& decoded) { - switch (decoded.data_bits) { - case 8u: return decoded.data_signed ? Opcode::DsReadSbyte : Opcode::DsReadUbyte; - case 16u: return decoded.data_signed ? Opcode::DsReadSshort : Opcode::DsReadUshort; - default: return Opcode::DsReadB32; - } -} - -Opcode FlatLoadOpcode(const Decoder::Instruction& decoded) { - switch (decoded.data_bits) { - case 8u: return decoded.data_signed ? Opcode::FlatLoadSbyte : Opcode::FlatLoadUbyte; - case 16u: return decoded.data_signed ? Opcode::FlatLoadSshort : Opcode::FlatLoadUshort; - default: return Opcode::FlatLoadDword; - } -} - -Opcode FlatStoreOpcode(uint32_t data_bits) { - switch (data_bits) { - case 8u: return Opcode::FlatStoreByte; - case 16u: return Opcode::FlatStoreShort; - default: return Opcode::FlatStoreDword; - } -} - -Opcode DsWriteOpcode(uint32_t data_bits) { - switch (data_bits) { - case 8u: return Opcode::DsWriteByte; - case 16u: return Opcode::DsWriteShort; - default: return Opcode::DsWriteB32; - } -} - -Opcode LoweredImageOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::IMAGE_GET_RESINFO: return Opcode::ImageGetResinfo; - case Decoder::Opcode::IMAGE_GET_LOD: return Opcode::ImageGetLod; - case Decoder::Opcode::IMAGE_LOAD: - case Decoder::Opcode::IMAGE_LOAD_MIP: return Opcode::ImageLoad; - case Decoder::Opcode::IMAGE_GATHER4_LZ: - case Decoder::Opcode::IMAGE_GATHER4_C: - case Decoder::Opcode::IMAGE_GATHER4_C_LZ: - case Decoder::Opcode::IMAGE_GATHER4_LZ_O: - case Decoder::Opcode::IMAGE_GATHER4_C_O: - case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O: - case Decoder::Opcode::IMAGE_GATHER4H: return Opcode::ImageGather4; - default: return Opcode::ImageSample; - } -} - -bool LowerScalarMemoryLoadDword(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op, - std::string* error) { - uint32_t destination_code = 0; - const auto alignment = decoded.data_dwords == 1u ? 1u - : decoded.data_dwords == 2u ? 2u - : 4u; - if (!TryGetEncodedScalarCode(decoded.dst, destination_code) || - destination_code % alignment != 0u || - destination_code + decoded.data_dwords > 108u) { - SetError(error, "scalar-memory destination has an invalid aligned SGPR span"); - return false; - } - uint32_t base_code = 0; - const auto base_alignment = op == Opcode::SLoadDword ? 2u : 4u; - const auto base_width = op == Opcode::SLoadDword ? 2u : 4u; - if (!TryGetEncodedScalarCode(decoded.src0, base_code) || - base_code % base_alignment != 0u || base_code + base_width > 108u) { - SetError(error, "scalar-memory base has an invalid aligned SGPR span"); - return false; - } - Instruction inst; - inst.pc = decoded.pc; - inst.op = op; - inst.src_count = 1; - const auto kind = - op == Opcode::SLoadDword ? ResourceKind::ScalarAddress : ResourceKind::ScalarBuffer; - inst.memory = MemoryInfoFromDecoded(decoded, kind); - // Raw s_load uses a two-SGPR pointer base; s_buffer_load uses a four-SGPR - // descriptor resource index. - inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0) - : ResourceIndexFromOperand(decoded.src0); - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src1, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerBufferAddressSources(const Decoder::Instruction& decoded, Instruction& inst, - uint32_t first_src, std::string* error) { - uint32_t src = first_src; - if (decoded.idxen) { - if (!LowerSourceOperand(decoded.src0, inst.src[src++], error)) { - return false; - } - } - if (decoded.offen) { - const auto offset_source = - decoded.idxen ? OffsetDecodedRegister(decoded.src0, 1) : decoded.src0; - if (!LowerSourceOperand(offset_source, inst.src[src++], error)) { - return false; - } - } - if (!LowerSourceOperand(decoded.src2, inst.src[src++], error)) { - return false; - } - inst.src_count = src; - return true; -} - -bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = BufferLoadOpcode(decoded); - inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer); - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerBufferAddressSources(decoded, inst, 0, error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = BufferStoreOpcode(decoded.data_bits); - inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer); - inst.dst.kind = OperandKind::Null; - if (!LowerSourceOperand(decoded.dst, inst.src[0], error) || - !LowerBufferAddressSources(decoded, inst, 1, error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerBufferAtomicDword(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = op; - inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer); - inst.dst.kind = OperandKind::Null; - if ((decoded.glc && !LowerRegisterOperand(decoded.dst, inst.dst, error)) || - !LowerSourceOperand(decoded.dst, inst.src[0], error) || - !LowerBufferAddressSources(decoded, inst, 1, error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -ResourceKind DsMemoryKind(const Decoder::Instruction& decoded) { - return decoded.gds ? ResourceKind::Gds : ResourceKind::Lds; -} - -bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - const auto ir_op = DsReadOpcode(decoded); - Instruction inst; - inst.pc = decoded.pc; - inst.op = ir_op; - inst.src_count = 1; - inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock& block, uint32_t dwords_per_read, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::DsRead2B32; - inst.src_count = 1; - inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); - inst.memory.data_dwords = dwords_per_read * 2u; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerDsAtomicU32(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op, - bool return_old, std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = op; - inst.src_count = 2; - inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); - inst.dst.kind = return_old ? OperandKind::Register : OperandKind::Null; - if ((return_old && !LowerRegisterOperand(decoded.dst, inst.dst, error)) || - !LowerSourceOperand(decoded.src1, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerDsSwizzleB32(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::DsSwizzleB32; - inst.src_count = 2; - inst.src[1].kind = OperandKind::ImmediateU32; - inst.src[1].imm = decoded.offset & 0xffffu; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerDsFloatMinMaxF32(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = op; - inst.src_count = 2; - inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); - inst.memory.resource = 0; - inst.dst.kind = OperandKind::Null; - if (!LowerSourceOperand(decoded.src1, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error) || - !LowerSourceOperand(decoded.src2, inst.src[2], error)) { - return false; - } - inst.src_count = 3; - block.instructions.push_back(inst); - return true; -} - -bool LowerDsWriteAddtidB32(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::DsWriteAddtidB32; - inst.src_count = 2; - inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); - inst.memory.resource = 0; - inst.dst.kind = OperandKind::Null; - const auto m0 = M0Operand(); - if (!LowerSourceOperand(decoded.src1, inst.src[0], error) || - !LowerSourceOperand(m0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerDsReadAddtidB32(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::DsReadAddtidB32; - inst.src_count = 1; - inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); - inst.memory.resource = 0; - const auto m0 = M0Operand(); - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(m0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerDsAppendConsume(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = op; - inst.src_count = 1; - inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); - const auto m0 = M0Operand(); - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(m0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -Opcode ImageAtomicIrOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::IMAGE_ATOMIC_ADD: return Opcode::AtomicAddU32; - case Decoder::Opcode::IMAGE_ATOMIC_UMIN: return Opcode::AtomicUMinU32; - case Decoder::Opcode::IMAGE_ATOMIC_UMAX: return Opcode::AtomicUMaxU32; - case Decoder::Opcode::IMAGE_ATOMIC_AND: return Opcode::AtomicAndU32; - case Decoder::Opcode::IMAGE_ATOMIC_OR: return Opcode::AtomicOrU32; - case Decoder::Opcode::IMAGE_ATOMIC_XOR: return Opcode::AtomicXorU32; - default: return Opcode::AtomicAddU32; - } -} - -bool LowerImageAtomicU32(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = ImageAtomicIrOpcode(decoded.opcode); - inst.src_count = 2; - inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::StorageImageUint); - inst.memory.sampler = 0; - inst.dst.kind = OperandKind::Null; - if ((decoded.glc && !LowerRegisterOperand(decoded.dst, inst.dst, error)) || - !LowerSourceOperand(decoded.dst, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerFlatLoad(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - const auto kind = FlatSegmentResourceKind(decoded.memory_segment); - const auto ir_op = FlatLoadOpcode(decoded); - const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; - if (decoded.data_bits == 32u && count > 1u) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = ir_op; - inst.src_count = decoded.src_count; - inst.memory = MemoryInfoFromDecoded(decoded, kind); - inst.memory.resource = 0; - if (!LowerRegisterOperand(decoded.dst, inst.dst, error)) { - return false; - } - for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) { - if (!LowerSourceOperand(DecodedSourceAt(decoded, i), inst.src[i], error)) { - return false; - } - } - block.instructions.push_back(inst); - return true; - } - - for (uint32_t dword = 0; dword < count; dword++) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = ir_op; - inst.src_count = decoded.src_count; - inst.memory = OffsetMemoryInfo(decoded, kind, dword); - inst.memory.resource = 0; - if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, dword), inst.dst, error)) { - return false; - } - for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) { - if (!LowerSourceOperand(DecodedSourceAt(decoded, i), inst.src[i], error)) { - return false; - } - } - block.instructions.push_back(inst); - } - return true; -} - -bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - const auto kind = FlatSegmentResourceKind(decoded.memory_segment); - const auto ir_op = FlatStoreOpcode(decoded.data_bits); - const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; - for (uint32_t dword = 0; dword < count; dword++) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = ir_op; - inst.src_count = decoded.src_count + 1u; - inst.memory = OffsetMemoryInfo(decoded, kind, dword); - inst.memory.resource = 0; - inst.dst.kind = OperandKind::Null; - if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, dword), inst.src[0], error)) { - return false; - } - for (uint32_t i = 0; i < decoded.src_count && i + 1u < 3u; i++) { - if (!LowerSourceOperand(DecodedSourceAt(decoded, i), inst.src[i + 1u], error)) { - return false; - } - } - block.instructions.push_back(inst); - } - return true; -} - -bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - const auto ir_op = DsWriteOpcode(decoded.data_bits); - Instruction inst; - inst.pc = decoded.pc; - inst.op = ir_op; - inst.src_count = 2; - inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); - inst.dst.kind = OperandKind::Null; - if (!LowerSourceOperand(decoded.src1, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock& block, - uint32_t dwords_per_write, std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::DsWrite2B32; - inst.src_count = 3; - inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); - inst.memory.data_dwords = dwords_per_write * 2u; - inst.dst.kind = OperandKind::Null; - if (!LowerSourceOperand(decoded.src1, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error) || - !LowerSourceOperand(decoded.src2, inst.src[2], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -bool LowerImageOperation(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - Instruction inst; - inst.pc = decoded.pc; - if (decoded.opcode == Decoder::Opcode::IMAGE_STORE || - decoded.opcode == Decoder::Opcode::IMAGE_STORE_MIP) { - inst.op = Opcode::ImageStore; - inst.src_count = 2; - inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::StorageImage); - inst.memory.sampler = 0; - inst.dst.kind = OperandKind::Null; - if (!LowerSourceOperand(decoded.dst, inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - return true; - } - - inst.op = LoweredImageOpcode(decoded.opcode); - inst.src_count = 1; - inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Image); - if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -} // namespace - -bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& block, - std::string* error) { - switch (decoded.opcode) { - case Decoder::Opcode::S_LOAD_DWORD: - case Decoder::Opcode::S_LOAD_DWORDX2: - case Decoder::Opcode::S_LOAD_DWORDX4: - case Decoder::Opcode::S_LOAD_DWORDX8: - case Decoder::Opcode::S_LOAD_DWORDX16: - return LowerScalarMemoryLoadDword(decoded, block, Opcode::SLoadDword, error); - case Decoder::Opcode::S_BUFFER_LOAD_DWORD: - case Decoder::Opcode::S_BUFFER_LOAD_DWORDX2: - case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4: - case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8: - case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16: - return LowerScalarMemoryLoadDword(decoded, block, Opcode::SBufferLoadDword, error); - case Decoder::Opcode::BUFFER_LOAD_UBYTE: - case Decoder::Opcode::BUFFER_LOAD_SBYTE: - case Decoder::Opcode::BUFFER_LOAD_USHORT: - case Decoder::Opcode::BUFFER_LOAD_SSHORT: - case Decoder::Opcode::BUFFER_LOAD_DWORD: - case Decoder::Opcode::BUFFER_LOAD_DWORDX2: - case Decoder::Opcode::BUFFER_LOAD_DWORDX3: - case Decoder::Opcode::BUFFER_LOAD_DWORDX4: - case Decoder::Opcode::BUFFER_LOAD_FORMAT_X: - case Decoder::Opcode::BUFFER_LOAD_FORMAT_XY: - case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ: - case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW: - case Decoder::Opcode::TBUFFER_LOAD_FORMAT_X: - case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XY: - case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZ: - case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZW: - return LowerBufferLoad(decoded, block, error); - case Decoder::Opcode::BUFFER_STORE_DWORD: - case Decoder::Opcode::BUFFER_STORE_DWORDX2: - case Decoder::Opcode::BUFFER_STORE_DWORDX3: - case Decoder::Opcode::BUFFER_STORE_DWORDX4: - case Decoder::Opcode::BUFFER_STORE_BYTE: - case Decoder::Opcode::BUFFER_STORE_SHORT: - case Decoder::Opcode::BUFFER_STORE_FORMAT_X: - case Decoder::Opcode::BUFFER_STORE_FORMAT_XY: - case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZ: - case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW: - case Decoder::Opcode::TBUFFER_STORE_FORMAT_X: - case Decoder::Opcode::TBUFFER_STORE_FORMAT_XY: - case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZ: - case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW: - return LowerBufferStore(decoded, block, error); - case Decoder::Opcode::BUFFER_ATOMIC_SWAP: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSwapU32, error); - case Decoder::Opcode::BUFFER_ATOMIC_ADD: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicAddU32, error); - case Decoder::Opcode::BUFFER_ATOMIC_SUB: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSubU32, error); - case Decoder::Opcode::BUFFER_ATOMIC_SMIN: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSMinI32, error); - case Decoder::Opcode::BUFFER_ATOMIC_UMIN: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicUMinU32, error); - case Decoder::Opcode::BUFFER_ATOMIC_SMAX: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSMaxI32, error); - case Decoder::Opcode::BUFFER_ATOMIC_UMAX: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicUMaxU32, error); - case Decoder::Opcode::BUFFER_ATOMIC_AND: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicAndU32, error); - case Decoder::Opcode::BUFFER_ATOMIC_OR: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicOrU32, error); - case Decoder::Opcode::BUFFER_ATOMIC_XOR: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicXorU32, error); - case Decoder::Opcode::BUFFER_ATOMIC_FMIN: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicFMinF32, error); - case Decoder::Opcode::BUFFER_ATOMIC_FMAX: - return LowerBufferAtomicDword(decoded, block, Opcode::AtomicFMaxF32, error); - case Decoder::Opcode::FLAT_LOAD_UBYTE: - case Decoder::Opcode::FLAT_LOAD_SBYTE: - case Decoder::Opcode::FLAT_LOAD_USHORT: - case Decoder::Opcode::FLAT_LOAD_SSHORT: - case Decoder::Opcode::FLAT_LOAD_DWORD: - case Decoder::Opcode::FLAT_LOAD_DWORDX2: - case Decoder::Opcode::FLAT_LOAD_DWORDX3: - case Decoder::Opcode::FLAT_LOAD_DWORDX4: return LowerFlatLoad(decoded, block, error); - case Decoder::Opcode::FLAT_STORE_BYTE: - case Decoder::Opcode::FLAT_STORE_SHORT: - case Decoder::Opcode::FLAT_STORE_DWORD: - case Decoder::Opcode::FLAT_STORE_DWORDX2: - case Decoder::Opcode::FLAT_STORE_DWORDX3: - case Decoder::Opcode::FLAT_STORE_DWORDX4: return LowerFlatStore(decoded, block, error); - case Decoder::Opcode::DS_ADD_U32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicAddU32, false, error); - case Decoder::Opcode::DS_ADD_RTN_U32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicAddU32, true, error); - case Decoder::Opcode::DS_SUB_U32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicSubU32, false, error); - case Decoder::Opcode::DS_SUB_RTN_U32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicSubU32, true, error); - case Decoder::Opcode::DS_MIN_I32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMinI32, false, error); - case Decoder::Opcode::DS_MIN_RTN_I32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMinI32, true, error); - case Decoder::Opcode::DS_MAX_I32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMaxI32, false, error); - case Decoder::Opcode::DS_MAX_RTN_I32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMaxI32, true, error); - case Decoder::Opcode::DS_MIN_U32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMinU32, false, error); - case Decoder::Opcode::DS_MIN_RTN_U32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMinU32, true, error); - case Decoder::Opcode::DS_MAX_U32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMaxU32, false, error); - case Decoder::Opcode::DS_MAX_RTN_U32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMaxU32, true, error); - case Decoder::Opcode::DS_AND_B32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicAndU32, false, error); - case Decoder::Opcode::DS_AND_RTN_B32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicAndU32, true, error); - case Decoder::Opcode::DS_OR_B32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicOrU32, false, error); - case Decoder::Opcode::DS_OR_RTN_B32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicOrU32, true, error); - case Decoder::Opcode::DS_XOR_B32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicXorU32, false, error); - case Decoder::Opcode::DS_XOR_RTN_B32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicXorU32, true, error); - case Decoder::Opcode::DS_WRXCHG_RTN_B32: - return LowerDsAtomicU32(decoded, block, Opcode::AtomicSwapU32, true, error); - case Decoder::Opcode::DS_MIN_F32: - return LowerDsFloatMinMaxF32(decoded, block, Opcode::DsMinF32, error); - case Decoder::Opcode::DS_MAX_F32: - return LowerDsFloatMinMaxF32(decoded, block, Opcode::DsMaxF32, error); - case Decoder::Opcode::DS_SWIZZLE_B32: return LowerDsSwizzleB32(decoded, block, error); - case Decoder::Opcode::DS_CONSUME: - return LowerDsAppendConsume(decoded, block, Opcode::DsConsume, error); - case Decoder::Opcode::DS_APPEND: - return LowerDsAppendConsume(decoded, block, Opcode::DsAppend, error); - case Decoder::Opcode::DS_WRITE_ADDTID_B32: - return LowerDsWriteAddtidB32(decoded, block, error); - case Decoder::Opcode::DS_READ_ADDTID_B32: - return LowerDsReadAddtidB32(decoded, block, error); - case Decoder::Opcode::DS_READ2_B32: - case Decoder::Opcode::DS_READ2ST64_B32: return LowerDsRead2(decoded, block, 1, error); - case Decoder::Opcode::DS_READ2_B64: - case Decoder::Opcode::DS_READ2ST64_B64: return LowerDsRead2(decoded, block, 2, error); - case Decoder::Opcode::DS_READ_I8: - case Decoder::Opcode::DS_READ_U8: - case Decoder::Opcode::DS_READ_I16: - case Decoder::Opcode::DS_READ_U16: - case Decoder::Opcode::DS_READ_U16_D16: - case Decoder::Opcode::DS_READ_B32: - case Decoder::Opcode::DS_READ_B64: - case Decoder::Opcode::DS_READ_B96: - case Decoder::Opcode::DS_READ_B128: return LowerDsRead(decoded, block, error); - case Decoder::Opcode::DS_WRITE2_B32: - case Decoder::Opcode::DS_WRITE2ST64_B32: return LowerDsWrite2(decoded, block, 1, error); - case Decoder::Opcode::DS_WRITE2_B64: - case Decoder::Opcode::DS_WRITE2ST64_B64: return LowerDsWrite2(decoded, block, 2, error); - case Decoder::Opcode::DS_WRITE_B8: - case Decoder::Opcode::DS_WRITE_B16: - case Decoder::Opcode::DS_WRITE_B32: - case Decoder::Opcode::DS_WRITE_B64: - case Decoder::Opcode::DS_WRITE_B96: - case Decoder::Opcode::DS_WRITE_B128: return LowerDsWrite(decoded, block, error); - case Decoder::Opcode::IMAGE_GET_RESINFO: - case Decoder::Opcode::IMAGE_GET_LOD: - case Decoder::Opcode::IMAGE_LOAD: - case Decoder::Opcode::IMAGE_LOAD_MIP: - case Decoder::Opcode::IMAGE_STORE: - case Decoder::Opcode::IMAGE_STORE_MIP: - case Decoder::Opcode::IMAGE_GATHER4_LZ: - case Decoder::Opcode::IMAGE_GATHER4_C: - case Decoder::Opcode::IMAGE_GATHER4_C_LZ: - case Decoder::Opcode::IMAGE_GATHER4_LZ_O: - case Decoder::Opcode::IMAGE_GATHER4_C_O: - case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O: - case Decoder::Opcode::IMAGE_GATHER4H: - case Decoder::Opcode::IMAGE_SAMPLE: return LowerImageOperation(decoded, block, error); - case Decoder::Opcode::IMAGE_ATOMIC_ADD: - case Decoder::Opcode::IMAGE_ATOMIC_UMIN: - case Decoder::Opcode::IMAGE_ATOMIC_UMAX: - case Decoder::Opcode::IMAGE_ATOMIC_AND: - case Decoder::Opcode::IMAGE_ATOMIC_OR: - case Decoder::Opcode::IMAGE_ATOMIC_XOR: return LowerImageAtomicU32(decoded, block, error); - default: - if (error != nullptr) { - *error = fmt::format("memory opcode has no specialized IR lowering: {}", - magic_enum::enum_name(decoded.opcode)); - } - return false; - } -} - -bool IsMemoryOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::S_LOAD_DWORD: - case Decoder::Opcode::S_LOAD_DWORDX2: - case Decoder::Opcode::S_LOAD_DWORDX4: - case Decoder::Opcode::S_LOAD_DWORDX8: - case Decoder::Opcode::S_LOAD_DWORDX16: - case Decoder::Opcode::S_BUFFER_LOAD_DWORD: - case Decoder::Opcode::S_BUFFER_LOAD_DWORDX2: - case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4: - case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8: - case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16: - case Decoder::Opcode::BUFFER_LOAD_UBYTE: - case Decoder::Opcode::BUFFER_LOAD_SBYTE: - case Decoder::Opcode::BUFFER_LOAD_USHORT: - case Decoder::Opcode::BUFFER_LOAD_SSHORT: - case Decoder::Opcode::BUFFER_LOAD_DWORD: - case Decoder::Opcode::BUFFER_LOAD_DWORDX2: - case Decoder::Opcode::BUFFER_LOAD_DWORDX3: - case Decoder::Opcode::BUFFER_LOAD_DWORDX4: - case Decoder::Opcode::BUFFER_LOAD_FORMAT_X: - case Decoder::Opcode::BUFFER_LOAD_FORMAT_XY: - case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ: - case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW: - case Decoder::Opcode::TBUFFER_LOAD_FORMAT_X: - case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XY: - case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZ: - case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZW: - case Decoder::Opcode::BUFFER_STORE_DWORD: - case Decoder::Opcode::BUFFER_STORE_DWORDX2: - case Decoder::Opcode::BUFFER_STORE_DWORDX3: - case Decoder::Opcode::BUFFER_STORE_DWORDX4: - case Decoder::Opcode::BUFFER_STORE_BYTE: - case Decoder::Opcode::BUFFER_STORE_SHORT: - case Decoder::Opcode::BUFFER_STORE_FORMAT_X: - case Decoder::Opcode::BUFFER_STORE_FORMAT_XY: - case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZ: - case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW: - case Decoder::Opcode::TBUFFER_STORE_FORMAT_X: - case Decoder::Opcode::TBUFFER_STORE_FORMAT_XY: - case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZ: - case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW: - case Decoder::Opcode::BUFFER_ATOMIC_SWAP: - case Decoder::Opcode::BUFFER_ATOMIC_ADD: - case Decoder::Opcode::BUFFER_ATOMIC_SUB: - case Decoder::Opcode::BUFFER_ATOMIC_SMIN: - case Decoder::Opcode::BUFFER_ATOMIC_UMIN: - case Decoder::Opcode::BUFFER_ATOMIC_SMAX: - case Decoder::Opcode::BUFFER_ATOMIC_UMAX: - case Decoder::Opcode::BUFFER_ATOMIC_AND: - case Decoder::Opcode::BUFFER_ATOMIC_OR: - case Decoder::Opcode::BUFFER_ATOMIC_XOR: - case Decoder::Opcode::BUFFER_ATOMIC_FMIN: - case Decoder::Opcode::BUFFER_ATOMIC_FMAX: - case Decoder::Opcode::FLAT_LOAD_UBYTE: - case Decoder::Opcode::FLAT_LOAD_SBYTE: - case Decoder::Opcode::FLAT_LOAD_USHORT: - case Decoder::Opcode::FLAT_LOAD_SSHORT: - case Decoder::Opcode::FLAT_LOAD_DWORD: - case Decoder::Opcode::FLAT_LOAD_DWORDX2: - case Decoder::Opcode::FLAT_LOAD_DWORDX3: - case Decoder::Opcode::FLAT_LOAD_DWORDX4: - case Decoder::Opcode::FLAT_STORE_BYTE: - case Decoder::Opcode::FLAT_STORE_SHORT: - case Decoder::Opcode::FLAT_STORE_DWORD: - case Decoder::Opcode::FLAT_STORE_DWORDX2: - case Decoder::Opcode::FLAT_STORE_DWORDX3: - case Decoder::Opcode::FLAT_STORE_DWORDX4: - case Decoder::Opcode::DS_ADD_U32: - case Decoder::Opcode::DS_ADD_RTN_U32: - case Decoder::Opcode::DS_SUB_U32: - case Decoder::Opcode::DS_SUB_RTN_U32: - case Decoder::Opcode::DS_MIN_I32: - case Decoder::Opcode::DS_MIN_RTN_I32: - case Decoder::Opcode::DS_MAX_I32: - case Decoder::Opcode::DS_MAX_RTN_I32: - case Decoder::Opcode::DS_MIN_U32: - case Decoder::Opcode::DS_MIN_RTN_U32: - case Decoder::Opcode::DS_MAX_U32: - case Decoder::Opcode::DS_MAX_RTN_U32: - case Decoder::Opcode::DS_AND_B32: - case Decoder::Opcode::DS_AND_RTN_B32: - case Decoder::Opcode::DS_OR_B32: - case Decoder::Opcode::DS_OR_RTN_B32: - case Decoder::Opcode::DS_XOR_B32: - case Decoder::Opcode::DS_XOR_RTN_B32: - case Decoder::Opcode::DS_WRXCHG_RTN_B32: - case Decoder::Opcode::DS_MIN_F32: - case Decoder::Opcode::DS_MAX_F32: - case Decoder::Opcode::DS_SWIZZLE_B32: - case Decoder::Opcode::DS_CONSUME: - case Decoder::Opcode::DS_APPEND: - case Decoder::Opcode::DS_READ_I8: - case Decoder::Opcode::DS_READ_U8: - case Decoder::Opcode::DS_READ_I16: - case Decoder::Opcode::DS_READ_U16: - case Decoder::Opcode::DS_READ_U16_D16: - case Decoder::Opcode::DS_READ2_B32: - case Decoder::Opcode::DS_READ2ST64_B32: - case Decoder::Opcode::DS_READ_B32: - case Decoder::Opcode::DS_READ_B64: - case Decoder::Opcode::DS_READ2_B64: - case Decoder::Opcode::DS_READ2ST64_B64: - case Decoder::Opcode::DS_READ_B96: - case Decoder::Opcode::DS_READ_B128: - case Decoder::Opcode::DS_WRITE_B8: - case Decoder::Opcode::DS_WRITE_B16: - case Decoder::Opcode::DS_WRITE2_B32: - case Decoder::Opcode::DS_WRITE2ST64_B32: - case Decoder::Opcode::DS_WRITE2_B64: - case Decoder::Opcode::DS_WRITE2ST64_B64: - case Decoder::Opcode::DS_WRITE_B32: - case Decoder::Opcode::DS_WRITE_B64: - case Decoder::Opcode::DS_WRITE_B96: - case Decoder::Opcode::DS_WRITE_B128: - case Decoder::Opcode::DS_WRITE_ADDTID_B32: - case Decoder::Opcode::DS_READ_ADDTID_B32: - case Decoder::Opcode::IMAGE_GET_RESINFO: - case Decoder::Opcode::IMAGE_GET_LOD: - case Decoder::Opcode::IMAGE_LOAD: - case Decoder::Opcode::IMAGE_LOAD_MIP: - case Decoder::Opcode::IMAGE_STORE: - case Decoder::Opcode::IMAGE_STORE_MIP: - case Decoder::Opcode::IMAGE_ATOMIC_ADD: - case Decoder::Opcode::IMAGE_ATOMIC_UMIN: - case Decoder::Opcode::IMAGE_ATOMIC_UMAX: - case Decoder::Opcode::IMAGE_ATOMIC_AND: - case Decoder::Opcode::IMAGE_ATOMIC_OR: - case Decoder::Opcode::IMAGE_ATOMIC_XOR: - case Decoder::Opcode::IMAGE_GATHER4_LZ: - case Decoder::Opcode::IMAGE_GATHER4_C: - case Decoder::Opcode::IMAGE_GATHER4_C_LZ: - case Decoder::Opcode::IMAGE_GATHER4_LZ_O: - case Decoder::Opcode::IMAGE_GATHER4_C_O: - case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O: - case Decoder::Opcode::IMAGE_GATHER4H: - case Decoder::Opcode::IMAGE_SAMPLE: return true; - default: return false; - } -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.h b/src/graphics/shader/recompiler/ir/ValueProgram.h deleted file mode 100644 index 6d1f7f866..000000000 --- a/src/graphics/shader/recompiler/ir/ValueProgram.h +++ /dev/null @@ -1,76 +0,0 @@ -#pragma once - -#include "graphics/shader/recompiler/ir/Block.h" -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -#include -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { - -struct ValueBlockInfo { - uint32_t id = 0; - uint32_t start_pc = 0; - uint32_t end_pc = 0; - CFG::Terminator terminator; - Value condition; - Value indirect_target; -}; - -struct DescriptorSource { - struct IndirectImage { - uint32_t material_source = 0; - uint32_t heap_source = 0; - uint32_t selector_stride = 0; - uint32_t selector_offset = 0; - uint32_t key_arg = 0; - - bool operator==(const IndirectImage& other) const = default; - }; - - std::array dwords {}; - uint32_t dword_count = 0; - std::optional indirect_image; - - bool operator==(const DescriptorSource& other) const = default; -}; - -struct SrtRead { - Value value; - uint32_t flat_offset = 0; - uint32_t use_pc = 0; - - bool operator==(const SrtRead& other) const = default; -}; - -struct ValueProgram { - ValueProgram() = default; - ~ValueProgram(); - - ValueProgram(const ValueProgram&) = delete; - ValueProgram& operator=(const ValueProgram&) = delete; - ValueProgram(ValueProgram&&) = default; - ValueProgram& operator=(ValueProgram&&) = default; - - std::vector> block_storage; - BlockList blocks; - std::vector block_info; - // Decoded MIMG/VMEM metadata carries details such as RDNA2 NSA address registers and - // storage-image swizzles. Typed memory instructions carry a dense index into these shader-local - // tables until those fields are consumed by emission. - std::vector memory_info; - std::vector export_info; - std::vector descriptor_sources; - std::vector srt_reads; - std::vector dynamic_reads; -}; - -bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::string* error); -void ResolveControlFlowIdentities(ValueProgram& program); -bool EquivalentValue(const ValueProgram& program, Value left, Value right); -Value ResolveInvariantPhi(const ValueProgram& program, Value value); -std::string ValueProgramToString(const ValueProgram& program); - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/Opcodes.h b/src/graphics/shader/recompiler/ir/opcodes/Opcodes.h deleted file mode 100644 index be62c27c4..000000000 --- a/src/graphics/shader/recompiler/ir/opcodes/Opcodes.h +++ /dev/null @@ -1,71 +0,0 @@ -#pragma once - -#include "graphics/shader/recompiler/ir/Type.h" - -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { - -enum class Opcode { -#define IR_OPCODE(name, category, lowering, ...) name, -#include "graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc" -#undef IR_OPCODE - Count, -}; - -enum class OpcodeClass { General, Compare, Compare64 }; -enum class LoweringClass { - Control, - Move, - Lane, - State, - Memory, - Attribute, - IntegerCompare, - Integer16Compare, - FloatCompare, - Conversion, - Integer16, - PackedInteger16, - PackedFloat16, - Float16, - Float, - U64Mask, - SimpleInteger, - ComposedInteger, - ExtendedInteger, -}; - -struct OpcodeInfo { - std::string_view name; - OpcodeClass opcode_class = OpcodeClass::General; - LoweringClass lowering_class = LoweringClass::Control; -}; - -inline constexpr OpcodeInfo OPCODE_INFO[] = { -#define IR_OPCODE(name, category, lowering, ...) \ - {#name, OpcodeClass::category, LoweringClass::lowering}, -#include "graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc" -#undef IR_OPCODE -}; -static_assert(std::size(OPCODE_INFO) == static_cast(Opcode::Count)); - -constexpr const OpcodeInfo& GetOpcodeInfo(Opcode opcode) { - return OPCODE_INFO[static_cast(opcode)]; -} - -constexpr std::string_view OpcodeName(Opcode opcode) { - return GetOpcodeInfo(opcode).name; -} - -constexpr bool IsCompareOpcode(Opcode opcode) { - return GetOpcodeInfo(opcode).opcode_class != OpcodeClass::General; -} - -constexpr bool IsCompare64Opcode(Opcode opcode) { - return GetOpcodeInfo(opcode).opcode_class == OpcodeClass::Compare64; -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp deleted file mode 100644 index e890961c7..000000000 --- a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp +++ /dev/null @@ -1,548 +0,0 @@ -#include "common/assert.h" -#include "graphics/shader/recompiler/BufferFormat.h" -#include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/ShaderIRInternal.h" - -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { -namespace { - -struct LowerMap { - Decoder::Opcode decoded = Decoder::Opcode::UNKNOWN; - Opcode ir = Opcode::MoveU32; -}; - -constexpr LowerMap LOWER_OPS[] = { - {Decoder::Opcode::S_MOV_B32, Opcode::MoveU32}, - {Decoder::Opcode::S_MOV_B64, Opcode::MoveU64}, - {Decoder::Opcode::S_WQM_B64, Opcode::WqmB64}, - {Decoder::Opcode::S_MOVK_I32, Opcode::MoveU32}, - {Decoder::Opcode::S_ABS_I32, Opcode::AbsI32}, - {Decoder::Opcode::S_BREV_B32, Opcode::BitReverseU32}, - {Decoder::Opcode::S_BCNT1_I32_B32, Opcode::BitCountU32}, - {Decoder::Opcode::S_BCNT1_I32_B64, Opcode::BitCountU64}, - {Decoder::Opcode::S_FF1_I32_B32, Opcode::FindLsbU32}, - {Decoder::Opcode::S_FF1_I32_B64, Opcode::FindLsbU64}, - {Decoder::Opcode::S_FLBIT_I32_B64, Opcode::FindMsbFromHighU64}, - {Decoder::Opcode::S_BITREPLICATE_B64_B32, Opcode::BitReplicateB64B32}, - {Decoder::Opcode::S_QUADMASK_B64, Opcode::QuadmaskB64}, - {Decoder::Opcode::S_NOT_B32, Opcode::BitwiseNotU32}, - {Decoder::Opcode::S_NOT_B64, Opcode::BitwiseNotU64}, - {Decoder::Opcode::S_ADD_U32, Opcode::IAddU32}, - {Decoder::Opcode::S_SUB_U32, Opcode::ISubU32}, - {Decoder::Opcode::S_ADD_I32, Opcode::IAddU32}, - {Decoder::Opcode::S_SUB_I32, Opcode::ISubU32}, - {Decoder::Opcode::S_MIN_I32, Opcode::IMinI32}, - {Decoder::Opcode::S_MAX_I32, Opcode::IMaxI32}, - {Decoder::Opcode::S_MIN_U32, Opcode::UMinU32}, - {Decoder::Opcode::S_MAX_U32, Opcode::UMaxU32}, - {Decoder::Opcode::S_AND_B32, Opcode::BitwiseAndU32}, - {Decoder::Opcode::S_AND_B64, Opcode::BitwiseAndU64}, - {Decoder::Opcode::S_ANDN2_B32, Opcode::BitwiseAndNotU32}, - {Decoder::Opcode::S_ANDN2_B64, Opcode::BitwiseAndNotU64}, - {Decoder::Opcode::S_OR_B32, Opcode::BitwiseOrU32}, - {Decoder::Opcode::S_OR_B64, Opcode::BitwiseOrU64}, - {Decoder::Opcode::S_ORN2_B32, Opcode::BitwiseOrNotU32}, - {Decoder::Opcode::S_ORN2_B64, Opcode::BitwiseOrNotU64}, - {Decoder::Opcode::S_XOR_B32, Opcode::BitwiseXorU32}, - {Decoder::Opcode::S_XOR_B64, Opcode::BitwiseXorU64}, - {Decoder::Opcode::S_NAND_B32, Opcode::BitwiseNandU32}, - {Decoder::Opcode::S_NAND_B64, Opcode::BitwiseNandU64}, - {Decoder::Opcode::S_NOR_B32, Opcode::BitwiseNorU32}, - {Decoder::Opcode::S_NOR_B64, Opcode::BitwiseNorU64}, - {Decoder::Opcode::S_XNOR_B32, Opcode::BitwiseXnorU32}, - {Decoder::Opcode::S_XNOR_B64, Opcode::BitwiseXnorU64}, - {Decoder::Opcode::S_LSHL_B32, Opcode::ShiftLeftLogicalU32}, - {Decoder::Opcode::S_LSHL_B64, Opcode::ShiftLeftLogicalU64}, - {Decoder::Opcode::S_LSHR_B32, Opcode::ShiftRightLogicalU32}, - {Decoder::Opcode::S_LSHR_B64, Opcode::ShiftRightLogicalU64}, - {Decoder::Opcode::S_ASHR_I32, Opcode::ShiftRightArithmeticI32}, - {Decoder::Opcode::S_MUL_I32, Opcode::IMulU32}, - {Decoder::Opcode::S_MUL_HI_U32, Opcode::UMulHighU32}, - {Decoder::Opcode::S_MULK_I32, Opcode::IMulU32}, - {Decoder::Opcode::S_BFE_U32, Opcode::BitFieldExtractU32}, - {Decoder::Opcode::S_BFE_I32, Opcode::BitFieldExtractI32}, - {Decoder::Opcode::S_BFE_U64, Opcode::BitFieldExtractU64}, - {Decoder::Opcode::S_BFM_B32, Opcode::BitFieldMaskU32}, - {Decoder::Opcode::S_BFM_B64, Opcode::BitFieldMaskU64}, - {Decoder::Opcode::S_BITCMP0_B32, Opcode::BitCompare0B32}, - {Decoder::Opcode::S_BITCMP1_B32, Opcode::BitCompare1B32}, - {Decoder::Opcode::S_PACK_LL_B32_B16, Opcode::PackLowLowU16}, - {Decoder::Opcode::S_PACK_LH_B32_B16, Opcode::PackLowHighU16}, - {Decoder::Opcode::S_PACK_HH_B32_B16, Opcode::PackHighHighU16}, - {Decoder::Opcode::S_CMP_EQ_I32, Opcode::CompareEqI32}, - {Decoder::Opcode::S_CMP_LG_I32, Opcode::CompareNeI32}, - {Decoder::Opcode::S_CMP_GT_I32, Opcode::CompareGtI32}, - {Decoder::Opcode::S_CMP_GE_I32, Opcode::CompareGeI32}, - {Decoder::Opcode::S_CMP_LT_I32, Opcode::CompareLtI32}, - {Decoder::Opcode::S_CMP_LE_I32, Opcode::CompareLeI32}, - {Decoder::Opcode::S_CMP_EQ_U32, Opcode::CompareEqU32}, - {Decoder::Opcode::S_CMP_LG_U32, Opcode::CompareNeU32}, - {Decoder::Opcode::S_CMP_GT_U32, Opcode::CompareGtU32}, - {Decoder::Opcode::S_CMP_GE_U32, Opcode::CompareGeU32}, - {Decoder::Opcode::S_CMP_LT_U32, Opcode::CompareLtU32}, - {Decoder::Opcode::S_CMP_LE_U32, Opcode::CompareLeU32}, - {Decoder::Opcode::S_CMP_EQ_U64, Opcode::CompareEqU64}, - {Decoder::Opcode::S_CMP_LG_U64, Opcode::CompareNeU64}, - {Decoder::Opcode::V_READFIRSTLANE_B32, Opcode::ReadFirstLaneU32}, - {Decoder::Opcode::V_READLANE_B32, Opcode::ReadLaneU32}, - {Decoder::Opcode::V_WRITELANE_B32, Opcode::WriteLaneU32}, - {Decoder::Opcode::V_PERMLANE16_B32, Opcode::Permlane16B32}, - {Decoder::Opcode::V_PERMLANEX16_B32, Opcode::Permlanex16B32}, - {Decoder::Opcode::V_CVT_F32_I32, Opcode::ConvertI32ToF32}, - {Decoder::Opcode::V_CVT_F32_U32, Opcode::ConvertU32ToF32}, - {Decoder::Opcode::V_CVT_U32_F32, Opcode::ConvertF32ToU32}, - {Decoder::Opcode::V_CVT_I32_F32, Opcode::ConvertF32ToI32}, - {Decoder::Opcode::V_CVT_F16_F32, Opcode::ConvertF32ToF16}, - {Decoder::Opcode::V_CVT_F32_F16, Opcode::ConvertF16ToF32}, - {Decoder::Opcode::V_CVT_F16_U16, Opcode::ConvertU16ToF16}, - {Decoder::Opcode::V_CVT_U16_F16, Opcode::ConvertF16ToU16}, - {Decoder::Opcode::V_CVT_F16_I16, Opcode::ConvertI16ToF16}, - {Decoder::Opcode::V_CVT_I16_F16, Opcode::ConvertF16ToI16}, - {Decoder::Opcode::V_CVT_RPI_I32_F32, Opcode::ConvertRoundPlusInfF32ToI32}, - {Decoder::Opcode::V_CVT_FLR_I32_F32, Opcode::ConvertFloorF32ToI32}, - {Decoder::Opcode::V_FREXP_EXP_I32_F32, Opcode::FrexpExpI32F32}, - {Decoder::Opcode::V_FREXP_MANT_F32, Opcode::FrexpMantF32}, - {Decoder::Opcode::V_CVT_OFF_F32_I4, Opcode::ConvertI4ToOffsetF32}, - {Decoder::Opcode::V_RCP_F32, Opcode::RcpF32}, - {Decoder::Opcode::V_RCP_IFLAG_F32, Opcode::RcpIflagF32}, - {Decoder::Opcode::V_FRACT_F32, Opcode::FractF32}, - {Decoder::Opcode::V_TRUNC_F32, Opcode::TruncF32}, - {Decoder::Opcode::V_CEIL_F32, Opcode::CeilF32}, - {Decoder::Opcode::V_RNDNE_F32, Opcode::RoundEvenF32}, - {Decoder::Opcode::V_FLOOR_F32, Opcode::FloorF32}, - {Decoder::Opcode::V_EXP_F32, Opcode::Exp2F32}, - {Decoder::Opcode::V_LOG_F32, Opcode::Log2F32}, - {Decoder::Opcode::V_RSQ_F32, Opcode::InverseSqrtF32}, - {Decoder::Opcode::V_SQRT_F32, Opcode::SqrtF32}, - {Decoder::Opcode::V_RCP_F16, Opcode::RcpF16}, - {Decoder::Opcode::V_SQRT_F16, Opcode::SqrtF16}, - {Decoder::Opcode::V_RSQ_F16, Opcode::InverseSqrtF16}, - {Decoder::Opcode::V_LOG_F16, Opcode::Log2F16}, - {Decoder::Opcode::V_EXP_F16, Opcode::Exp2F16}, - {Decoder::Opcode::V_FLOOR_F16, Opcode::FloorF16}, - {Decoder::Opcode::V_CEIL_F16, Opcode::CeilF16}, - {Decoder::Opcode::V_TRUNC_F16, Opcode::TruncF16}, - {Decoder::Opcode::V_RNDNE_F16, Opcode::RoundEvenF16}, - {Decoder::Opcode::V_SIN_F32, Opcode::SinF32}, - {Decoder::Opcode::V_COS_F32, Opcode::CosF32}, - {Decoder::Opcode::V_NOT_B32, Opcode::BitwiseNotU32}, - {Decoder::Opcode::V_BFREV_B32, Opcode::BitReverseU32}, - {Decoder::Opcode::V_FFBL_B32, Opcode::FindLsbU32}, - {Decoder::Opcode::V_FFBH_U32, Opcode::FindMsbFromHighU32}, - {Decoder::Opcode::V_ADD_F32, Opcode::FAddF32}, - {Decoder::Opcode::V_SUB_F32, Opcode::FSubF32}, - {Decoder::Opcode::V_SUBREV_F32, Opcode::FSubF32}, - {Decoder::Opcode::V_MUL_F32, Opcode::FMulF32}, - {Decoder::Opcode::V_MAC_F32, Opcode::FMadF32}, - {Decoder::Opcode::V_MADMK_F32, Opcode::FMadF32}, - {Decoder::Opcode::V_MADAK_F32, Opcode::FMadF32}, - {Decoder::Opcode::V_MIN_F32, Opcode::FMinF32}, - {Decoder::Opcode::V_MAX_F32, Opcode::FMaxF32}, - {Decoder::Opcode::V_MIN_I32, Opcode::IMinI32}, - {Decoder::Opcode::V_MAX_I32, Opcode::IMaxI32}, - {Decoder::Opcode::V_MAD_F32, Opcode::FMadF32}, - {Decoder::Opcode::V_MAD_I32_I24, Opcode::IMadI24U32}, - {Decoder::Opcode::V_MAD_U32_U24, Opcode::UMadU24U32}, - {Decoder::Opcode::V_CUBEID_F32, Opcode::CubeIdF32}, - {Decoder::Opcode::V_CUBESC_F32, Opcode::CubeScF32}, - {Decoder::Opcode::V_CUBETC_F32, Opcode::CubeTcF32}, - {Decoder::Opcode::V_CUBEMA_F32, Opcode::CubeMaF32}, - {Decoder::Opcode::V_FMA_F32, Opcode::FMadF32}, - {Decoder::Opcode::V_BFE_U32, Opcode::BitFieldExtract3U32}, - {Decoder::Opcode::V_BFE_I32, Opcode::BitFieldExtract3I32}, - {Decoder::Opcode::V_BFI_B32, Opcode::BitFieldInsertSelectU32}, - {Decoder::Opcode::V_ALIGNBIT_B32, Opcode::AlignBitU32}, - {Decoder::Opcode::V_ALIGNBYTE_B32, Opcode::AlignByteU32}, - {Decoder::Opcode::V_MIN3_F32, Opcode::FMin3F32}, - {Decoder::Opcode::V_MIN3_I32, Opcode::IMin3I32}, - {Decoder::Opcode::V_MIN3_U32, Opcode::UMin3U32}, - {Decoder::Opcode::V_MIN3_F16, Opcode::Min3F16}, - {Decoder::Opcode::V_MAX3_F32, Opcode::FMax3F32}, - {Decoder::Opcode::V_MAX3_I32, Opcode::IMax3I32}, - {Decoder::Opcode::V_MAX3_U32, Opcode::UMax3U32}, - {Decoder::Opcode::V_MAX3_F16, Opcode::Max3F16}, - {Decoder::Opcode::V_MED3_F32, Opcode::FMed3F32}, - {Decoder::Opcode::V_MED3_I32, Opcode::IMed3I32}, - {Decoder::Opcode::V_MED3_U32, Opcode::UMed3U32}, - {Decoder::Opcode::V_MED3_F16, Opcode::Med3F16}, - {Decoder::Opcode::V_MED3_I16, Opcode::IMed3I16}, - {Decoder::Opcode::V_SAD_U32, Opcode::SadU32}, - {Decoder::Opcode::V_ADD3_U32, Opcode::IAdd3U32}, - {Decoder::Opcode::V_LSHL_ADD_U32, Opcode::ShiftLeftAddU32}, - {Decoder::Opcode::V_ADD_LSHL_U32, Opcode::AddShiftLeftU32}, - {Decoder::Opcode::V_XAD_U32, Opcode::XorAddU32}, - {Decoder::Opcode::V_LSHL_OR_B32, Opcode::ShiftLeftOrU32}, - {Decoder::Opcode::V_AND_OR_B32, Opcode::BitwiseAndOrU32}, - {Decoder::Opcode::V_OR3_B32, Opcode::BitwiseOr3U32}, - {Decoder::Opcode::V_XOR3_B32, Opcode::BitwiseXor3U32}, - {Decoder::Opcode::V_ADD_NC_U32, Opcode::IAddU32}, - {Decoder::Opcode::V_SUB_NC_U32, Opcode::ISubU32}, - {Decoder::Opcode::V_SUBREV_NC_U32, Opcode::ISubU32}, - {Decoder::Opcode::V_ADD_NC_U16, Opcode::IAddU16}, - {Decoder::Opcode::V_SUB_NC_U16, Opcode::ISubI16}, - {Decoder::Opcode::V_MAX_U16, Opcode::UMaxU16}, - {Decoder::Opcode::V_MAX_I16, Opcode::IMaxI16}, - {Decoder::Opcode::V_MIN_U16, Opcode::UMinU16}, - {Decoder::Opcode::V_MIN_I16, Opcode::IMinI16}, - {Decoder::Opcode::V_ADD_NC_I16, Opcode::IAddU16}, - {Decoder::Opcode::V_SUB_NC_I16, Opcode::ISubI16}, - {Decoder::Opcode::V_MUL_I32_I24, Opcode::IMulI24U32}, - {Decoder::Opcode::V_MUL_U32_U24, Opcode::UMulU24U32}, - {Decoder::Opcode::V_MUL_LO_U32, Opcode::IMulU32}, - {Decoder::Opcode::V_MUL_HI_U32, Opcode::UMulHighU32}, - {Decoder::Opcode::V_MUL_LO_I32, Opcode::IMulU32}, - {Decoder::Opcode::V_MUL_HI_I32, Opcode::SMulHighI32}, - {Decoder::Opcode::V_ADD_I32, Opcode::IAddU32}, - {Decoder::Opcode::V_SUB_I32, Opcode::ISubU32}, - {Decoder::Opcode::V_SUBREV_I32, Opcode::ISubU32}, - {Decoder::Opcode::V_BFM_B32, Opcode::BitFieldMaskU32}, - {Decoder::Opcode::V_CVT_PKRTZ_F16_F32, Opcode::PackF32ToF16Rtz}, - {Decoder::Opcode::V_LDEXP_F32, Opcode::LdexpF32}, - {Decoder::Opcode::V_CVT_PKNORM_I16_F32, Opcode::PackSnorm2x16F32}, - {Decoder::Opcode::V_CVT_PKNORM_U16_F32, Opcode::PackUnorm2x16F32}, - {Decoder::Opcode::V_CVT_PK_U16_U32, Opcode::PackU16U32}, - {Decoder::Opcode::V_CVT_PK_I16_I32, Opcode::PackU16U32}, - {Decoder::Opcode::V_CVT_PK_U8_F32, Opcode::PackU8F32}, - {Decoder::Opcode::V_PACK_B32_F16, Opcode::PackB32F16}, - {Decoder::Opcode::V_PK_MAD_I16, Opcode::PackedMadI16}, - {Decoder::Opcode::V_PK_MUL_LO_U16, Opcode::PackedMulLoU16}, - {Decoder::Opcode::V_PK_ADD_I16, Opcode::PackedAddI16}, - {Decoder::Opcode::V_PK_SUB_I16, Opcode::PackedSubI16}, - {Decoder::Opcode::V_PK_LSHLREV_B16, Opcode::PackedLshlrevB16}, - {Decoder::Opcode::V_PK_LSHRREV_B16, Opcode::PackedLshrrevB16}, - {Decoder::Opcode::V_PK_ASHRREV_I16, Opcode::PackedAshrrevI16}, - {Decoder::Opcode::V_PK_MAX_I16, Opcode::PackedMaxI16}, - {Decoder::Opcode::V_PK_MIN_I16, Opcode::PackedMinI16}, - {Decoder::Opcode::V_PK_MAD_U16, Opcode::PackedMadU16}, - {Decoder::Opcode::V_PK_ADD_U16, Opcode::PackedAddU16}, - {Decoder::Opcode::V_PK_SUB_U16, Opcode::PackedSubU16}, - {Decoder::Opcode::V_PK_MAX_U16, Opcode::PackedMaxU16}, - {Decoder::Opcode::V_PK_MIN_U16, Opcode::PackedMinU16}, - {Decoder::Opcode::V_PK_ADD_F16, Opcode::PackedAddF16}, - {Decoder::Opcode::V_PK_MUL_F16, Opcode::PackedMulF16}, - {Decoder::Opcode::V_PK_MIN_F16, Opcode::PackedMinF16}, - {Decoder::Opcode::V_PK_MAX_F16, Opcode::PackedMaxF16}, - {Decoder::Opcode::V_PK_FMA_F16, Opcode::PackedFmaF16}, - {Decoder::Opcode::V_PK_FMAC_F16, Opcode::PackedFmaF16}, - {Decoder::Opcode::V_ADD_F16, Opcode::AddF16}, - {Decoder::Opcode::V_SUB_F16, Opcode::SubF16}, - {Decoder::Opcode::V_SUBREV_F16, Opcode::SubF16}, - {Decoder::Opcode::V_MUL_F16, Opcode::MulF16}, - {Decoder::Opcode::V_FMAC_F16, Opcode::FmaF16}, - {Decoder::Opcode::V_FMAMK_F16, Opcode::FmaF16}, - {Decoder::Opcode::V_FMAAK_F16, Opcode::FmaF16}, - {Decoder::Opcode::V_FMA_F16, Opcode::FmaF16}, - {Decoder::Opcode::V_MAD_MIXLO_F16, Opcode::MadMixF16}, - {Decoder::Opcode::V_MAD_MIXHI_F16, Opcode::MadMixF16}, - {Decoder::Opcode::V_AND_B32, Opcode::BitwiseAndU32}, - {Decoder::Opcode::V_OR_B32, Opcode::BitwiseOrU32}, - {Decoder::Opcode::V_XOR_B32, Opcode::BitwiseXorU32}, - {Decoder::Opcode::V_XNOR_B32, Opcode::BitwiseXnorU32}, - {Decoder::Opcode::V_BCNT_U32_B32, Opcode::BitCountAddU32}, - {Decoder::Opcode::V_MBCNT_LO_U32_B32, Opcode::MaskedBitCountLowU32}, - {Decoder::Opcode::V_MBCNT_HI_U32_B32, Opcode::MaskedBitCountHighU32}, - {Decoder::Opcode::V_LSHL_B32, Opcode::ShiftLeftLogicalU32}, - {Decoder::Opcode::V_LSHLREV_B32, Opcode::ShiftLeftLogicalU32}, - {Decoder::Opcode::V_LSHLREV_B16, Opcode::ShiftLeftLogicalU16}, - {Decoder::Opcode::V_LSHR_B32, Opcode::ShiftRightLogicalU32}, - {Decoder::Opcode::V_LSHRREV_B32, Opcode::ShiftRightLogicalU32}, - {Decoder::Opcode::V_LSHLREV_B64, Opcode::ShiftLeftLogicalU64}, - {Decoder::Opcode::V_LSHRREV_B64, Opcode::ShiftRightLogicalU64}, - {Decoder::Opcode::V_LSHRREV_B16, Opcode::ShiftRightLogicalU16}, - {Decoder::Opcode::V_ASHR_I32, Opcode::ShiftRightArithmeticI32}, - {Decoder::Opcode::V_ASHRREV_I32, Opcode::ShiftRightArithmeticI32}, - {Decoder::Opcode::V_ASHRREV_I16, Opcode::ShiftRightArithmeticI16}, - {Decoder::Opcode::V_MIN_U32, Opcode::UMinU32}, - {Decoder::Opcode::V_MAX_U32, Opcode::UMaxU32}, - {Decoder::Opcode::V_MAX_F16, Opcode::MaxF16}, - {Decoder::Opcode::V_MIN_F16, Opcode::MinF16}, - {Decoder::Opcode::V_CMP_F_F32, Opcode::CompareFalse}, - {Decoder::Opcode::V_CMP_LT_F32, Opcode::CompareLtF32}, - {Decoder::Opcode::V_CMP_EQ_F32, Opcode::CompareEqF32}, - {Decoder::Opcode::V_CMP_LE_F32, Opcode::CompareLeF32}, - {Decoder::Opcode::V_CMP_GT_F32, Opcode::CompareGtF32}, - {Decoder::Opcode::V_CMP_LG_F32, Opcode::CompareNeF32}, - {Decoder::Opcode::V_CMP_GE_F32, Opcode::CompareGeF32}, - {Decoder::Opcode::V_CMP_O_F32, Opcode::CompareOrderedF32}, - {Decoder::Opcode::V_CMP_U_F32, Opcode::CompareUnorderedF32}, - {Decoder::Opcode::V_CMP_NGE_F32, Opcode::CompareUnordLtF32}, - {Decoder::Opcode::V_CMP_NLG_F32, Opcode::CompareUnordEqF32}, - {Decoder::Opcode::V_CMP_NGT_F32, Opcode::CompareUnordLeF32}, - {Decoder::Opcode::V_CMP_NLE_F32, Opcode::CompareUnordGtF32}, - {Decoder::Opcode::V_CMP_NEQ_F32, Opcode::CompareUnordNeF32}, - {Decoder::Opcode::V_CMP_NLT_F32, Opcode::CompareUnordGeF32}, - {Decoder::Opcode::V_CMP_TRU_F32, Opcode::CompareTrue}, - {Decoder::Opcode::V_CMP_CLASS_F32, Opcode::CompareClassF32}, - {Decoder::Opcode::V_CMP_LT_F16, Opcode::CompareLtF16}, - {Decoder::Opcode::V_CMP_EQ_F16, Opcode::CompareEqF16}, - {Decoder::Opcode::V_CMP_LE_F16, Opcode::CompareLeF16}, - {Decoder::Opcode::V_CMP_GT_F16, Opcode::CompareGtF16}, - {Decoder::Opcode::V_CMP_LG_F16, Opcode::CompareNeF16}, - {Decoder::Opcode::V_CMP_GE_F16, Opcode::CompareGeF16}, - {Decoder::Opcode::V_CMP_NEQ_F16, Opcode::CompareUnordNeF16}, - {Decoder::Opcode::V_CMPX_LT_F16, Opcode::CompareMaskLtF16}, - {Decoder::Opcode::V_CMPX_EQ_F16, Opcode::CompareMaskEqF16}, - {Decoder::Opcode::V_CMPX_LE_F16, Opcode::CompareMaskLeF16}, - {Decoder::Opcode::V_CMPX_GT_F16, Opcode::CompareMaskGtF16}, - {Decoder::Opcode::V_CMPX_GE_F16, Opcode::CompareMaskGeF16}, - {Decoder::Opcode::V_CMPX_NEQ_F16, Opcode::CompareMaskUnordNeF16}, - {Decoder::Opcode::V_CMPX_NLT_F16, Opcode::CompareMaskUnordGeF16}, - {Decoder::Opcode::V_CMPX_LT_F32, Opcode::CompareMaskLtF32}, - {Decoder::Opcode::V_CMPX_EQ_F32, Opcode::CompareMaskEqF32}, - {Decoder::Opcode::V_CMPX_LE_F32, Opcode::CompareMaskLeF32}, - {Decoder::Opcode::V_CMPX_GT_F32, Opcode::CompareMaskGtF32}, - {Decoder::Opcode::V_CMPX_LG_F32, Opcode::CompareMaskNeF32}, - {Decoder::Opcode::V_CMPX_GE_F32, Opcode::CompareMaskGeF32}, - {Decoder::Opcode::V_CMPX_NGE_F32, Opcode::CompareMaskUnordLtF32}, - {Decoder::Opcode::V_CMPX_NLG_F32, Opcode::CompareMaskUnordEqF32}, - {Decoder::Opcode::V_CMPX_NGT_F32, Opcode::CompareMaskUnordLeF32}, - {Decoder::Opcode::V_CMPX_NLE_F32, Opcode::CompareMaskUnordGtF32}, - {Decoder::Opcode::V_CMPX_NEQ_F32, Opcode::CompareMaskUnordNeF32}, - {Decoder::Opcode::V_CMPX_NLT_F32, Opcode::CompareMaskUnordGeF32}, - {Decoder::Opcode::V_CMP_F_I32, Opcode::CompareFalse}, - {Decoder::Opcode::V_CMP_LT_I32, Opcode::CompareLtI32}, - {Decoder::Opcode::V_CMP_EQ_I32, Opcode::CompareEqI32}, - {Decoder::Opcode::V_CMP_LE_I32, Opcode::CompareLeI32}, - {Decoder::Opcode::V_CMP_GT_I32, Opcode::CompareGtI32}, - {Decoder::Opcode::V_CMP_NE_I32, Opcode::CompareNeI32}, - {Decoder::Opcode::V_CMP_GE_I32, Opcode::CompareGeI32}, - {Decoder::Opcode::V_CMP_T_I32, Opcode::CompareTrue}, - {Decoder::Opcode::V_CMP_LT_I16, Opcode::CompareLtI16}, - {Decoder::Opcode::V_CMP_EQ_I16, Opcode::CompareEqI16}, - {Decoder::Opcode::V_CMP_LE_I16, Opcode::CompareLeI16}, - {Decoder::Opcode::V_CMP_GT_I16, Opcode::CompareGtI16}, - {Decoder::Opcode::V_CMP_NE_I16, Opcode::CompareNeI16}, - {Decoder::Opcode::V_CMP_GE_I16, Opcode::CompareGeI16}, - {Decoder::Opcode::V_CMPX_LT_I32, Opcode::CompareMaskLtI32}, - {Decoder::Opcode::V_CMPX_EQ_I32, Opcode::CompareMaskEqI32}, - {Decoder::Opcode::V_CMPX_LE_I32, Opcode::CompareMaskLeI32}, - {Decoder::Opcode::V_CMPX_GT_I32, Opcode::CompareMaskGtI32}, - {Decoder::Opcode::V_CMPX_NE_I32, Opcode::CompareMaskNeI32}, - {Decoder::Opcode::V_CMPX_GE_I32, Opcode::CompareMaskGeI32}, - {Decoder::Opcode::V_CMP_LT_U16, Opcode::CompareLtU16}, - {Decoder::Opcode::V_CMP_EQ_U16, Opcode::CompareEqU16}, - {Decoder::Opcode::V_CMP_LE_U16, Opcode::CompareLeU16}, - {Decoder::Opcode::V_CMP_GT_U16, Opcode::CompareGtU16}, - {Decoder::Opcode::V_CMP_NE_U16, Opcode::CompareNeU16}, - {Decoder::Opcode::V_CMP_GE_U16, Opcode::CompareGeU16}, - {Decoder::Opcode::V_CMP_F_U32, Opcode::CompareFalse}, - {Decoder::Opcode::V_CMP_LT_U32, Opcode::CompareLtU32}, - {Decoder::Opcode::V_CMP_EQ_U32, Opcode::CompareEqU32}, - {Decoder::Opcode::V_CMP_LE_U32, Opcode::CompareLeU32}, - {Decoder::Opcode::V_CMP_GT_U32, Opcode::CompareGtU32}, - {Decoder::Opcode::V_CMP_NE_U32, Opcode::CompareNeU32}, - {Decoder::Opcode::V_CMP_GE_U32, Opcode::CompareGeU32}, - {Decoder::Opcode::V_CMP_T_U32, Opcode::CompareTrue}, - {Decoder::Opcode::V_CMP_EQ_I64, Opcode::CompareEqU64}, - {Decoder::Opcode::V_CMP_GT_U64, Opcode::CompareGtU64}, - {Decoder::Opcode::V_CMP_NE_U64, Opcode::CompareNeU64}, - {Decoder::Opcode::V_CMPX_NE_I64, Opcode::CompareNeU64}, - {Decoder::Opcode::V_CMPX_NE_U64, Opcode::CompareNeU64}, - {Decoder::Opcode::V_CMPX_LT_U32, Opcode::CompareMaskLtU32}, - {Decoder::Opcode::V_CMPX_EQ_U32, Opcode::CompareMaskEqU32}, - {Decoder::Opcode::V_CMPX_LE_U32, Opcode::CompareMaskLeU32}, - {Decoder::Opcode::V_CMPX_GT_U32, Opcode::CompareMaskGtU32}, - {Decoder::Opcode::V_CMPX_NE_U32, Opcode::CompareMaskNeU32}, - {Decoder::Opcode::V_CMPX_GE_U32, Opcode::CompareMaskGeU32}, - {Decoder::Opcode::S_LOAD_DWORD, Opcode::SLoadDword}, - {Decoder::Opcode::S_BUFFER_LOAD_DWORD, Opcode::SBufferLoadDword}, - {Decoder::Opcode::BUFFER_LOAD_DWORD, Opcode::BufferLoadDword}, - {Decoder::Opcode::BUFFER_LOAD_SBYTE, Opcode::BufferLoadSbyte}, - {Decoder::Opcode::BUFFER_LOAD_SSHORT, Opcode::BufferLoadSshort}, - {Decoder::Opcode::BUFFER_STORE_BYTE, Opcode::BufferStoreByte}, - {Decoder::Opcode::BUFFER_STORE_SHORT, Opcode::BufferStoreShort}, - {Decoder::Opcode::BUFFER_STORE_DWORD, Opcode::BufferStoreDword}, - {Decoder::Opcode::BUFFER_ATOMIC_SWAP, Opcode::AtomicSwapU32}, - {Decoder::Opcode::BUFFER_ATOMIC_ADD, Opcode::AtomicAddU32}, - {Decoder::Opcode::BUFFER_ATOMIC_SUB, Opcode::AtomicSubU32}, - {Decoder::Opcode::BUFFER_ATOMIC_SMIN, Opcode::AtomicSMinI32}, - {Decoder::Opcode::BUFFER_ATOMIC_UMIN, Opcode::AtomicUMinU32}, - {Decoder::Opcode::BUFFER_ATOMIC_SMAX, Opcode::AtomicSMaxI32}, - {Decoder::Opcode::BUFFER_ATOMIC_UMAX, Opcode::AtomicUMaxU32}, - {Decoder::Opcode::BUFFER_ATOMIC_AND, Opcode::AtomicAndU32}, - {Decoder::Opcode::BUFFER_ATOMIC_OR, Opcode::AtomicOrU32}, - {Decoder::Opcode::BUFFER_ATOMIC_XOR, Opcode::AtomicXorU32}, - {Decoder::Opcode::BUFFER_ATOMIC_FMIN, Opcode::AtomicFMinF32}, - {Decoder::Opcode::BUFFER_ATOMIC_FMAX, Opcode::AtomicFMaxF32}, - {Decoder::Opcode::FLAT_LOAD_UBYTE, Opcode::FlatLoadUbyte}, - {Decoder::Opcode::FLAT_LOAD_SBYTE, Opcode::FlatLoadSbyte}, - {Decoder::Opcode::FLAT_LOAD_SSHORT, Opcode::FlatLoadSshort}, - {Decoder::Opcode::FLAT_STORE_BYTE, Opcode::FlatStoreByte}, - {Decoder::Opcode::FLAT_STORE_SHORT, Opcode::FlatStoreShort}, - {Decoder::Opcode::FLAT_STORE_DWORD, Opcode::FlatStoreDword}, - {Decoder::Opcode::DS_ADD_U32, Opcode::AtomicAddU32}, - {Decoder::Opcode::DS_ADD_RTN_U32, Opcode::AtomicAddU32}, - {Decoder::Opcode::DS_SUB_U32, Opcode::AtomicSubU32}, - {Decoder::Opcode::DS_SUB_RTN_U32, Opcode::AtomicSubU32}, - {Decoder::Opcode::DS_MIN_I32, Opcode::AtomicSMinI32}, - {Decoder::Opcode::DS_MIN_RTN_I32, Opcode::AtomicSMinI32}, - {Decoder::Opcode::DS_MAX_I32, Opcode::AtomicSMaxI32}, - {Decoder::Opcode::DS_MAX_RTN_I32, Opcode::AtomicSMaxI32}, - {Decoder::Opcode::DS_MIN_U32, Opcode::AtomicUMinU32}, - {Decoder::Opcode::DS_MIN_RTN_U32, Opcode::AtomicUMinU32}, - {Decoder::Opcode::DS_MAX_U32, Opcode::AtomicUMaxU32}, - {Decoder::Opcode::DS_MAX_RTN_U32, Opcode::AtomicUMaxU32}, - {Decoder::Opcode::DS_AND_B32, Opcode::AtomicAndU32}, - {Decoder::Opcode::DS_AND_RTN_B32, Opcode::AtomicAndU32}, - {Decoder::Opcode::DS_OR_B32, Opcode::AtomicOrU32}, - {Decoder::Opcode::DS_OR_RTN_B32, Opcode::AtomicOrU32}, - {Decoder::Opcode::DS_XOR_B32, Opcode::AtomicXorU32}, - {Decoder::Opcode::DS_XOR_RTN_B32, Opcode::AtomicXorU32}, - {Decoder::Opcode::DS_WRXCHG_RTN_B32, Opcode::AtomicSwapU32}, - {Decoder::Opcode::DS_READ_I8, Opcode::DsReadSbyte}, - {Decoder::Opcode::DS_READ_U8, Opcode::DsReadUbyte}, - {Decoder::Opcode::DS_READ_I16, Opcode::DsReadSshort}, - {Decoder::Opcode::DS_READ_U16, Opcode::DsReadUshort}, - {Decoder::Opcode::DS_READ_U16_D16, Opcode::DsReadUshort}, - {Decoder::Opcode::DS_READ2_B32, Opcode::DsRead2B32}, - {Decoder::Opcode::DS_READ2ST64_B32, Opcode::DsRead2B32}, - {Decoder::Opcode::DS_READ_B32, Opcode::DsReadB32}, - {Decoder::Opcode::DS_READ_B64, Opcode::DsReadB32}, - {Decoder::Opcode::DS_READ2_B64, Opcode::DsRead2B32}, - {Decoder::Opcode::DS_READ2ST64_B64, Opcode::DsRead2B32}, - {Decoder::Opcode::DS_READ_B96, Opcode::DsReadB32}, - {Decoder::Opcode::DS_READ_B128, Opcode::DsReadB32}, - {Decoder::Opcode::DS_WRITE_B8, Opcode::DsWriteByte}, - {Decoder::Opcode::DS_WRITE_B16, Opcode::DsWriteShort}, - {Decoder::Opcode::DS_WRITE2_B32, Opcode::DsWrite2B32}, - {Decoder::Opcode::DS_WRITE2ST64_B32, Opcode::DsWrite2B32}, - {Decoder::Opcode::DS_WRITE2_B64, Opcode::DsWrite2B32}, - {Decoder::Opcode::DS_WRITE2ST64_B64, Opcode::DsWrite2B32}, - {Decoder::Opcode::DS_WRITE_B32, Opcode::DsWriteB32}, - {Decoder::Opcode::DS_WRITE_B64, Opcode::DsWriteB32}, - {Decoder::Opcode::DS_WRITE_B96, Opcode::DsWriteB32}, - {Decoder::Opcode::DS_WRITE_B128, Opcode::DsWriteB32}, - {Decoder::Opcode::DS_MIN_F32, Opcode::DsMinF32}, - {Decoder::Opcode::DS_MAX_F32, Opcode::DsMaxF32}, - {Decoder::Opcode::DS_CONSUME, Opcode::DsConsume}, - {Decoder::Opcode::DS_APPEND, Opcode::DsAppend}, - {Decoder::Opcode::DS_WRITE_ADDTID_B32, Opcode::DsWriteAddtidB32}, - {Decoder::Opcode::DS_READ_ADDTID_B32, Opcode::DsReadAddtidB32}, - {Decoder::Opcode::IMAGE_GET_RESINFO, Opcode::ImageGetResinfo}, - {Decoder::Opcode::IMAGE_GET_LOD, Opcode::ImageGetLod}, - {Decoder::Opcode::IMAGE_LOAD, Opcode::ImageLoad}, - {Decoder::Opcode::IMAGE_LOAD_MIP, Opcode::ImageLoad}, - {Decoder::Opcode::IMAGE_STORE, Opcode::ImageStore}, - {Decoder::Opcode::IMAGE_STORE_MIP, Opcode::ImageStore}, - {Decoder::Opcode::IMAGE_ATOMIC_ADD, Opcode::AtomicAddU32}, - {Decoder::Opcode::IMAGE_ATOMIC_UMIN, Opcode::AtomicUMinU32}, - {Decoder::Opcode::IMAGE_ATOMIC_UMAX, Opcode::AtomicUMaxU32}, - {Decoder::Opcode::IMAGE_ATOMIC_AND, Opcode::AtomicAndU32}, - {Decoder::Opcode::IMAGE_ATOMIC_OR, Opcode::AtomicOrU32}, - {Decoder::Opcode::IMAGE_ATOMIC_XOR, Opcode::AtomicXorU32}, - {Decoder::Opcode::IMAGE_SAMPLE, Opcode::ImageSample}, - {Decoder::Opcode::IMAGE_GATHER4_LZ, Opcode::ImageGather4}, - {Decoder::Opcode::IMAGE_GATHER4_C, Opcode::ImageGather4}, - {Decoder::Opcode::IMAGE_GATHER4_C_LZ, Opcode::ImageGather4}, - {Decoder::Opcode::IMAGE_GATHER4_LZ_O, Opcode::ImageGather4}, - {Decoder::Opcode::IMAGE_GATHER4_C_O, Opcode::ImageGather4}, - {Decoder::Opcode::IMAGE_GATHER4_C_LZ_O, Opcode::ImageGather4}, - {Decoder::Opcode::IMAGE_GATHER4H, Opcode::ImageGather4}, -}; - -} // namespace - -void SetError(std::string* error, const char* message) { - if (error != nullptr) { - *error = message; - } -} - -std::optional LookupIrOpcode(Decoder::Opcode opcode) { - for (const auto& op: LOWER_OPS) { - if (op.decoded == opcode) { - return op.ir; - } - } - return std::nullopt; -} - -bool IsReversedBinary(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::V_SUBREV_F32: - case Decoder::Opcode::V_SUBREV_F16: - case Decoder::Opcode::V_SUBREV_NC_U32: - case Decoder::Opcode::V_LSHLREV_B32: - case Decoder::Opcode::V_LSHRREV_B32: - case Decoder::Opcode::V_ASHRREV_I32: - case Decoder::Opcode::V_LSHLREV_B64: - case Decoder::Opcode::V_LSHRREV_B64: - case Decoder::Opcode::V_LSHLREV_B16: - case Decoder::Opcode::V_LSHRREV_B16: - case Decoder::Opcode::V_ASHRREV_I16: - case Decoder::Opcode::V_SUBREV_I32: return true; - default: return false; - } -} - -bool IsVectorCarryOutOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::V_ADD_I32: - case Decoder::Opcode::V_SUB_I32: - case Decoder::Opcode::V_SUBREV_I32: return true; - default: return false; - } -} - -bool ScalarResultWritesSccNonZero(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::S_ABS_I32: - case Decoder::Opcode::S_BCNT1_I32_B32: - case Decoder::Opcode::S_BCNT1_I32_B64: - case Decoder::Opcode::S_AND_B32: - case Decoder::Opcode::S_ANDN2_B32: - case Decoder::Opcode::S_OR_B32: - case Decoder::Opcode::S_ORN2_B32: - case Decoder::Opcode::S_XOR_B32: - case Decoder::Opcode::S_NAND_B32: - case Decoder::Opcode::S_NOR_B32: - case Decoder::Opcode::S_XNOR_B32: - case Decoder::Opcode::S_NOT_B32: - case Decoder::Opcode::S_LSHL_B32: - case Decoder::Opcode::S_LSHR_B32: - case Decoder::Opcode::S_ASHR_I32: - case Decoder::Opcode::S_BFE_U32: - case Decoder::Opcode::S_BFE_I32: - case Decoder::Opcode::S_AND_B64: - case Decoder::Opcode::S_ANDN2_B64: - case Decoder::Opcode::S_NOT_B64: - case Decoder::Opcode::S_OR_B64: - case Decoder::Opcode::S_ORN2_B64: - case Decoder::Opcode::S_XOR_B64: - case Decoder::Opcode::S_NAND_B64: - case Decoder::Opcode::S_NOR_B64: - case Decoder::Opcode::S_XNOR_B64: - case Decoder::Opcode::S_LSHL_B64: - case Decoder::Opcode::S_LSHR_B64: - case Decoder::Opcode::S_BFE_U64: - case Decoder::Opcode::S_WQM_B64: - case Decoder::Opcode::S_QUADMASK_B64: return true; - default: return false; - } -} - -bool ScalarResultIs64Bit(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::S_AND_B64: - case Decoder::Opcode::S_ANDN2_B64: - case Decoder::Opcode::S_NOT_B64: - case Decoder::Opcode::S_OR_B64: - case Decoder::Opcode::S_ORN2_B64: - case Decoder::Opcode::S_XOR_B64: - case Decoder::Opcode::S_NAND_B64: - case Decoder::Opcode::S_NOR_B64: - case Decoder::Opcode::S_XNOR_B64: - case Decoder::Opcode::S_LSHL_B64: - case Decoder::Opcode::S_LSHR_B64: - case Decoder::Opcode::S_BFE_U64: - case Decoder::Opcode::S_WQM_B64: - case Decoder::Opcode::S_QUADMASK_B64: return true; - default: return false; - } -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc deleted file mode 100644 index a48f88155..000000000 --- a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc +++ /dev/null @@ -1,354 +0,0 @@ -// Semantic operations produced by RDNA2 instruction lowering. The typed frontend consumes this -// baseline set before optimization and resource planning; it must not grow without decoder support. -IR_OPCODE(ControlNop, General, Control, Void) -IR_OPCODE(Waitcnt, General, Control, Void) -IR_OPCODE(Barrier, General, Control, Void) -IR_OPCODE(Sendmsg, General, Control, Void) -IR_OPCODE(TtraceData, General, Control, Void) -IR_OPCODE(InstPrefetch, General, Control, Void) -IR_OPCODE(MoveU32, General, Move) -IR_OPCODE(MoveF32Bits, General, Move) -IR_OPCODE(MoveRelDestU32, General, Lane) -IR_OPCODE(MoveRelSourceU32, General, Lane) -IR_OPCODE(MoveU64, General, Move) -IR_OPCODE(WqmB64, General, Move) -IR_OPCODE(SaveexecB32, General, State) -IR_OPCODE(SaveexecB64, General, State) -IR_OPCODE(ReadFirstLaneU32, General, Lane) -IR_OPCODE(ReadLaneU32, General, Lane) -IR_OPCODE(WriteLaneU32, General, Lane) -IR_OPCODE(Permlane16B32, General, Lane) -IR_OPCODE(Permlanex16B32, General, Lane) -IR_OPCODE(AbsI32, General, SimpleInteger) -IR_OPCODE(IAddU32, General, SimpleInteger) -IR_OPCODE(IAddCarryU32, General, State) -IR_OPCODE(ISubBorrowCarryU32, General, State) -IR_OPCODE(ISubBorrowU32, General, State) -IR_OPCODE(ScalarAddCarryU32, General, State) -IR_OPCODE(ScalarSubBorrowU32, General, State) -IR_OPCODE(ScalarSubBorrowCarryU32, General, State) -IR_OPCODE(ScalarSignedAddOverflowI32, General, State) -IR_OPCODE(ScalarSignedSubOverflowI32, General, State) -IR_OPCODE(ScalarShiftLeftAddCarryU32, General, State) -IR_OPCODE(ISubU32, General, SimpleInteger) -IR_OPCODE(IMulU32, General, SimpleInteger) -IR_OPCODE(UMulHighU32, General, SimpleInteger) -IR_OPCODE(SMulHighI32, General, SimpleInteger) -IR_OPCODE(IMadI24U32, General, ExtendedInteger) -IR_OPCODE(UMadU24U32, General, ExtendedInteger) -IR_OPCODE(UMadU64U32, General, ExtendedInteger) -IR_OPCODE(SadU32, General, ExtendedInteger) -IR_OPCODE(IAdd3U32, General, ExtendedInteger) -IR_OPCODE(IMulI24U32, General, ExtendedInteger) -IR_OPCODE(UMulU24U32, General, ExtendedInteger) -IR_OPCODE(IMinI32, General, SimpleInteger) -IR_OPCODE(IMaxI32, General, SimpleInteger) -IR_OPCODE(IMin3I32, General, SimpleInteger) -IR_OPCODE(IMax3I32, General, SimpleInteger) -IR_OPCODE(IMed3I32, General, SimpleInteger) -IR_OPCODE(UMinU32, General, SimpleInteger) -IR_OPCODE(UMaxU32, General, SimpleInteger) -IR_OPCODE(UMin3U32, General, SimpleInteger) -IR_OPCODE(UMax3U32, General, SimpleInteger) -IR_OPCODE(UMed3U32, General, SimpleInteger) -IR_OPCODE(BitwiseAndU32, General, SimpleInteger) -IR_OPCODE(BitwiseAndU64, General, U64Mask) -IR_OPCODE(BitwiseAndNotU32, General, ComposedInteger) -IR_OPCODE(BitwiseAndNotU64, General, U64Mask) -IR_OPCODE(BitwiseOrU32, General, SimpleInteger) -IR_OPCODE(BitwiseOrU64, General, U64Mask) -IR_OPCODE(BitwiseOrNotU32, General, ComposedInteger) -IR_OPCODE(BitwiseOrNotU64, General, U64Mask) -IR_OPCODE(BitwiseAndOrU32, General, ComposedInteger) -IR_OPCODE(BitwiseOr3U32, General, ComposedInteger) -IR_OPCODE(BitwiseXorU32, General, SimpleInteger) -IR_OPCODE(BitwiseXorU64, General, U64Mask) -IR_OPCODE(BitwiseXor3U32, General, ComposedInteger) -IR_OPCODE(BitwiseNandU32, General, ComposedInteger) -IR_OPCODE(BitwiseNandU64, General, U64Mask) -IR_OPCODE(BitwiseNorU32, General, ComposedInteger) -IR_OPCODE(BitwiseNorU64, General, U64Mask) -IR_OPCODE(BitwiseXnorU32, General, ComposedInteger) -IR_OPCODE(BitwiseXnorU64, General, U64Mask) -IR_OPCODE(BitwiseNotU32, General, SimpleInteger) -IR_OPCODE(BitwiseNotU64, General, U64Mask) -IR_OPCODE(BitClearU32, General, ExtendedInteger) -IR_OPCODE(BitSetU32, General, ExtendedInteger) -IR_OPCODE(BitReverseU32, General, SimpleInteger) -IR_OPCODE(BitCountU32, General, SimpleInteger) -IR_OPCODE(BitCountU64, General, SimpleInteger) -IR_OPCODE(BitReplicateB64B32, General, ExtendedInteger) -IR_OPCODE(QuadmaskB64, General, ExtendedInteger) -IR_OPCODE(BitCountAddU32, General, ExtendedInteger) -IR_OPCODE(MaskedBitCountLowU32, General, ExtendedInteger) -IR_OPCODE(MaskedBitCountHighU32, General, ExtendedInteger) -IR_OPCODE(FindLsbU32, General, SimpleInteger) -IR_OPCODE(FindLsbU64, General, ComposedInteger) -IR_OPCODE(FindMsbFromHighU32, General, ComposedInteger) -IR_OPCODE(FindMsbFromHighU64, General, ComposedInteger) -IR_OPCODE(BitFieldMaskU32, General, ExtendedInteger) -IR_OPCODE(BitFieldMaskU64, General, ExtendedInteger) -IR_OPCODE(BitFieldExtractU32, General, ExtendedInteger) -IR_OPCODE(BitFieldExtractI32, General, ExtendedInteger) -IR_OPCODE(BitFieldExtractU64, General, ExtendedInteger) -IR_OPCODE(BitFieldExtract3U32, General, ExtendedInteger) -IR_OPCODE(BitFieldExtract3I32, General, ExtendedInteger) -IR_OPCODE(BitFieldInsertSelectU32, General, ExtendedInteger) -IR_OPCODE(BitCompare0B32, General, ExtendedInteger) -IR_OPCODE(BitCompare1B32, General, ExtendedInteger) -IR_OPCODE(AlignBitU32, General, ExtendedInteger) -IR_OPCODE(AlignByteU32, General, ExtendedInteger) -IR_OPCODE(ShiftLeftAddU32, General, ExtendedInteger) -IR_OPCODE(AddShiftLeftU32, General, ExtendedInteger) -IR_OPCODE(XorAddU32, General, ExtendedInteger) -IR_OPCODE(ShiftLeftOrU32, General, ExtendedInteger) -IR_OPCODE(ShiftLeftLogicalU32, General, SimpleInteger) -IR_OPCODE(ShiftLeftLogicalU64, General, SimpleInteger) -IR_OPCODE(ShiftLeftLogicalU16, General, Integer16) -IR_OPCODE(ShiftRightLogicalU32, General, SimpleInteger) -IR_OPCODE(ShiftRightLogicalU64, General, SimpleInteger) -IR_OPCODE(ShiftRightLogicalU16, General, Integer16) -IR_OPCODE(ShiftRightArithmeticI32, General, SimpleInteger) -IR_OPCODE(ShiftRightArithmeticI16, General, Integer16) -IR_OPCODE(SelectU32, General, ExtendedInteger, U32, U1, U32, U32) -IR_OPCODE(SelectMaskU32, General, ExtendedInteger) -IR_OPCODE(SelectF32Bits, General, ExtendedInteger) -IR_OPCODE(SelectMaskF32Bits, General, ExtendedInteger) -IR_OPCODE(SelectU64, General, ExtendedInteger) -IR_OPCODE(PackLowLowU16, General, ExtendedInteger) -IR_OPCODE(PackLowHighU16, General, ExtendedInteger) -IR_OPCODE(PackHighHighU16, General, ExtendedInteger) -IR_OPCODE(CompareFalse, Compare, IntegerCompare) -IR_OPCODE(CompareTrue, Compare, IntegerCompare) -IR_OPCODE(CompareEqU32, Compare, IntegerCompare) -IR_OPCODE(CompareNeU32, Compare, IntegerCompare) -IR_OPCODE(CompareGtU32, Compare, IntegerCompare) -IR_OPCODE(CompareGeU32, Compare, IntegerCompare) -IR_OPCODE(CompareLtU32, Compare, IntegerCompare) -IR_OPCODE(CompareLeU32, Compare, IntegerCompare) -IR_OPCODE(CompareEqU64, Compare64, IntegerCompare) -IR_OPCODE(CompareGtU64, Compare64, IntegerCompare) -IR_OPCODE(CompareNeU64, Compare64, IntegerCompare) -IR_OPCODE(CompareMaskEqU32, Compare, IntegerCompare) -IR_OPCODE(CompareMaskNeU32, Compare, IntegerCompare) -IR_OPCODE(CompareMaskGtU32, Compare, IntegerCompare) -IR_OPCODE(CompareMaskGeU32, Compare, IntegerCompare) -IR_OPCODE(CompareMaskLtU32, Compare, IntegerCompare) -IR_OPCODE(CompareMaskLeU32, Compare, IntegerCompare) -IR_OPCODE(CompareEqI32, Compare, IntegerCompare) -IR_OPCODE(CompareNeI32, Compare, IntegerCompare) -IR_OPCODE(CompareGtI32, Compare, IntegerCompare) -IR_OPCODE(CompareGeI32, Compare, IntegerCompare) -IR_OPCODE(CompareLtI32, Compare, IntegerCompare) -IR_OPCODE(CompareLeI32, Compare, IntegerCompare) -IR_OPCODE(CompareEqI16, Compare, Integer16Compare) -IR_OPCODE(CompareNeI16, Compare, Integer16Compare) -IR_OPCODE(CompareGtI16, Compare, Integer16Compare) -IR_OPCODE(CompareGeI16, Compare, Integer16Compare) -IR_OPCODE(CompareLtI16, Compare, Integer16Compare) -IR_OPCODE(CompareLeI16, Compare, Integer16Compare) -IR_OPCODE(CompareMaskEqI32, Compare, IntegerCompare) -IR_OPCODE(CompareMaskNeI32, Compare, IntegerCompare) -IR_OPCODE(CompareMaskGtI32, Compare, IntegerCompare) -IR_OPCODE(CompareMaskGeI32, Compare, IntegerCompare) -IR_OPCODE(CompareMaskLtI32, Compare, IntegerCompare) -IR_OPCODE(CompareMaskLeI32, Compare, IntegerCompare) -IR_OPCODE(CompareEqU16, Compare, Integer16Compare) -IR_OPCODE(CompareNeU16, Compare, Integer16Compare) -IR_OPCODE(CompareGtU16, Compare, Integer16Compare) -IR_OPCODE(CompareGeU16, Compare, Integer16Compare) -IR_OPCODE(CompareLtU16, Compare, Integer16Compare) -IR_OPCODE(CompareLeU16, Compare, Integer16Compare) -IR_OPCODE(CompareEqF32, Compare, FloatCompare) -IR_OPCODE(CompareNeF32, Compare, FloatCompare) -IR_OPCODE(CompareGtF32, Compare, FloatCompare) -IR_OPCODE(CompareGeF32, Compare, FloatCompare) -IR_OPCODE(CompareLtF32, Compare, FloatCompare) -IR_OPCODE(CompareLeF32, Compare, FloatCompare) -IR_OPCODE(CompareOrderedF32, Compare, FloatCompare) -IR_OPCODE(CompareUnorderedF32, Compare, FloatCompare) -IR_OPCODE(CompareUnordEqF32, Compare, FloatCompare) -IR_OPCODE(CompareUnordNeF32, Compare, FloatCompare) -IR_OPCODE(CompareUnordGtF32, Compare, FloatCompare) -IR_OPCODE(CompareUnordGeF32, Compare, FloatCompare) -IR_OPCODE(CompareUnordLtF32, Compare, FloatCompare) -IR_OPCODE(CompareUnordLeF32, Compare, FloatCompare) -IR_OPCODE(CompareClassF32, Compare, FloatCompare) -IR_OPCODE(CompareEqF16, Compare, FloatCompare) -IR_OPCODE(CompareNeF16, Compare, FloatCompare) -IR_OPCODE(CompareGtF16, Compare, FloatCompare) -IR_OPCODE(CompareGeF16, Compare, FloatCompare) -IR_OPCODE(CompareLtF16, Compare, FloatCompare) -IR_OPCODE(CompareLeF16, Compare, FloatCompare) -IR_OPCODE(CompareUnordNeF16, Compare, FloatCompare) -IR_OPCODE(CompareMaskEqF16, Compare, FloatCompare) -IR_OPCODE(CompareMaskNeF16, Compare, FloatCompare) -IR_OPCODE(CompareMaskGtF16, Compare, FloatCompare) -IR_OPCODE(CompareMaskGeF16, Compare, FloatCompare) -IR_OPCODE(CompareMaskLtF16, Compare, FloatCompare) -IR_OPCODE(CompareMaskLeF16, Compare, FloatCompare) -IR_OPCODE(CompareMaskUnordNeF16, Compare, FloatCompare) -IR_OPCODE(CompareMaskUnordGeF16, Compare, FloatCompare) -IR_OPCODE(CompareMaskEqF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskNeF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskGtF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskGeF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskLtF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskLeF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskUnordEqF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskUnordNeF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskUnordGtF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskUnordGeF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskUnordLtF32, Compare, FloatCompare) -IR_OPCODE(CompareMaskUnordLeF32, Compare, FloatCompare) -IR_OPCODE(ConvertByteU32ToF32, General, Conversion) -IR_OPCODE(ConvertU32ToF32, General, Conversion) -IR_OPCODE(ConvertI32ToF32, General, Conversion) -IR_OPCODE(ConvertF32ToU32, General, Conversion) -IR_OPCODE(ConvertF32ToI32, General, Conversion) -IR_OPCODE(ConvertF32ToF16, General, Conversion) -IR_OPCODE(ConvertF16ToF32, General, Conversion) -IR_OPCODE(ConvertU16ToF16, General, Conversion) -IR_OPCODE(ConvertF16ToU16, General, Conversion) -IR_OPCODE(ConvertI16ToF16, General, Conversion) -IR_OPCODE(ConvertF16ToI16, General, Conversion) -IR_OPCODE(ConvertRoundPlusInfF32ToI32, General, Conversion) -IR_OPCODE(ConvertFloorF32ToI32, General, Conversion) -IR_OPCODE(FrexpExpI32F32, General, Conversion) -IR_OPCODE(FrexpMantF32, General, Float) -IR_OPCODE(ConvertI4ToOffsetF32, General, Conversion) -IR_OPCODE(LdexpF32, General, Float) -IR_OPCODE(PackF32ToF16Rtz, General, Conversion) -IR_OPCODE(PackSnorm2x16F32, General, Conversion) -IR_OPCODE(PackUnorm2x16F32, General, Conversion) -IR_OPCODE(PackU16U32, General, ExtendedInteger) -IR_OPCODE(PackU8F32, General, Conversion) -IR_OPCODE(PackB32F16, General, Conversion) -IR_OPCODE(PackedMadI16, General, PackedInteger16) -IR_OPCODE(PackedMulLoU16, General, PackedInteger16) -IR_OPCODE(PackedAddI16, General, PackedInteger16) -IR_OPCODE(PackedSubI16, General, PackedInteger16) -IR_OPCODE(PackedLshlrevB16, General, PackedInteger16) -IR_OPCODE(PackedLshrrevB16, General, PackedInteger16) -IR_OPCODE(PackedAshrrevI16, General, PackedInteger16) -IR_OPCODE(PackedMaxI16, General, PackedInteger16) -IR_OPCODE(PackedMinI16, General, PackedInteger16) -IR_OPCODE(PackedMadU16, General, PackedInteger16) -IR_OPCODE(PackedAddU16, General, PackedInteger16) -IR_OPCODE(PackedSubU16, General, PackedInteger16) -IR_OPCODE(PackedMaxU16, General, PackedInteger16) -IR_OPCODE(PackedMinU16, General, PackedInteger16) -IR_OPCODE(PackedAddF16, General, PackedFloat16) -IR_OPCODE(PackedMulF16, General, PackedFloat16) -IR_OPCODE(PackedMinF16, General, PackedFloat16) -IR_OPCODE(PackedMaxF16, General, PackedFloat16) -IR_OPCODE(PackedFmaF16, General, PackedFloat16) -IR_OPCODE(AddF16, General, Float16) -IR_OPCODE(SubF16, General, Float16) -IR_OPCODE(MulF16, General, Float16) -IR_OPCODE(MinF16, General, Float16) -IR_OPCODE(MaxF16, General, Float16) -IR_OPCODE(FmaF16, General, Float16) -IR_OPCODE(MadMixF16, General, Float16) -IR_OPCODE(IAddU16, General, Integer16) -IR_OPCODE(ISubI16, General, Integer16) -IR_OPCODE(IMinI16, General, Integer16) -IR_OPCODE(IMaxI16, General, Integer16) -IR_OPCODE(IMed3I16, General, Integer16) -IR_OPCODE(UMinU16, General, Integer16) -IR_OPCODE(UMaxU16, General, Integer16) -IR_OPCODE(RcpF32, General, Float) -IR_OPCODE(RcpIflagF32, General, Float) -IR_OPCODE(FractF32, General, Float) -IR_OPCODE(TruncF32, General, Float) -IR_OPCODE(CeilF32, General, Float) -IR_OPCODE(RoundEvenF32, General, Float) -IR_OPCODE(FloorF32, General, Float) -IR_OPCODE(Exp2F32, General, Float) -IR_OPCODE(Log2F32, General, Float) -IR_OPCODE(InverseSqrtF32, General, Float) -IR_OPCODE(SqrtF32, General, Float) -IR_OPCODE(RcpF16, General, Float16) -IR_OPCODE(SqrtF16, General, Float16) -IR_OPCODE(InverseSqrtF16, General, Float16) -IR_OPCODE(Log2F16, General, Float16) -IR_OPCODE(Exp2F16, General, Float16) -IR_OPCODE(FloorF16, General, Float16) -IR_OPCODE(CeilF16, General, Float16) -IR_OPCODE(TruncF16, General, Float16) -IR_OPCODE(RoundEvenF16, General, Float16) -IR_OPCODE(SinF32, General, Float) -IR_OPCODE(CosF32, General, Float) -IR_OPCODE(CubeIdF32, General, Float) -IR_OPCODE(CubeScF32, General, Float) -IR_OPCODE(CubeTcF32, General, Float) -IR_OPCODE(CubeMaF32, General, Float) -IR_OPCODE(FAddF32, General, Float) -IR_OPCODE(FSubF32, General, Float) -IR_OPCODE(FMulF32, General, Float) -IR_OPCODE(FMinF32, General, Float) -IR_OPCODE(FMaxF32, General, Float) -IR_OPCODE(FMadF32, General, Float) -IR_OPCODE(Dot2AccF32F16, General, Float) -IR_OPCODE(FMin3F32, General, Float) -IR_OPCODE(FMax3F32, General, Float) -IR_OPCODE(FMed3F32, General, Float) -IR_OPCODE(Min3F16, General, Float16) -IR_OPCODE(Max3F16, General, Float16) -IR_OPCODE(Med3F16, General, Float16) -IR_OPCODE(LoadSrtDword, General, Memory) -IR_OPCODE(SLoadDword, General, Memory) -IR_OPCODE(SBufferLoadDword, General, Memory) -IR_OPCODE(BufferLoadUbyte, General, Memory) -IR_OPCODE(BufferLoadSbyte, General, Memory) -IR_OPCODE(BufferLoadUshort, General, Memory) -IR_OPCODE(BufferLoadSshort, General, Memory) -IR_OPCODE(BufferLoadDword, General, Memory) -IR_OPCODE(BufferStoreByte, General, Memory) -IR_OPCODE(BufferStoreShort, General, Memory) -IR_OPCODE(BufferStoreDword, General, Memory) -IR_OPCODE(AtomicSwapU32, General, Memory) -IR_OPCODE(AtomicAddU32, General, Memory) -IR_OPCODE(AtomicSubU32, General, Memory) -IR_OPCODE(AtomicSMinI32, General, Memory) -IR_OPCODE(AtomicUMinU32, General, Memory) -IR_OPCODE(AtomicSMaxI32, General, Memory) -IR_OPCODE(AtomicUMaxU32, General, Memory) -IR_OPCODE(AtomicAndU32, General, Memory) -IR_OPCODE(AtomicOrU32, General, Memory) -IR_OPCODE(AtomicXorU32, General, Memory) -IR_OPCODE(AtomicFMinF32, General, Memory) -IR_OPCODE(AtomicFMaxF32, General, Memory) -IR_OPCODE(FlatLoadUbyte, General, Memory) -IR_OPCODE(FlatLoadSbyte, General, Memory) -IR_OPCODE(FlatLoadUshort, General, Memory) -IR_OPCODE(FlatLoadSshort, General, Memory) -IR_OPCODE(FlatLoadDword, General, Memory) -IR_OPCODE(FlatStoreByte, General, Memory) -IR_OPCODE(FlatStoreShort, General, Memory) -IR_OPCODE(FlatStoreDword, General, Memory) -IR_OPCODE(DsReadUbyte, General, Memory) -IR_OPCODE(DsReadSbyte, General, Memory) -IR_OPCODE(DsReadUshort, General, Memory) -IR_OPCODE(DsReadSshort, General, Memory) -IR_OPCODE(DsReadB32, General, Memory) -IR_OPCODE(DsRead2B32, General, Memory) -IR_OPCODE(DsWriteByte, General, Memory) -IR_OPCODE(DsWriteShort, General, Memory) -IR_OPCODE(DsWriteB32, General, Memory) -IR_OPCODE(DsWrite2B32, General, Memory) -IR_OPCODE(DsMinF32, General, Memory) -IR_OPCODE(DsMaxF32, General, Memory) -IR_OPCODE(DsSwizzleB32, General, Memory) -IR_OPCODE(DsConsume, General, Memory) -IR_OPCODE(DsAppend, General, Memory) -IR_OPCODE(DsWriteAddtidB32, General, Memory) -IR_OPCODE(DsReadAddtidB32, General, Memory) -IR_OPCODE(ImageGetResinfo, General, Memory) -IR_OPCODE(ImageGetLod, General, Memory) -IR_OPCODE(ImageLoad, General, Memory) -IR_OPCODE(ImageStore, General, Memory) -IR_OPCODE(ImageSample, General, Memory) -IR_OPCODE(ImageGather4, General, Memory) -IR_OPCODE(LoadInputF32, General, Attribute) -IR_OPCODE(Export, General, Attribute) diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp index 9680eaea9..d0f2a2192 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp @@ -1,6 +1,6 @@ #include "graphics/shader/recompiler/ir/passes/BindingLayout.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include #include @@ -16,7 +16,7 @@ constexpr uint32_t ImageBindingCount = bool CollectUserData(const Program& program, std::vector& result) { std::array registers {}; - for (const auto* block: program.values->blocks) { + for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (inst.GetOpcode() != ValueOpcode::GetUserData || !inst.HasUses()) { continue; @@ -47,16 +47,16 @@ void AddBinding(BindingLayout& layout, DescriptorBindingKind kind, bool UsesGds(const Program& program, bool& uses_gds) { uses_gds = false; - for (const auto* block: program.values->blocks) { + for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (SharedAccessOf(inst.GetOpcode()) == SharedAccess::None) { continue; } const auto index = inst.Flags().index; - if (index >= program.values->memory_info.size()) { + if (index >= program.memory_info.size()) { return false; } - const auto kind = program.values->memory_info[index].kind; + const auto kind = program.memory_info[index].kind; if (kind != ResourceKind::Lds && kind != ResourceKind::Gds) { return false; } @@ -76,12 +76,6 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri } return false; } - if (program.values == nullptr) { - if (error != nullptr) { - *error = "typed value program is not available"; - } - return false; - } if (push_constant_offset > NativePushConstantSize || push_constant_offset % sizeof(uint32_t) != 0) { if (error != nullptr) { @@ -171,7 +165,7 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri AddBinding(next, DescriptorBindingKind::AddressMemory, std::move(resources)); } const bool uses_flattened_runtime = - !program.values->srt_reads.empty() || + !program.srt_reads.empty() || std::ranges::any_of(program.info.images, [](const ImageResource& image) { return image.indirect_mapping_capacity != 0u; }); diff --git a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp index d9fa76921..6ba46a853 100644 --- a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp @@ -101,7 +101,7 @@ Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_s } // namespace -ReadLaneStats EliminateReadLane(ValueProgram& program, uint32_t wave_size) { +ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size) { ReadLaneStats stats; if (wave_size != 32u && wave_size != 64u) { return stats; diff --git a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h index 487a43a23..6e6a4671c 100644 --- a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h +++ b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h @@ -1,6 +1,6 @@ #pragma once -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" namespace Libs::Graphics::ShaderRecompiler::IR { @@ -8,6 +8,6 @@ struct ReadLaneStats { uint32_t rewritten_reads = 0; }; -[[nodiscard]] ReadLaneStats EliminateReadLane(ValueProgram& program, uint32_t wave_size); +[[nodiscard]] ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 14efdc560..7248281d4 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -1,7 +1,7 @@ #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/guest_gpu/gpu_format.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/shaderBindings.h" #include @@ -105,10 +105,10 @@ bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResou } const DescriptorSource* Source(const Program& program, uint32_t source) { - if (program.values == nullptr || source >= program.values->descriptor_sources.size()) { + if (source >= program.descriptor_sources.size()) { return nullptr; } - return &program.values->descriptor_sources[source]; + return &program.descriptor_sources[source]; } void MarkCleanFlatSlots(const Program& program, const DescriptorSource* source, @@ -131,7 +131,7 @@ void MarkCleanFlatSlots(const Program& program, const DescriptorSource* source, const auto slot = inst->Arg(1).Resolve(); if (slot.IsImmediate() && slot.GetType() == Type::U32 && slot.U32() < slots.size()) { slots[slot.U32()] = 1u; - pending.push_back(program.values->srt_reads[slot.U32()].value); + pending.push_back(program.srt_reads[slot.U32()].value); } continue; } @@ -275,7 +275,7 @@ uint64_t AddressSpecialization(const AddressResource& resource, } size_t FlattenedRuntimeDwords(const Program& program) { - size_t size = program.values != nullptr ? program.values->srt_reads.size() : 0u; + size_t size = program.srt_reads.size(); for (uint32_t resource = 0; resource < program.info.images.size(); resource++) { const auto& image = program.info.images[resource]; if (image.indirect_root == resource) { @@ -305,8 +305,7 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn } return false; } - if (program.values == nullptr || - snapshot.flattened_srt.size() != FlattenedRuntimeDwords(program)) { + if (snapshot.flattened_srt.size() != FlattenedRuntimeDwords(program)) { if (error != nullptr) { *error = "flattened SRT snapshot does not match the shader plan"; } @@ -549,7 +548,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime, } std::vector requests; - std::vector clean_flat_slots(program.values->srt_reads.size()); + std::vector clean_flat_slots(program.srt_reads.size()); requests.reserve(program.info.buffers.size() + program.info.images.size() * 2u + program.info.samplers.size() + program.info.addresses.size()); for (const auto& buffer: program.info.buffers) { @@ -925,13 +924,7 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri } } } - if (program.values == nullptr) { - if (error != nullptr) { - *error = "typed SSA is not ready"; - } - return false; - } - auto memory_info = program.values->memory_info; + auto memory_info = program.memory_info; struct SamplerUsage { bool native = false; @@ -980,7 +973,7 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri pair.sampler = point_sampler[pair.sampler]; } } - for (const auto* block: program.values->blocks) { + for (const auto* block: program.blocks) { for (const auto& inst: *block) { const auto image_opcode = ImageOpcodeInfoOf(inst.GetOpcode()); if (image_opcode.access == ImageAccess::None) { @@ -1023,7 +1016,7 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri } } program.info = std::move(next); - program.values->memory_info = std::move(memory_info); + program.memory_info = std::move(memory_info); snapshot = std::move(next_snapshot); return true; } diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index a9e62693e..390c2b3c4 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -1,6 +1,6 @@ #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include "graphics/shader/shader.h" @@ -94,8 +94,7 @@ ImageMipMode MipMode(const MemoryInfo& memory) { class Tracker { public: - Tracker(Program& program, ValueProgram& values) - : m_program(program), m_values(values), m_info(program.info) { + explicit Tracker(Program& program): m_program(program), m_info(program.info) { m_info.buffers.clear(); m_info.addresses.clear(); m_info.images.clear(); @@ -111,7 +110,7 @@ public: return Fail(0, error, "SRT plan is not ready"); } PlanIndirectImages(); - for (auto* block: m_values.blocks) { + for (auto* block: m_program.blocks) { for (auto& inst: *block) { if (!Collect(inst, error)) { return false; @@ -123,7 +122,7 @@ public: patch.handle->SetFlags(patch.resource); } for (const auto& patch: m_memory_patches) { - auto& memory = m_values.memory_info[patch.index]; + auto& memory = m_program.memory_info[patch.index]; memory.resource = patch.resource; if (patch.has_sampler) { memory.sampler = patch.sampler; @@ -138,16 +137,16 @@ public: plan.handle->SetArg(dword, plan.key); } for (const auto index: plan.memory) { - m_values.memory_info[index].planning_only = true; + m_program.memory_info[index].planning_only = true; } } - std::erase_if(m_values.dynamic_reads, [&](Value value) { + std::erase_if(m_program.dynamic_reads, [&](Value value) { const auto* inst = value.Resolve().TryInstruction(); return std::ranges::any_of(m_indirect_images, [&](const IndirectImagePlan& plan) { return std::ranges::find(plan.reads, inst) != plan.reads.end(); }); }); - m_values.descriptor_sources = std::move(m_sources); + m_program.descriptor_sources = std::move(m_sources); m_program.info = std::move(m_info); m_program.resource_tracking_complete = true; return true; @@ -208,19 +207,19 @@ private: if (left.value.IsEmpty() || right.value.IsEmpty()) { return require_both ? AddressPart {} : (left.value.IsEmpty() ? right : left); } - return EquivalentValue(m_values, left.value, right.value) + return EquivalentValue(m_program, left.value, right.value) ? AddressPart {left.value, left.rooted || right.rooted} : AddressPart {}; }; switch (inst->GetOpcode()) { case ValueOpcode::SelectU32: - if (EquivalentValue(m_values, inst->Arg(0), active)) { + if (EquivalentValue(m_program, inst->Arg(0), active)) { return finish(FindAddressPart(inst->Arg(1), active, visiting)); } return finish(merge(FindAddressPart(inst->Arg(1), active, visiting), FindAddressPart(inst->Arg(2), active, visiting), true)); case ValueOpcode::Phi: { - const auto invariant = ResolveInvariantPhi(m_values, value); + const auto invariant = ResolveInvariantPhi(m_program, value); return finish(invariant.IsEmpty() ? AddressPart {} : FindAddressPart(invariant, active, visiting)); } @@ -312,7 +311,7 @@ private: } bool same = true; for (uint32_t i = 0; i < descriptor.dword_count; i++) { - same = same && EquivalentValue(m_values, current.dwords[i], descriptor.dwords[i]); + same = same && EquivalentValue(m_program, current.dwords[i], descriptor.dwords[i]); } if (same) { return candidate; @@ -342,10 +341,10 @@ private: return nullptr; } index = read.Flags().index; - if (index >= m_values.memory_info.size()) { + if (index >= m_program.memory_info.size()) { return nullptr; } - const auto& memory = m_values.memory_info[index]; + const auto& memory = m_program.memory_info[index]; return memory.kind == ResourceKind::ScalarBuffer && memory.data_bits == 32u && memory.data_dwords == 1u ? &memory @@ -353,7 +352,7 @@ private: } bool MemoryIndexBelongsTo(uint32_t index, const Inst& owner) const { - for (const auto* block: m_values.blocks) { + for (const auto* block: m_program.blocks) { for (const auto& inst: *block) { const auto op = inst.GetOpcode(); if ((BufferAccessOf(op) == BufferAccess::None && @@ -446,7 +445,7 @@ private: heap_handle = current_handle; if (dword == 0u) { heap_offset = heap_reads[dword]->Arg(1).Resolve(); - } else if (!EquivalentValue(m_values, heap_offset, heap_reads[dword]->Arg(1))) { + } else if (!EquivalentValue(m_program, heap_offset, heap_reads[dword]->Arg(1))) { return false; } plan.memory[dword] = memory_index; @@ -537,7 +536,7 @@ private: } void PlanIndirectImages() { - for (auto* block: m_values.blocks) { + for (auto* block: m_program.blocks) { for (auto& inst: *block) { if (ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None || inst.NumArgs() == 0u) { @@ -588,7 +587,7 @@ private: return Fail(pc, error, "GetAddressResource must have two address dwords"); } DescriptorSource descriptor; - const auto& memory = m_values.memory_info[memory_inst.Flags().index]; + const auto& memory = m_program.memory_info[memory_inst.Flags().index]; if (memory.address_is_full) { const auto active = memory_inst.Arg(memory_inst.NumArgs() - 1u); if (memory.kind != ResourceKind::Flat || @@ -797,14 +796,14 @@ private: return true; } const auto flags = inst.Flags(); - if (flags.index >= m_values.memory_info.size()) { + if (flags.index >= m_program.memory_info.size()) { return Fail(flags.pc, error, fmt::format("memory metadata index {} is out of range", flags.index)); } if (inst.NumArgs() == 0) { return Fail(flags.pc, error, "memory operation has no resource handle"); } - const auto& memory = m_values.memory_info[flags.index]; + const auto& memory = m_program.memory_info[flags.index]; if (memory.planning_only || IsIndirectPlanningMemory(flags.index)) { return true; } @@ -914,7 +913,7 @@ private: } bool alias = true; for (uint32_t dword = 0; dword < 4; dword++) { - alias = alias && EquivalentValue(m_values, buffer_source->dwords[dword], + alias = alias && EquivalentValue(m_program, buffer_source->dwords[dword], image_source->dwords[dword]); } if (alias) { @@ -926,7 +925,6 @@ private: } Program& m_program; - ValueProgram& m_values; ShaderInfo m_info; std::vector m_sources; std::vector m_handle_patches; @@ -937,13 +935,7 @@ private: } // namespace bool TrackResources(Program& program, std::string* error) { - if (program.values == nullptr) { - if (error != nullptr) { - *error = "shader resource tracking requires typed IR"; - } - return false; - } - return Tracker(program, *program.values).Run(error); + return Tracker(program).Run(error); } } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index 7224a3070..346cca4db 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -1,6 +1,6 @@ #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include #include @@ -66,7 +66,7 @@ bool ValidateValueReferences(const Program& program, const ShaderInfoOptions& op } return false; }; - for (const auto* block: program.values->blocks) { + for (const auto* block: program.blocks) { for (const auto& inst: *block) { switch (inst.GetOpcode()) { case ValueOpcode::GetAttribute: { @@ -140,7 +140,7 @@ bool ValidateValueReferences(const Program& program, const ShaderInfoOptions& op break; } case ValueOpcode::SetAttribute: - if (inst.Flags().index >= program.values->export_info.size()) { + if (inst.Flags().index >= program.export_info.size()) { return Fail("typed export metadata index is out of range"); } break; @@ -156,7 +156,7 @@ void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* ve AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex"); AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex"); uint32_t used_components[ShaderVertexInputInfo::RES_MAX] = {}; - for (const auto* block: program.values->blocks) { + for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (inst.GetOpcode() == ValueOpcode::GetAttribute) { const auto attr = inst.Arg(0).U32(); @@ -185,7 +185,7 @@ void CollectPixelInputs(const Program& program, const ShaderPixelInputInfo* pixe } std::array per_vertex {}; std::array interpolated {}; - for (const auto* block: program.values->blocks) { + for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (inst.GetOpcode() == ValueOpcode::GetAttribute) { interpolated[inst.Arg(0).U32()] = true; @@ -228,7 +228,7 @@ void CollectComputeInputs(const ShaderComputeInputInfo* compute, ShaderInfo& inf } void CollectBuiltinInputs(const Program& program, ShaderInfo& info) { - for (const auto* block: program.values->blocks) { + for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (inst.GetOpcode() != ValueOpcode::GetBuiltin) { continue; @@ -277,12 +277,12 @@ bool CollectOutputs(const Program& program, const ShaderVertexInputInfo* vertex, } return false; }; - for (const auto* block: program.values->blocks) { + for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (inst.GetOpcode() != ValueOpcode::SetAttribute) { continue; } - const auto& export_info = program.values->export_info[inst.Flags().index]; + const auto& export_info = program.export_info[inst.Flags().index]; if (export_info.kind == ExportTargetKind::MrtZ) { if (program.stage == ShaderType::Pixel && (export_info.en & 0x1u) != 0 && pixel->ps_depth_export_enable) { @@ -360,12 +360,6 @@ bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std:: } return false; } - if (program.values == nullptr) { - if (error != nullptr) { - *error = "typed value program is not available"; - } - return false; - } if (!ValidateOptions(program, options, error)) { return false; } @@ -377,7 +371,7 @@ bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std:: next.inputs.clear(); next.outputs.clear(); next.has_bitwise_xor = std::any_of( - program.values->blocks.begin(), program.values->blocks.end(), [](const auto* block) { + program.blocks.begin(), program.blocks.end(), [](const auto* block) { return std::any_of(block->begin(), block->end(), [](const auto& inst) { return inst.GetOpcode() == ValueOpcode::BitwiseXor32; }); diff --git a/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp b/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp index a211f55c3..61d479536 100644 --- a/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp @@ -1,6 +1,6 @@ #include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include #include @@ -12,7 +12,7 @@ namespace { enum class LdsPhase { None, Read, Write }; -LdsPhase GetLdsPhase(const ValueProgram& program, const Inst& inst) { +LdsPhase GetLdsPhase(const Program& program, const Inst& inst) { const auto access = SharedAccessOf(inst.GetOpcode()); if (access != SharedAccess::Read && access != SharedAccess::Write) { return LdsPhase::None; @@ -59,7 +59,7 @@ uint32_t InsertBarrier(Block& block, Block::iterator position) { return 1; } -uint32_t InsertBlockBarriers(ValueProgram& program, Block& block) { +uint32_t InsertBlockBarriers(Program& program, Block& block) { uint32_t inserted = 0; LdsPhase phase = LdsPhase::None; for (auto inst = block.begin(); inst != block.end(); ++inst) { @@ -94,7 +94,7 @@ uint32_t InsertMergeBarrier(Block& block) { } // namespace -SharedMemoryBarrierStats InsertSharedMemoryBarriers(ValueProgram& program, uint32_t wave_size, +SharedMemoryBarrierStats InsertSharedMemoryBarriers(Program& program, uint32_t wave_size, const ShaderComputeInputInfo& compute_info) { SharedMemoryBarrierStats stats; const auto threadgroup_size = compute_info.threads_num[0] * compute_info.threads_num[1] * diff --git a/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h b/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h index a42faf456..0e5e0b77c 100644 --- a/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h +++ b/src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h @@ -9,7 +9,7 @@ struct SharedMemoryBarrierStats { }; [[nodiscard]] SharedMemoryBarrierStats -InsertSharedMemoryBarriers(ValueProgram& program, uint32_t wave_size, +InsertSharedMemoryBarriers(Program& program, uint32_t wave_size, const ShaderComputeInputInfo& compute_info); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 34ec7e5a8..bda39f1ff 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -1,6 +1,6 @@ #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/shader.h" #include @@ -50,7 +50,7 @@ bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) { return true; } -bool IsRawRead(const ValueProgram& values, const Inst& inst) { +bool IsRawRead(const Program& values, const Inst& inst) { const auto op = inst.GetOpcode(); if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer) { return false; @@ -118,8 +118,7 @@ bool IsRuntimeIntegerOp(ValueOpcode op) { class RuntimeValidator { public: - RuntimeValidator(const Program& program, const ValueProgram& values) - : m_program(program), m_values(values) {} + explicit RuntimeValidator(const Program& program): m_program(program) {} bool Run(Value value, std::string& reason) { return Validate(value, reason); } @@ -176,7 +175,7 @@ private: return finish(true); } if (op == ValueOpcode::Phi) { - const auto invariant = ResolveInvariantPhi(m_values, value); + const auto invariant = ResolveInvariantPhi(m_program, value); if (invariant.IsEmpty()) { reason = "contains a control-dependent phi"; return finish(false); @@ -196,7 +195,7 @@ private: inst->Arg(0).Resolve().TryInstruction()->GetOpcode() != ValueOpcode::GetSrtResource || !slot.IsImmediate() || slot.GetType() != Type::U32 || - slot.U32() >= m_values.srt_reads.size()) { + slot.U32() >= m_program.srt_reads.size()) { reason = "contains a malformed flattened SRT read"; return finish(false); } @@ -205,7 +204,7 @@ private: ? ValueOpcode::GetAddressResource : ValueOpcode::GetBufferResource; const auto* handle = inst->NumArgs() != 0 ? inst->Arg(0).ResolveInstruction() : nullptr; - if (!IsRawRead(m_values, *inst) || handle == nullptr || + if (!IsRawRead(m_program, *inst) || handle == nullptr || handle->GetOpcode() != expected) { reason = fmt::format("contains a non-scalar {}", ValueOpcodeName(op)); return finish(false); @@ -252,24 +251,23 @@ private: } const Program& m_program; - const ValueProgram& m_values; std::unordered_set m_visiting; }; class PlanBuilder { public: - PlanBuilder(Program& program, ValueProgram& values): m_program(program), m_values(values) {} + explicit PlanBuilder(Program& program): m_program(program) {} bool Run(std::string* error) { - m_values.srt_reads.clear(); - m_values.dynamic_reads.clear(); - for (auto* block: m_values.blocks) { + m_program.srt_reads.clear(); + m_program.dynamic_reads.clear(); + for (auto* block: m_program.blocks) { for (auto& inst: *block) { const auto op = inst.GetOpcode(); if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) { const auto flags = inst.Flags(); - if (flags.index < m_values.memory_info.size()) { - const auto kind = m_values.memory_info[flags.index].kind; + if (flags.index < m_program.memory_info.size()) { + const auto kind = m_program.memory_info[flags.index].kind; const bool crosswired = (op == ValueOpcode::LoadAddressU32 && kind == ResourceKind::ScalarBuffer) || (op == ValueOpcode::ReadConstBuffer && @@ -290,10 +288,10 @@ public: } } } - for (auto* block: m_values.blocks) { + for (auto* block: m_program.blocks) { for (auto& inst: *block) { std::string reason; - if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_values, inst) && + if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_program, inst) && inst.Arg(1).Resolve().IsImmediate() && ValidateRuntimeValue(m_program, Value(&inst), reason) && !Collect(Value(&inst), inst.Flags().pc, error)) { @@ -348,25 +346,25 @@ private: } m_visiting.pop_back(); m_visited.push_back(inst); - if (!IsRawRead(m_values, *inst)) { + if (!IsRawRead(m_program, *inst)) { return true; } const auto offset = inst->Arg(1).Resolve(); if (!offset.IsImmediate() || offset.GetType() != Type::U32) { - if (std::ranges::find(m_values.dynamic_reads, value) == m_values.dynamic_reads.end()) { - m_values.dynamic_reads.push_back(value); + if (std::ranges::find(m_program.dynamic_reads, value) == m_program.dynamic_reads.end()) { + m_program.dynamic_reads.push_back(value); } return true; } use_pc = inst->Flags().pc; - for (uint32_t slot = 0; slot < m_values.srt_reads.size(); slot++) { - if (EquivalentValue(m_values, value, m_values.srt_reads[slot].value)) { + for (uint32_t slot = 0; slot < m_program.srt_reads.size(); slot++) { + if (EquivalentValue(m_program, value, m_program.srt_reads[slot].value)) { m_patches.push_back({inst, slot, false}); return true; } } - const auto slot = static_cast(m_values.srt_reads.size()); - m_values.srt_reads.push_back({value, slot, use_pc}); + const auto slot = static_cast(m_program.srt_reads.size()); + m_program.srt_reads.push_back({value, slot, use_pc}); m_patches.push_back({inst, slot, true}); return true; } @@ -385,7 +383,7 @@ private: for (const auto& use: uses) { use.user->SetArg(use.operand, flat); } - for (auto& info: m_values.block_info) { + for (auto& info: m_program.block_info) { if (info.condition.Resolve() == Value(patch.inst)) { info.condition = flat; } @@ -395,8 +393,8 @@ private: } if (patch.keep) { const auto memory = patch.inst->Flags().index; - if (memory < m_values.memory_info.size()) { - m_values.memory_info[memory].planning_only = true; + if (memory < m_program.memory_info.size()) { + m_program.memory_info[memory].planning_only = true; } block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(patch.inst)}); } @@ -404,7 +402,6 @@ private: } Program& m_program; - ValueProgram& m_values; std::vector m_visiting; std::vector m_visited; std::vector m_patches; @@ -414,7 +411,7 @@ class Evaluator { public: Evaluator(const Program& program, const SrtRuntime& runtime, std::span clean_flat_slots = {}, Evaluator* clean_evaluator = nullptr) - : m_program(program), m_values(*program.values), m_runtime(runtime), + : m_program(program), m_runtime(runtime), m_clean_flat_slots(clean_flat_slots), m_clean_evaluator(clean_evaluator) {} void SetUsePc(uint32_t pc) { m_use_pc = pc; } @@ -472,7 +469,7 @@ private: } bool EvaluatePhi(const Inst& inst, uint64_t& result, std::string* error) { - const auto value = ResolveInvariantPhi(m_values, Value(const_cast(&inst))); + const auto value = ResolveInvariantPhi(m_program, Value(const_cast(&inst))); return !value.IsEmpty() ? EvaluateWide(value, result, error) : Fail(error, "typed phi has runtime-dependent values"); } @@ -518,10 +515,10 @@ private: bool EvaluateRawRead(const Inst& inst, uint64_t& result, std::string* error) { const auto flags = inst.Flags(); - if (flags.index >= m_values.memory_info.size()) { + if (flags.index >= m_program.memory_info.size()) { return Fail(error, "raw scalar read has invalid metadata"); } - const auto& mem = m_values.memory_info[flags.index]; + const auto& mem = m_program.memory_info[flags.index]; const auto* handle = inst.Arg(0).ResolveInstruction(); if (handle == nullptr) { return Fail(error, "raw scalar read has no descriptor handle"); @@ -609,20 +606,20 @@ private: case ValueOpcode::ReadConst: { const auto slot = inst.Arg(1).Resolve(); if (!slot.IsImmediate() || slot.GetType() != Type::U32 || - slot.U32() >= m_values.srt_reads.size()) { + slot.U32() >= m_program.srt_reads.size()) { return Fail(error, "invalid flattened SRT index"); } if (slot.U32() < m_clean_flat_slots.size() && m_clean_flat_slots[slot.U32()] != 0u && m_clean_evaluator != nullptr) { m_clean_evaluator->SetUsePc(m_use_pc); - return m_clean_evaluator->EvaluateWide(m_values.srt_reads[slot.U32()].value, + return m_clean_evaluator->EvaluateWide(m_program.srt_reads[slot.U32()].value, result, error); } - return EvaluateWide(m_values.srt_reads[slot.U32()].value, result, error); + return EvaluateWide(m_program.srt_reads[slot.U32()].value, result, error); } case ValueOpcode::LoadAddressU32: case ValueOpcode::ReadConstBuffer: - if (IsRawRead(m_values, inst)) { + if (IsRawRead(m_program, inst)) { return EvaluateRawRead(inst, result, error); } break; @@ -856,7 +853,6 @@ private: } const Program& m_program; - const ValueProgram& m_values; const SrtRuntime& m_runtime; std::span m_clean_flat_slots; Evaluator* m_clean_evaluator = nullptr; @@ -866,10 +862,10 @@ private: }; const DescriptorSource* Source(const Program& program, uint32_t source) { - if (program.values == nullptr || source >= program.values->descriptor_sources.size()) { + if (source >= program.descriptor_sources.size()) { return nullptr; } - return &program.values->descriptor_sources[source]; + return &program.descriptor_sources[source]; } bool EvaluateRuntimeSourcesImpl(const Program& program, @@ -877,7 +873,7 @@ bool EvaluateRuntimeSourcesImpl(const Program& program const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluate_flat, std::span clean_flat_slots, std::string* error) { - if (program.values == nullptr || !program.srt_plan_complete) { + if (!program.srt_plan_complete) { if (error != nullptr) { *error = Diagnostic(program, 0, "typed SRT plan is not ready"); } @@ -917,8 +913,8 @@ bool EvaluateRuntimeSourcesImpl(const Program& program } std::vector flattened; if (evaluate_flat) { - flattened.resize(program.values->srt_reads.size()); - for (const auto& read: program.values->srt_reads) { + flattened.resize(program.srt_reads.size()); + for (const auto& read: program.srt_reads) { const bool clean = read.flat_offset < clean_flat_slots.size() && clean_flat_slots[read.flat_offset] != 0u; auto& selected = clean ? clean_evaluator : evaluator; @@ -939,23 +935,18 @@ bool EvaluateRuntimeSourcesImpl(const Program& program } // namespace bool ValidateRuntimeValue(const Program& program, Value value, std::string& reason) { - if (program.values == nullptr) { - reason = "typed SSA is not ready"; - return false; - } - return RuntimeValidator(program, *program.values).Run(value, reason); + return RuntimeValidator(program).Run(value, reason); } bool BuildSrtPlan(Program& program, std::string* error) { - if (program.values == nullptr || program.resource_tracking_complete) { + if (program.resource_tracking_complete) { if (error != nullptr) { - *error = program.values == nullptr ? "typed SSA is not ready" - : "cannot rebuild SRT after resource tracking"; + *error = "cannot rebuild SRT after resource tracking"; } return false; } program.srt_plan_complete = false; - if (!PlanBuilder(program, *program.values).Run(error)) { + if (!PlanBuilder(program).Run(error)) { return false; } program.srt_plan_complete = true; diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index 1a04c2183..eefb2d822 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -15,7 +15,7 @@ #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/shaderVertexMetadata.h" #include "kernel/memory.h" #include "libs/errno.h" @@ -49,9 +49,6 @@ namespace Libs::Graphics { namespace { -constexpr uint32_t PsInputOffsetMask = 0x0000001fu; -constexpr uint32_t PsInputFlatShade = 0x00000400u; - bool ReadShaderGuestMemory(void*, uint64_t address, uint32_t* value) { return value != nullptr && Libs::LibKernel::Memory::TryReadGpuCleanBacking(address, value, sizeof(*value)); @@ -59,40 +56,6 @@ bool ReadShaderGuestMemory(void*, uint64_t address, uint32_t* value) { } // namespace -uint32_t ShaderPixelParameterMappedLocation(const ShaderPixelInputInfo& info, uint32_t input) { - return input < info.input_num ? info.interpolator_settings[input] & PsInputOffsetMask : input; -} - -uint32_t ShaderPixelParameterLocation(const ShaderPixelInputInfo& info, - std::span active_inputs, uint32_t input) { - std::array used_locations {}; - for (const auto active_input: active_inputs) { - auto location = ShaderPixelParameterMappedLocation(info, active_input); - if (location < used_locations.size() && used_locations[location]) { - location = active_input; - while (location < used_locations.size() && used_locations[location]) { - location++; - } - EXIT_NOT_IMPLEMENTED(location >= used_locations.size()); - } - - if (active_input == input) { - return location; - } - used_locations[location] = true; - } - return ShaderPixelParameterMappedLocation(info, input); -} - -bool ShaderPixelParameterIsFlat(const ShaderPixelInputInfo& info, uint32_t input) { - return input < info.input_num && (info.interpolator_settings[input] & PsInputFlatShade) != 0 && - !ShaderPixelParameterIsCustom(info, input); -} - -bool ShaderPixelParameterIsCustom(const ShaderPixelInputInfo& info, uint32_t input) { - return input < 32u && (info.custom_interpolation_mask & (1u << input)) != 0; -} - struct ShaderBinaryInfo { uint8_t signature[7]; uint8_t version; @@ -1035,7 +998,7 @@ static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Prog program.bindings.push_constant_size, program.bindings.descriptors.size(), program.bindings.user_data_registers.size(), program.info.buffers.size(), program.info.images.size(), program.info.samplers.size(), program.info.addresses.size(), - program.values != nullptr ? program.values->srt_reads.size() : 0u); + program.srt_reads.size()); for (uint32_t i = 0; i < program.info.buffers.size(); i++) { const auto& buffer = program.info.buffers[i]; ret += diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 26af3f98f..14f91b152 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -84,7 +84,6 @@ struct ShaderVertexInputInfo { ShaderBufferResource resources[RES_MAX]; ShaderVertexDestination resources_dst[RES_MAX]; - int resource_fetch_components[RES_MAX] = {}; ShaderVertexInputBuffer buffers[RES_MAX]; ShaderStageRuntime stage; int resources_num = 0; diff --git a/src/graphics/shader/shaderPixelParameter.cpp b/src/graphics/shader/shaderPixelParameter.cpp new file mode 100644 index 000000000..89cc629e9 --- /dev/null +++ b/src/graphics/shader/shaderPixelParameter.cpp @@ -0,0 +1,48 @@ +#include "graphics/shader/shader.h" + +#include "common/assert.h" + +namespace Libs::Graphics { + +namespace { + +constexpr uint32_t PsInputOffsetMask = 0x0000001fu; +constexpr uint32_t PsInputFlatShade = 0x00000400u; + +} // namespace + +uint32_t ShaderPixelParameterMappedLocation(const ShaderPixelInputInfo& info, uint32_t input) { + return input < info.input_num ? info.interpolator_settings[input] & PsInputOffsetMask : input; +} + +uint32_t ShaderPixelParameterLocation(const ShaderPixelInputInfo& info, + std::span active_inputs, uint32_t input) { + std::array used_locations {}; + for (const auto active_input: active_inputs) { + auto location = ShaderPixelParameterMappedLocation(info, active_input); + if (location < used_locations.size() && used_locations[location]) { + location = active_input; + while (location < used_locations.size() && used_locations[location]) { + location++; + } + EXIT_NOT_IMPLEMENTED(location >= used_locations.size()); + } + + if (active_input == input) { + return location; + } + used_locations[location] = true; + } + return ShaderPixelParameterMappedLocation(info, input); +} + +bool ShaderPixelParameterIsFlat(const ShaderPixelInputInfo& info, uint32_t input) { + return input < info.input_num && (info.interpolator_settings[input] & PsInputFlatShade) != 0 && + !ShaderPixelParameterIsCustom(info, input); +} + +bool ShaderPixelParameterIsCustom(const ShaderPixelInputInfo& info, uint32_t input) { + return input < 32u && (info.custom_interpolation_mask & (1u << input)) != 0; +} + +} // namespace Libs::Graphics diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index aa56a021e..61b94bbf8 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -1,5 +1,5 @@ #include "graphics/guest_gpu/gpu_defs.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" @@ -35,7 +35,6 @@ struct Fixture { explicit Fixture(ShaderType stage = ShaderType::Compute) { program.stage = stage; - program.values = std::make_shared(); program.user_data_count = 64; block = AddBlock(); } @@ -43,10 +42,10 @@ struct Fixture { Block *AddBlock() { auto storage = std::make_unique(); auto *result = storage.get(); - program.values->block_storage.push_back(std::move(storage)); - program.values->blocks.push_back(result); - program.values->block_info.push_back( - {.id = static_cast(program.values->block_info.size())}); + program.block_storage.push_back(std::move(storage)); + program.blocks.push_back(result); + program.block_info.push_back( + {.id = static_cast(program.block_info.size())}); return result; } @@ -77,8 +76,8 @@ struct Fixture { MemoryFlags AddMemory(MemoryInfo memory, uint32_t pc) { const auto index = - static_cast(program.values->memory_info.size()); - program.values->memory_info.push_back(memory); + static_cast(program.memory_info.size()); + program.memory_info.push_back(memory); return {index, pc}; } @@ -241,19 +240,19 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0, void TestInvariantIndirectImageMaterialization() { auto fixture = MakeIndirectImageFixture(false); fixture->PlanAndTrack(); - EliminateDeadCode(fixture->program.values->blocks); + EliminateDeadCode(fixture->program.blocks); std::string validation_error; Check( - ValidateValueProgram(*fixture->program.values, true, &validation_error), + ValidateProgram(fixture->program, true, &validation_error), "post-tracking dead-code elimination invalidated descriptor provenance"); Check(fixture->program.info.buffers.size() == 1 && fixture->program.info.images.size() == 1 && - fixture->program.values->dynamic_reads.size() == 1, + fixture->program.dynamic_reads.size() == 1, "indirect image key was not retained as a scalar-buffer read"); const auto source = fixture->program.info.images[0].source; - Check(source < fixture->program.values->descriptor_sources.size() && - fixture->program.values->descriptor_sources[source] + Check(source < fixture->program.descriptor_sources.size() && + fixture->program.descriptor_sources[source] .indirect_image.has_value(), "indirect image source was not retained for runtime proof"); const auto image_handle = @@ -407,7 +406,7 @@ void TestInvariantIndirectImageMaterialization() { auto memory_backed = MakeIndirectImageFixture(false, 0u, true); memory_backed->PlanAndTrack(); - EliminateDeadCode(memory_backed->program.values->blocks); + EliminateDeadCode(memory_backed->program.blocks); std::array memory_backed_user_data{0x1000u, 224u << 16u, 2u, 0u, 0x2000u, 16u << 16u, 4u, 0u, 7u, 0x3100u, 0u}; @@ -431,7 +430,7 @@ void TestInvariantIndirectImageMaterialization() { error.find("ReadFirstLane") != std::string::npos && !malformed->program.resource_tracking_complete && malformed->program.info.images.empty() && - malformed->program.values->descriptor_sources.empty(), + malformed->program.descriptor_sources.empty(), "malformed indirect image pattern was partially accepted"); auto wrapped_immediate = MakeIndirectImageFixture(false, 4u); @@ -484,7 +483,7 @@ void TestDenseBufferTracking() { Check(fixture.program.info.buffers.size() == 2, "typed buffer sources were not densely interned"); - Check(fixture.program.values->descriptor_sources.size() == 2, + Check(fixture.program.descriptor_sources.size() == 2, "descriptor source table did not match dense topology"); const auto &resource = fixture.program.info.buffers[0]; Check(resource.read && resource.written && resource.atomic && @@ -494,10 +493,10 @@ void TestDenseBufferTracking() { Check(first.Instruction()->Flags() == 0 && second.Instruction()->Flags() == 1, "typed handles were not assigned dense indices"); - Check(fixture.program.values->memory_info[load_flags.index].resource == 0 && - fixture.program.values->memory_info[store_flags.index].resource == + Check(fixture.program.memory_info[load_flags.index].resource == 0 && + fixture.program.memory_info[store_flags.index].resource == 0 && - fixture.program.values->memory_info[other_flags.index].resource == + fixture.program.memory_info[other_flags.index].resource == 1, "typed memory metadata was not patched to dense indices"); @@ -531,8 +530,8 @@ void TestScalarAndVectorBufferAlias() { Check(fixture.program.info.buffers.size() == 1 && fixture.program.info.buffers[0].scalar, "typed scalar and vector uses of one descriptor were split"); - Check(fixture.program.values->memory_info[scalar_flags.index].resource == 0 && - fixture.program.values->memory_info[vector_flags.index].resource == + Check(fixture.program.memory_info[scalar_flags.index].resource == 0 && + fixture.program.memory_info[vector_flags.index].resource == 0, "scalar/vector alias did not share a dense index"); } @@ -625,7 +624,7 @@ void TestImagesSamplersAndAliases() { repeated.second.Instruction()->Flags() == 0, "unused sampler border colors prevented source interning"); const auto sampler_source = fixture.program.info.samplers[0].source; - Check(fixture.program.values->descriptor_sources[sampler_source] + Check(fixture.program.descriptor_sources[sampler_source] .dwords[3] .U32() == 0, "unused sampler border color was not canonicalized"); @@ -665,7 +664,7 @@ void TestSampleAdjustSamplerScratch() { fixture.PlanAndTrack(); const auto source = fixture.program.info.samplers[0].source; - const auto stored = fixture.program.values->descriptor_sources[source] + const auto stored = fixture.program.descriptor_sources[source] .dwords[3] .Resolve() .TryInstruction(); @@ -758,10 +757,10 @@ void TestDynamicStorageMipTracking() { mip1.first.Instruction()->Flags() == 1 && mip2.first.Instruction()->Flags() == 1 && dynamic.first.Instruction()->Flags() == 1 && - fixture.program.values->memory_info[plain.second].resource == 0 && - fixture.program.values->memory_info[mip1.second].resource == 1 && - fixture.program.values->memory_info[mip2.second].resource == 1 && - fixture.program.values->memory_info[dynamic.second].resource == 1, + fixture.program.memory_info[plain.second].resource == 0 && + fixture.program.memory_info[mip1.second].resource == 1 && + fixture.program.memory_info[mip2.second].resource == 1 && + fixture.program.memory_info[dynamic.second].resource == 1, "dynamic storage mip handles and memory metadata were not patched"); DescriptorValue descriptor{}; @@ -795,7 +794,6 @@ void TestDynamicStorageMipTracking() { "dynamic storage mip descriptors were not expanded consecutively"); Program null_program; - null_program.values = std::make_shared(); null_program.resource_tracking_complete = true; ImageResource null_image; null_image.kind = ResourceKind::StorageImage; @@ -847,12 +845,12 @@ void TestSrtFlatteningAndRuntimeMemoization() { fixture.AddMemory(buffer, 16)); fixture.PlanAndTrack(); - Check(fixture.program.values->srt_reads.size() == 1, + Check(fixture.program.srt_reads.size() == 1, "shared typed scalar read did not receive one flat SRT slot"); Check(fixture.program.info.buffers.size() == 1 && fixture.program.info.addresses.empty(), "planning-only scalar reads leaked into resource topology"); - Check(fixture.program.values->memory_info[0].planning_only, + Check(fixture.program.memory_info[0].planning_only, "canonical runtime scalar read was not marked planning-only"); std::array user_data{0x1000u, 0u}; @@ -910,8 +908,8 @@ void TestDynamicSrtReadRemainsExplicit() { fixture.AddMemory(buffer, 8)); fixture.PlanAndTrack(); - Check(fixture.program.values->srt_reads.empty() && - fixture.program.values->dynamic_reads.size() == 1 && + Check(fixture.program.srt_reads.empty() && + fixture.program.dynamic_reads.size() == 1 && fixture.program.info.addresses.size() == 1, "dynamic scalar read was incorrectly flattened or lost"); std::array user_data{0x1000u, 0u, 4u}; @@ -973,7 +971,7 @@ void TestPhiValidation() { error.find("control-dependent phi") != std::string::npos && !fixture.program.resource_tracking_complete && fixture.program.info.buffers.empty() && - fixture.program.values->descriptor_sources.empty(), + fixture.program.descriptor_sources.empty(), "control-dependent descriptor phi was not rejected transactionally"); } @@ -1295,7 +1293,7 @@ void TestResourceLimitIsTransactional() { error.find("buffer resource limit exceeded") != std::string::npos && !fixture.program.resource_tracking_complete && fixture.program.info.buffers.empty() && - fixture.program.values->descriptor_sources.empty(), + fixture.program.descriptor_sources.empty(), "resource-limit failure partially mutated typed resource state"); } @@ -1412,13 +1410,18 @@ int main() { // The full emulator supplies these assertion hooks through common. This focused // target links only fmt; keep assertion failures observable without widening -// its legacy build manifest. +// its focused build manifest. namespace Common { int DbgExitIfHandler(const char *expression, const char *file, int line) { throw std::runtime_error(std::string("typed IR assertion: ") + expression + " at " + file + ':' + std::to_string(line)); } +int DbgNotImplementedHandler(const char *expression, const char *file, int line) { + throw std::runtime_error(std::string("typed IR not implemented: ") + expression + + " at " + file + ':' + std::to_string(line)); +} + void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); } } // namespace Common @@ -1427,6 +1430,6 @@ void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); } #include "graphics/shader/recompiler/ir/Block.cpp" #include "graphics/shader/recompiler/ir/Type.cpp" #include "graphics/shader/recompiler/ir/Value.cpp" -#include "graphics/shader/recompiler/ir/ValueProgram.cpp" +#include "graphics/shader/recompiler/ir/Program.cpp" #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp" diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index a90678d3d..48d01a654 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" @@ -27,16 +27,15 @@ struct Fixture { program.stage = ShaderType::Compute; program.shader_hash = 0x12345678u; program.user_data_base = 2; - program.values = std::make_shared(); for (uint32_t index = 0; index < block_count; index++) { - program.values->block_storage.push_back(std::make_unique()); - auto *block = program.values->block_storage.back().get(); - program.values->blocks.push_back(block); - program.values->block_info.push_back({.id = index}); + program.block_storage.push_back(std::make_unique()); + auto *block = program.block_storage.back().get(); + program.blocks.push_back(block); + program.block_info.push_back({.id = index}); } } - Block &BlockAt(uint32_t index = 0) { return *program.values->blocks[index]; } + Block &BlockAt(uint32_t index = 0) { return *program.blocks[index]; } Value Emit(ValueOpcode opcode, std::initializer_list args = {}, uint64_t flags = 0, uint32_t block = 0) { @@ -55,8 +54,8 @@ struct Fixture { MemoryInfo info; info.kind = kind; info.offset = static_cast(offset); - program.values->memory_info.push_back(info); - return static_cast(program.values->memory_info.size() - 1u); + program.memory_info.push_back(info); + return static_cast(program.memory_info.size() - 1u); } void Plan() { @@ -103,11 +102,11 @@ void TestImmediateFlatteningAndGvn() { {first, second, Value(16u), Value(0u)}); fixture.Plan(); - Check(fixture.program.values->srt_reads.size() == 1, + Check(fixture.program.srt_reads.size() == 1, "equivalent typed scalar reads were not coalesced"); - Check(fixture.program.values->dynamic_reads.empty(), + Check(fixture.program.dynamic_reads.empty(), "immediate scalar read was classified as dynamic"); - Check(fixture.program.values->memory_info[memory].planning_only, + Check(fixture.program.memory_info[memory].planning_only, "flattened raw read was not kept as a planning-only root"); TestMemory memory_image{{{0x1020u, 0xfeedbeefu}}}; @@ -173,8 +172,8 @@ void TestDynamicReadRemainsTyped() { {read, Value(0u), Value(16u), Value(0u)}); fixture.Plan(); - Check(fixture.program.values->srt_reads.empty() && - fixture.program.values->dynamic_reads == std::vector{read}, + Check(fixture.program.srt_reads.empty() && + fixture.program.dynamic_reads == std::vector{read}, "dynamic scalar read received a fake flattened slot"); } @@ -209,7 +208,7 @@ void TestShaderBaseAndUserData() { {Value(static_cast(2))}); const auto sum = fixture.Emit(ValueOpcode::IAdd32, {user, Value(4u)}); fixture.Plan(); - fixture.program.values->descriptor_sources.push_back( + fixture.program.descriptor_sources.push_back( {.dwords = {low, high, sum}, .dword_count = 3}); const std::array user_data{0x20u}; @@ -239,7 +238,7 @@ void TestCarryAndBitFields() { const auto sign = fixture.Emit(ValueOpcode::BitFieldSExtract, {Value(0x000000f0u), Value(4u), Value(4u)}); fixture.Plan(); - fixture.program.values->descriptor_sources.push_back( + fixture.program.descriptor_sources.push_back( {.dwords = {low, high, inserted, sign}, .dword_count = 4}); DescriptorValue result; @@ -262,9 +261,9 @@ void TestInvariantAndDivergentPhi() { divergent.AddPhiOperand(&fixture.BlockAt(0), Value(7u)); divergent.AddPhiOperand(&fixture.BlockAt(1), Value(9u)); fixture.Plan(); - fixture.program.values->descriptor_sources.push_back( + fixture.program.descriptor_sources.push_back( {.dwords = {Value(&invariant)}, .dword_count = 1}); - fixture.program.values->descriptor_sources.push_back( + fixture.program.descriptor_sources.push_back( {.dwords = {Value(&divergent)}, .dword_count = 1}); DescriptorValue result; @@ -293,10 +292,10 @@ void TestControlDependentStandaloneLoadStaysTyped() { RawRead(fixture, Address(fixture, Value(&base), Value(0u), 2), Value(0u), memory, 2); fixture.Plan(); - Check(fixture.program.values->srt_reads.empty() && + Check(fixture.program.srt_reads.empty() && read.ResolveInstruction()->GetOpcode() == ValueOpcode::LoadAddressU32 && - !fixture.program.values->memory_info[memory].planning_only, + !fixture.program.memory_info[memory].planning_only, "control-dependent standalone scalar load was flattened into a host " "snapshot"); } @@ -315,7 +314,7 @@ void TestRuntime64BitDescriptorOps() { const auto high = fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(1u)}); fixture.Plan(); - fixture.program.values->descriptor_sources.push_back( + fixture.program.descriptor_sources.push_back( {.dwords = {low, high}, .dword_count = 2}); DescriptorValue result; @@ -370,7 +369,7 @@ void TestReadLaneElimination() { {undef, Value(0xdeadbeefu), Value(5u)}); const auto read = fixture.Emit(ValueOpcode::ReadLane, {write, Value(5u)}); const auto use = fixture.Emit(ValueOpcode::IAdd32, {read, Value(1u)}); - const auto stats = EliminateReadLane(*fixture.program.values, 64); + const auto stats = EliminateReadLane(fixture.program, 64); Check(stats.rewritten_reads == 1 && use.ResolveInstruction()->Arg(0).Resolve() == Value(0xdeadbeefu), "fixed-lane typed read was not rewritten from its SSA write chain"); @@ -379,7 +378,7 @@ void TestReadLaneElimination() { {Value(static_cast(2))}); const auto dynamic = fixture.Emit(ValueOpcode::ReadLane, {write, selector}); fixture.Emit(ValueOpcode::IAdd32, {dynamic, Value(1u)}); - Check(EliminateReadLane(*fixture.program.values, 64).rewritten_reads == 0, + Check(EliminateReadLane(fixture.program, 64).rewritten_reads == 0, "dynamic-lane read was rewritten unsafely"); } @@ -390,9 +389,9 @@ void TestOptimizationPipeline() { fixture.Emit(ValueOpcode::ReferenceU32, {kept}); fixture.Emit(ValueOpcode::IMul32, {Value(6u), Value(7u)}); - ConstantPropagationPass(fixture.program.values->blocks); - RemoveIdentities(fixture.program.values->blocks); - EliminateDeadCode(fixture.program.values->blocks); + ConstantPropagationPass(fixture.program.blocks); + RemoveIdentities(fixture.program.blocks); + EliminateDeadCode(fixture.program.blocks); const auto &instructions = fixture.BlockAt().Instructions(); Check(instructions.size() == 1 && @@ -404,20 +403,20 @@ void TestOptimizationPipeline() { void TestControlFlowValueSurvivesReadLaneFolding() { Fixture fixture(3); - auto *entry = fixture.program.values->blocks[0]; - auto *taken = fixture.program.values->blocks[1]; - auto *other = fixture.program.values->blocks[2]; + auto *entry = fixture.program.blocks[0]; + auto *taken = fixture.program.blocks[1]; + auto *other = fixture.program.blocks[2]; entry->AddBranch(taken); entry->AddBranch(other); - auto &entry_info = fixture.program.values->block_info[0]; + auto &entry_info = fixture.program.block_info[0]; entry_info.terminator.kind = Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; entry_info.terminator.true_block = 1; entry_info.terminator.false_block = 2; - fixture.program.values->block_info[1].terminator.kind = + fixture.program.block_info[1].terminator.kind = Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::Return; - fixture.program.values->block_info[2].terminator.kind = + fixture.program.block_info[2].terminator.kind = Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::Return; const auto undef = fixture.Emit(ValueOpcode::UndefU32); @@ -429,16 +428,16 @@ void TestControlFlowValueSurvivesReadLaneFolding() { fixture.Emit(ValueOpcode::IEqual32, {read, Value(42u)}); fixture.Emit(ValueOpcode::Reference, {entry_info.condition}); - Check(EliminateReadLane(*fixture.program.values, 64).rewritten_reads == 1, + Check(EliminateReadLane(fixture.program, 64).rewritten_reads == 1, "control-flow fixture did not eliminate its fixed-lane read"); - ConstantPropagationPass(fixture.program.values->blocks); - ResolveControlFlowIdentities(*fixture.program.values); - RemoveIdentities(fixture.program.values->blocks); - EliminateDeadCode(fixture.program.values->blocks); + ConstantPropagationPass(fixture.program.blocks); + ResolveControlFlowIdentities(fixture.program); + RemoveIdentities(fixture.program.blocks); + EliminateDeadCode(fixture.program.blocks); std::string error; Check(entry_info.condition == Value(true) && - ValidateValueProgram(*fixture.program.values, true, &error), + ValidateProgram(fixture.program, true, &error), error.empty() ? "folded branch condition did not survive identity removal" : error.c_str()); } @@ -447,7 +446,7 @@ void TestUndefinedRuntimeValueFails() { Fixture fixture; const auto undef = fixture.Emit(ValueOpcode::UndefU32); fixture.Plan(); - fixture.program.values->descriptor_sources.push_back( + fixture.program.descriptor_sources.push_back( {.dwords = {undef}, .dword_count = 1}); DescriptorValue result; result.dword_count = 3; @@ -499,7 +498,7 @@ int main() { #include "../src/graphics/shader/recompiler/ir/Block.cpp" #include "../src/graphics/shader/recompiler/ir/Type.cpp" #include "../src/graphics/shader/recompiler/ir/Value.cpp" -#include "../src/graphics/shader/recompiler/ir/ValueProgram.cpp" +#include "../src/graphics/shader/recompiler/ir/Program.cpp" #include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" #include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp" #include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp" diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index a17754da4..1186cad52 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -38,7 +38,7 @@ #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h" #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/rectListShader.h" #include "graphics/shader/shader.h" @@ -967,7 +967,6 @@ struct TestCase { std::vector expected_gds; std::vector> decoded_counts; std::vector> ir_counts; - std::vector> native_ir_counts; u32 expected_storage_mip_descriptors = 0; }; @@ -1192,32 +1191,6 @@ CompiledShader CompileCase(const TestCase &test) { text + " count=" + std::to_string(actual) + ", expected=" + std::to_string(expected)); } - if (!test.native_ir_counts.empty()) { - ShaderRecompiler::Decoder::Program decoded; - ShaderRecompiler::CFG::Graph cfg; - ShaderRecompiler::IR::Program native; - std::string native_error; - Require(test.name, "native decode", - ShaderRecompiler::Decoder::DecodeProgram(test.code, decoded, - &native_error), - native_error); - Require(test.name, "native CFG", - ShaderRecompiler::CFG::BuildGraph(decoded, cfg, &native_error) && - ShaderRecompiler::CFG::Structurize(cfg, &native_error), - native_error); - Require(test.name, "native lowering", - ShaderRecompiler::IR::LowerProgram(decoded, cfg, options.stage, - options.wave_size, native, - &native_error), - native_error); - const auto native_ir = ShaderRecompiler::IR::ProgramToString(native); - for (const auto &[text, expected] : test.native_ir_counts) { - const auto actual = CountText(native_ir, text); - Require(test.name, "native IR", actual == expected, - text + " count=" + std::to_string(actual) + - ", expected=" + std::to_string(expected)); - } - } if (test.expected_storage_mip_descriptors != 0) { const auto image = std::ranges::find_if( result.program.info.images, [](const auto &resource) { @@ -1242,7 +1215,7 @@ CompiledShader CompileCase(const TestCase &test) { "dynamic storage image did not receive one descriptor per mip"); } if (test.expand_shader_data_storage) { - auto &block = *result.program.values->blocks.front(); + auto &block = *result.program.blocks.front(); for (u32 reg = 0; reg < 33; reg++) { auto &user_data = block.AppendNewInst( ShaderRecompiler::IR::ValueOpcode::GetUserData, @@ -7519,8 +7492,6 @@ public: ShaderRecompiler::IR::Program null_program{}; null_program.stage = ShaderType::Compute; - null_program.values = - std::make_shared(); null_program.resource_tracking_complete = true; ShaderRecompiler::IR::ImageResource null_resource{}; null_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; @@ -7630,8 +7601,6 @@ public: storage.fields[5] = 0x00700000u; ShaderRecompiler::IR::Program storage_program{}; storage_program.stage = ShaderType::Vertex; - storage_program.values = - std::make_shared(); storage_program.resource_tracking_complete = true; ShaderRecompiler::IR::ImageResource storage_resource{}; storage_resource.kind = @@ -7911,8 +7880,6 @@ public: ShaderRecompiler::IR::Program sampled_program{}; sampled_program.stage = ShaderType::Pixel; - sampled_program.values = - std::make_shared(); sampled_program.resource_tracking_complete = true; ShaderRecompiler::IR::ImageResource sampled_resource{}; sampled_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; @@ -8003,15 +7970,19 @@ public: RenderExecutorTestAccess::ResetBindings(executor); auto vertex_sampled_program = - std::make_shared( - *sampled_runtime.program); + std::make_shared(); vertex_sampled_program->stage = ShaderType::Vertex; + vertex_sampled_program->resource_tracking_complete = true; + vertex_sampled_program->info.images.push_back(sampled_resource); + allocate_bindings(*vertex_sampled_program); ShaderStageRuntime vertex_sampled_runtime{ std::move(vertex_sampled_program), sampled_runtime.resources}; auto pixel_storage_program = - std::make_shared( - *storage_runtime.program); + std::make_shared(); pixel_storage_program->stage = ShaderType::Pixel; + pixel_storage_program->resource_tracking_complete = true; + pixel_storage_program->info.images.push_back(storage_resource); + allocate_bindings(*pixel_storage_program); ShaderStageRuntime pixel_storage_runtime{std::move(pixel_storage_program), storage_runtime.resources}; auto writable_alias_bindings = @@ -8083,8 +8054,6 @@ public: auto split_program = std::make_shared(); split_program->stage = ShaderType::Vertex; - split_program->values = - std::make_shared(); split_program->resource_tracking_complete = true; split_program->info.images = {storage_resource, sampled_resource}; allocate_bindings(*split_program); @@ -8459,8 +8428,6 @@ public: ShaderRecompiler::IR::Program array_program{}; array_program.stage = ShaderType::Compute; - array_program.values = - std::make_shared(); array_program.resource_tracking_complete = true; ShaderRecompiler::IR::ImageResource array_resource{}; array_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; @@ -8544,10 +8511,13 @@ public: colliding_msaa_descriptor.dwords.begin()); colliding_msaa_descriptor.dword_count = 8; auto colliding_msaa_program = - std::make_shared( - *array_runtime.program); - colliding_msaa_program->info.images[0].dimension = + std::make_shared(); + colliding_msaa_program->stage = ShaderType::Compute; + colliding_msaa_program->resource_tracking_complete = true; + auto colliding_msaa_resource = array_resource; + colliding_msaa_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa; + colliding_msaa_program->info.images.push_back(colliding_msaa_resource); auto colliding_msaa_snapshot = std::make_shared(); colliding_msaa_snapshot->images.push_back(colliding_msaa_descriptor); @@ -8604,10 +8574,13 @@ public: std::copy(std::begin(msaa_texture.fields), std::end(msaa_texture.fields), msaa_descriptor.dwords.begin()); msaa_descriptor.dword_count = 8; - auto msaa_program = std::make_shared( - *array_runtime.program); - msaa_program->info.images[0].dimension = + auto msaa_program = std::make_shared(); + msaa_program->stage = ShaderType::Compute; + msaa_program->resource_tracking_complete = true; + auto msaa_resource = array_resource; + msaa_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa; + msaa_program->info.images.push_back(msaa_resource); auto msaa_snapshot = std::make_shared(); msaa_snapshot->images.push_back(msaa_descriptor); @@ -8643,10 +8616,14 @@ public: std::end(msaa_array_texture.fields), msaa_array_descriptor.dwords.begin()); msaa_array_descriptor.dword_count = 8; - auto msaa_array_program = std::make_shared( - *msaa_runtime.program); - msaa_array_program->info.images[0].dimension = + auto msaa_array_program = + std::make_shared(); + msaa_array_program->stage = ShaderType::Compute; + msaa_array_program->resource_tracking_complete = true; + auto msaa_array_resource = array_resource; + msaa_array_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray; + msaa_array_program->info.images.push_back(msaa_array_resource); auto msaa_array_snapshot = std::make_shared(); msaa_array_snapshot->images.push_back(msaa_array_descriptor); @@ -8868,7 +8845,6 @@ public: ShaderRecompiler::IR::Program program{}; program.stage = ShaderType::Compute; - program.values = std::make_shared(); program.resource_tracking_complete = true; ShaderRecompiler::IR::ImageResource stencil_resource{}; stencil_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; @@ -8940,11 +8916,15 @@ public: RenderExecutorTestAccess::ResetBindings(executor); auto stencil_storage_program = - std::make_shared(*runtime.program); - stencil_storage_program->info.images[0].kind = + std::make_shared(); + stencil_storage_program->stage = ShaderType::Compute; + stencil_storage_program->resource_tracking_complete = true; + auto stencil_storage_resource = stencil_resource; + stencil_storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - stencil_storage_program->info.images[0].read = false; - stencil_storage_program->info.images[0].written = true; + stencil_storage_resource.read = false; + stencil_storage_resource.written = true; + stencil_storage_program->info.images.push_back(stencil_storage_resource); auto storage_stencil = stencil; storage_stencil.fields[5] = 0x00700000u; ShaderRecompiler::IR::DescriptorValue stencil_storage_descriptor{}; @@ -12132,7 +12112,6 @@ TestCase ScalarWaitcntDepctrCapturedVmVsrc() { test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_WAITCNT_DEPCTR, O::S_ENDPGM}; test.decoded_counts = {{"0x00000010: S_WAITCNT_DEPCTR 0x0000ffe3\n", 1}}; - test.native_ir_counts = {{"0x00000010: Waitcnt null, 0x0000ffe3", 1}}; test.forbidden_spirv = {"OpControlBarrier", "OpMemoryBarrier"}; return test; } @@ -13289,7 +13268,6 @@ TestCase Vop3Med3I16Captured() { test.decoded_counts = {{"0x00000020: V_MED3_I16 v3.sdwa(sel=5,sext=0), " "v5.opsel(lo=1,hi=0,neghi=0), s25, s27\n", 1}}; - test.native_ir_counts = {{"IMed3I16", 1}}; test.required_spirv = {"OpSLessThan"}; return test; } @@ -15227,8 +15205,6 @@ TestCase VectorVopcCmpxNeU64CapturedExecMask() { O::S_MOV_B32, O::S_ENDPGM}}; test.decoded_counts = {{"V_CMPX_NE_U64 exec_lo, 0, v7", 2}, {"V_CMPX_NE_U64 exec_lo, 0, v2", 2}}; - test.native_ir_counts = {{"CompareNeU64 exec_lo, 0x00000000, v7", 2}, - {"CompareNeU64 exec_lo, 0x00000000, v2", 2}}; return test; } @@ -15258,7 +15234,6 @@ TestCase VectorVop3CmpxNeI64CapturedExecMask() { {O::V_MOV_B32, O::V_CMPX_NE_I64, O::BUFFER_STORE_DWORD, O::S_MOV_B32, O::S_ENDPGM}}; test.decoded_counts = {{"V_CMPX_NE_I64 exec_lo, 0, v7", 2}}; - test.native_ir_counts = {{"CompareNeU64 exec_lo, 0x00000000, v7", 2}}; return test; } @@ -17738,7 +17713,6 @@ TestCase DsReadU16D16CapturedPreservesHighHalf() { {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_U16_D16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; test.decoded_counts = {{"DS_READ_U16_D16", 2}}; - test.native_ir_counts = {{"DsReadUshort", 2}}; return test; } @@ -18652,7 +18626,6 @@ TestCase ImageLoadR128IgnoresAdjacentMaskSgprs() { test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt); test.has_user_data = true; test.decoded_counts = {{"r128=1", 1}}; - test.native_ir_counts = {{"image_r128=1", 1}}; return test; } @@ -18974,8 +18947,6 @@ TestCase ImageD16GatherPacksHalfPairs() { O::S_ENDPGM}; test.sampled_image_rgba = image; test.decoded_counts = {{"d16=1", 1}}; - test.native_ir_counts = { - {"dmask=0x1 data_dwords=2 data_bits=16 component=0/4", 1}}; test.required_spirv = {"OpImageGather", "PackHalf2x16"}; return test; } @@ -19196,8 +19167,6 @@ TestCase ImageD16StoreUnpacksHalfPairs() { test.storage_image_rgba = MakeRgbaImage(4, 4); test.expected_storage_image_rgba = expected_image; test.decoded_counts = {{"d16=1", 1}}; - test.native_ir_counts = { - {"dmask=0xf data_dwords=2 data_bits=16 component=0/4", 1}}; test.required_spirv = {"UnpackHalf2x16", "OpImageWrite"}; return test; } @@ -19213,11 +19182,10 @@ void CheckIndirectImageKeySwitch() { program.srt_plan_complete = true; program.resource_tracking_complete = true; program.shader_info_complete = true; - program.values = std::make_shared(); - program.values->block_storage.push_back(std::make_unique()); - auto *block = program.values->block_storage.back().get(); - program.values->blocks.push_back(block); - program.values->block_info.push_back({.id = 0}); + program.block_storage.push_back(std::make_unique()); + auto *block = program.block_storage.back().get(); + program.blocks.push_back(block); + program.block_info.push_back({.id = 0}); auto &key = block->AppendNewInst(ValueOpcode::LaneId); auto &image = @@ -19242,7 +19210,7 @@ void CheckIndirectImageKeySwitch() { memory.image_sample_flags = ShaderRecompiler::Decoder::ImageSampleFlagLod; memory.image_dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; memory.image_address_components = 3; - program.values->memory_info.push_back(memory); + program.memory_info.push_back(memory); const MemoryFlags memory_flags{0u, 0x10f0u}; uint64_t memory_flag_bits = 0; std::memcpy(&memory_flag_bits, &memory_flags, sizeof(memory_flags)); @@ -19253,11 +19221,11 @@ void CheckIndirectImageKeySwitch() { {Value(&sample), Value(0u)}); block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(&sample_x)}); - program.values->descriptor_sources.resize(2); - program.values->descriptor_sources[0].dword_count = 8; - program.values->descriptor_sources[0].indirect_image = + program.descriptor_sources.resize(2); + program.descriptor_sources[0].dword_count = 8; + program.descriptor_sources[0].indirect_image = DescriptorSource::IndirectImage{0u, 0u, 224u, 12u, 0u}; - program.values->descriptor_sources[1].dword_count = 4; + program.descriptor_sources[1].dword_count = 4; ImageResource root{}; root.source = 0; @@ -20394,8 +20362,8 @@ void CheckPs5GameExampleImageClearRuntimeShape() { const auto code = MakeCode(); auto positive = Compile("exact Prospero kernel", code); - compute.stage.program = - std::make_shared(positive.program); + compute.stage.program = std::make_shared( + std::move(positive.program)); compute.stage.resources = std::make_shared( positive.resources); @@ -20488,7 +20456,8 @@ void CheckEmbeddedFetchVertexOffset() { Require(name, "compile", ShaderRecompiler::TryRecompile(code, options, result, &error), error); - Require(name, "fetch rewrite", vertex.resource_fetch_components[0] == 4, + Require(name, "fetch rewrite", + result.program.info.vertex_fetch_components[0] == 4, "encoded fetch sequence was not recognized and rewritten"); return result; }; @@ -20497,8 +20466,11 @@ void CheckEmbeddedFetchVertexOffset() { u32 index_offset) { ShaderVertexInputInfo vertex; vertex.fetch_embedded = true; - vertex.stage.program = - std::make_shared(result.program); + auto program = std::make_shared(); + program->user_data_base = result.program.user_data_base; + program->info.vertex_offset_sgpr = + result.program.info.vertex_offset_sgpr; + vertex.stage.program = std::move(program); vertex.stage.resources = std::make_shared( result.resources); @@ -22366,52 +22338,55 @@ void CheckImageSpecializationId() { HW::ComputeShaderInfo regs{}; regs.cs_regs.data_addr = reinterpret_cast(code.data()); - ShaderRecompiler::IR::Program identity_program; - identity_program.binding_layout_complete = true; ShaderRecompiler::IR::ImageResource image; image.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - identity_program.info.images.push_back(image); - auto rgb1_program = identity_program; - rgb1_program.info.images[0].shader_swizzle = DstSel(4, 5, 6, 1); - auto converted_program = identity_program; - converted_program.info.images[0].conversion_format = + const auto make_program = [](ShaderRecompiler::IR::ImageResource resource) { + ShaderRecompiler::IR::Program program; + program.binding_layout_complete = true; + program.info.images.push_back(std::move(resource)); + return program; + }; + auto rgb1_image = image; + rgb1_image.shader_swizzle = DstSel(4, 5, 6, 1); + auto converted_image = image; + converted_image.conversion_format = Prospero::BufferFormat::k11_11_10UInt; - auto mip1_program = identity_program; - mip1_program.info.images[0].mip_mode = - ShaderRecompiler::IR::ImageMipMode::DynamicStorage; - auto mip3_program = mip1_program; - mip3_program.info.images[0].mip_count = 3; - auto array_program = identity_program; - array_program.info.images[0].dimension = + auto mip1_image = image; + mip1_image.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage; + auto mip3_image = mip1_image; + mip3_image.mip_count = 3; + auto array_image = image; + array_image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - auto cube_program = array_program; - cube_program.info.images[0].cube = true; + auto cube_image = array_image; + cube_image.cube = true; const auto resources = std::make_shared(); ShaderComputeInputInfo identity_info; - identity_info.stage.program = - std::make_shared(identity_program); + identity_info.stage.program = std::make_shared( + make_program(image)); identity_info.stage.resources = resources; auto rgb1_info = identity_info; - rgb1_info.stage.program = - std::make_shared(rgb1_program); + rgb1_info.stage.program = std::make_shared( + make_program(rgb1_image)); auto converted_info = identity_info; converted_info.stage.program = - std::make_shared(converted_program); + std::make_shared( + make_program(converted_image)); auto mip1_info = identity_info; - mip1_info.stage.program = - std::make_shared(mip1_program); + mip1_info.stage.program = std::make_shared( + make_program(mip1_image)); auto mip3_info = identity_info; - mip3_info.stage.program = - std::make_shared(mip3_program); + mip3_info.stage.program = std::make_shared( + make_program(mip3_image)); auto array_info = identity_info; - array_info.stage.program = - std::make_shared(array_program); + array_info.stage.program = std::make_shared( + make_program(array_image)); auto cube_info = identity_info; - cube_info.stage.program = - std::make_shared(cube_program); + cube_info.stage.program = std::make_shared( + make_program(cube_image)); const auto identity_id = ShaderGetIdCS(regs, identity_info, true); const auto rgb1_id = ShaderGetIdCS(regs, rgb1_info, true); @@ -22452,8 +22427,6 @@ void CheckImageSpecializationId() { "forcing point filtering enabled mipmapping on a base-mip sampler"); ShaderRecompiler::IR::Program mixed_sampler_program; - mixed_sampler_program.values = - std::make_shared(); mixed_sampler_program.resource_tracking_complete = true; ShaderRecompiler::IR::ImageResource sampled_image; sampled_image.kind = ShaderRecompiler::IR::ResourceKind::Image; @@ -22500,8 +22473,6 @@ void CheckImageSpecializationId() { "native variants"); ShaderRecompiler::IR::Program packed_atomic_program; - packed_atomic_program.values = - std::make_shared(); packed_atomic_program.resource_tracking_complete = true; auto packed_atomic_image = sampled_image; packed_atomic_image.kind = @@ -23245,7 +23216,7 @@ void CheckEmbeddedFetchLaneSpill() { Require("EmbeddedFetchLaneSpill", "compile", ShaderRecompiler::TryRecompile(code, options, result, &error), error); Require("EmbeddedFetchLaneSpill", "fetch rewrite", - vertex.resource_fetch_components[0] == 4, + result.program.info.vertex_fetch_components[0] == 4, "lane-spilled fetch descriptor was not recognized and rewritten"); std::printf("[host] %-32s ok\n", "EmbeddedFetchLaneSpill"); } diff --git a/tests/ShaderStageRuntimeTests.cpp b/tests/ShaderStageRuntimeTests.cpp index 82b278ef0..25e5fbd03 100644 --- a/tests/ShaderStageRuntimeTests.cpp +++ b/tests/ShaderStageRuntimeTests.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/shader.h" @@ -23,12 +23,11 @@ bool RejectSpecializationRead(void *userdata, uint64_t, uint32_t *) { Libs::Graphics::ShaderRecompiler::IR::Block & AddValueBlock(Libs::Graphics::ShaderRecompiler::IR::Program &program) { using namespace Libs::Graphics::ShaderRecompiler::IR; - program.values = std::make_shared(); auto block = std::make_unique(); auto *result = block.get(); - program.values->blocks.push_back(result); - program.values->block_info.push_back({.id = 0}); - program.values->block_storage.push_back(std::move(block)); + program.blocks.push_back(result); + program.block_info.push_back({.id = 0}); + program.block_storage.push_back(std::move(block)); return *result; } @@ -44,7 +43,7 @@ SrtProgram(uint64_t address) { MemoryInfo memory; memory.kind = ResourceKind::ScalarAddress; memory.planning_only = true; - program.values->memory_info.push_back(memory); + program.memory_info.push_back(memory); const auto low = Value(static_cast(address)); const auto high = Value(static_cast(address >> 32u)); auto &handle = @@ -53,7 +52,7 @@ SrtProgram(uint64_t address) { ValueOpcode::LoadAddressU32, {Value(&handle), Value(0u), Value(0u), Value(true)}); raw.SetFlags(MemoryFlags{.index = 0, .pc = 0x40}); - program.values->srt_reads.push_back({Value(&raw), 0, 0x40}); + program.srt_reads.push_back({Value(&raw), 0, 0x40}); auto &srt = value_block.AppendNewInst(ValueOpcode::GetSrtResource); auto &flat = value_block.AppendNewInst(ValueOpcode::ReadConst, @@ -62,7 +61,7 @@ SrtProgram(uint64_t address) { source.dwords[0] = Value(&flat); source.dwords[1] = Value(0u); source.dword_count = 2; - program.values->descriptor_sources.push_back(source); + program.descriptor_sources.push_back(source); return std::make_shared(std::move(program)); } @@ -141,5 +140,5 @@ int main() { #include "graphics/shader/recompiler/ir/Block.cpp" #include "graphics/shader/recompiler/ir/Type.cpp" #include "graphics/shader/recompiler/ir/Value.cpp" -#include "graphics/shader/recompiler/ir/ValueProgram.cpp" +#include "graphics/shader/recompiler/ir/Program.cpp" #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index d26fc40e9..ba2d7a75c 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -16,7 +16,6 @@ #include "graphics/shader/recompiler/frontend/translate/Translate.h" #include "graphics/shader/recompiler/ir/IREmitter.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/ValueProgram.h" #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" @@ -1112,12 +1111,11 @@ void TestSpirvRequirementsAnalysis() { Program program; program.stage = ShaderType::Pixel; - program.values = std::make_shared(); - program.values->block_storage.push_back(std::make_unique()); - auto *block = program.values->block_storage.back().get(); - program.values->blocks.push_back(block); - program.values->memory_info.push_back({.kind = ResourceKind::Lds}); - program.values->export_info.push_back({.vm = true}); + program.block_storage.push_back(std::make_unique()); + auto *block = program.block_storage.back().get(); + program.blocks.push_back(block); + program.memory_info.push_back({.kind = ResourceKind::Lds}); + program.export_info.push_back({.vm = true}); block->AppendNewInst(ValueOpcode::DppMoveU32, {Value(0u), Value(true)}); block->AppendNewInst(ValueOpcode::ImageQueryLod, @@ -1174,11 +1172,10 @@ void TestSpirvRequirementsAnalysis() { add_tid.stage = ShaderType::Compute; add_tid.info.buffers.resize(1); add_tid.info.buffers[0].packed_stride = 1u << 20u; - add_tid.values = std::make_shared(); - add_tid.values->block_storage.push_back(std::make_unique()); - auto *add_tid_block = add_tid.values->block_storage.back().get(); - add_tid.values->blocks.push_back(add_tid_block); - add_tid.values->memory_info.push_back( + add_tid.block_storage.push_back(std::make_unique()); + auto *add_tid_block = add_tid.block_storage.back().get(); + add_tid.blocks.push_back(add_tid_block); + add_tid.memory_info.push_back( {.kind = ResourceKind::Buffer, .resource = 0}); auto &buffer = add_tid_block->AppendNewInst( ValueOpcode::GetBufferResource, @@ -1196,14 +1193,14 @@ void TestSpirvRequirementsAnalysis() { !add_tid_requirements.subgroup_shuffle, "buffer ADD_TID requested the wrong subgroup contract"); - add_tid.values->memory_info[0].resource = 1; + add_tid.memory_info[0].resource = 1; analysis_error.clear(); Check(!ShaderRecompiler::Spirv::AnalyzeProgramRequirements(add_tid, &analysis_error) && !add_tid.spirv_requirements.has_value() && Common::ContainsStr(analysis_error, "invalid resource metadata"), "invalid buffer requirement metadata was accepted"); - add_tid.values->memory_info[0].resource = 0; + add_tid.memory_info[0].resource = 0; add_tid.stage = ShaderType::Vertex; analysis_error.clear(); @@ -1586,7 +1583,7 @@ void TestNewShaderRecompilerSopkWaitcntMarkers() { Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x00000000"), "SOPK waitcnt did not lower to an IR marker"); Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x0000ffff"), - "SOPK waitcnt marker immediate was not lowered as 16-bit unsigned"); + "SOPK waitcnt marker immediate was not translated as 16-bit unsigned"); CheckSpirvBinaryValidates(result.spirv); } @@ -3287,7 +3284,7 @@ void TestNewShaderRecompilerCapturedVop1SdwaByteConvert() { size_t extracts = 0; size_t converts = 0; - for (const auto *block : result.program.values->blocks) { + for (const auto *block : result.program.blocks) { for (const auto &inst : *block) { extracts += inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::BitFieldUExtract; @@ -3657,7 +3654,7 @@ void TestNewShaderRecompilerScalarB64Alu() { Check(SpirvContainsOpcode(result.spirv, 205), "SPIR-V binary does not contain OpBitCount for scalar B64 bit count"); uint32_t componentwise_u64_values = 0; - for (const auto *block : result.program.values->blocks) { + for (const auto *block : result.program.blocks) { for (const auto &value : *block) { switch (value.GetOpcode()) { case ShaderRecompiler::IR::ValueOpcode::BitwiseAnd64: @@ -3674,7 +3671,7 @@ void TestNewShaderRecompilerScalarB64Alu() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerScalarB64LaneLowering() { +void TestNewShaderRecompilerScalarB64LaneTranslation() { const uint32_t shader[] = { EncodeSop1(0x04, 2, 0), // s_mov_b64 s[2:3], s[0:1] EncodeSop1(0x08, 4, 2), // s_not_b64 s[4:5], s[2:3] @@ -3698,7 +3695,7 @@ void TestNewShaderRecompilerScalarB64LaneLowering() { Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); uint32_t componentwise_u64_values = 0; - for (const auto *block : result.program.values->blocks) { + for (const auto *block : result.program.blocks) { for (const auto &value : *block) { switch (value.GetOpcode()) { case ShaderRecompiler::IR::ValueOpcode::BitwiseAnd64: @@ -3931,10 +3928,10 @@ void CheckNewDecoderUnsupported(const uint32_t *shader, uint32_t words, options.dump_ir = true; ShaderRecompiler::CompileResult result; Check(!ShaderRecompiler::TryRecompile(code, options, result, &error), - "unsupported opcode unexpectedly lowered without implementation"); - Check(Common::ContainsStr(error, "no IR lowering") || + "unsupported opcode unexpectedly translated without implementation"); + Check(Common::ContainsStr(error, "no IR translation") || Common::ContainsStr(error, "unsupported decoded"), - "unsupported lowering error was not explicit"); + "unsupported translation error was not explicit"); } void TestNewShaderDecoderArchitecture() { @@ -4084,29 +4081,141 @@ void TestNewShaderRecompilerIrLookupMissFailsExplicitly() { cfg.entry_block = 0u; ShaderRecompiler::IR::Program ir; + ShaderComputeInputInfo compute{}; + ShaderRecompiler::Frontend::TranslateOptions options{}; + options.stage = ShaderType::Compute; + options.wave_size = 64u; + options.compute = &compute; std::string error; - Check(!ShaderRecompiler::IR::LowerProgram(decoded, cfg, ShaderType::Compute, - 64u, ir, &error), - "missing decoder-to-IR mapping unexpectedly lowered as an instruction"); - Check(Common::ContainsStr(error, "no IR lowering"), + Check(!ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, + &error), + "missing decoder-to-IR mapping unexpectedly translated as an instruction"); + Check(Common::ContainsStr(error, "no IR translation"), "missing decoder-to-IR mapping did not report an explicit error"); Check(ir.blocks.empty(), "missing decoder-to-IR mapping emitted a fallback IR block"); + ir.block_storage.push_back(std::make_unique()); + ir.blocks.push_back(ir.block_storage.back().get()); + ir.block_info.emplace_back(); + options.wave_size = 16u; + error.clear(); + Check(!ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, + &error), + "invalid translation options unexpectedly succeeded"); + Check(ir.blocks.empty() && ir.block_storage.empty() && ir.block_info.empty(), + "early translation failure retained a stale output program"); + options.wave_size = 64u; + decoded.instructions.front().family = ShaderRecompiler::Decoder::Family::MUBUF; decoded.instructions.front().opcode = ShaderRecompiler::Decoder::Opcode::V_ADD_NC_U32; error.clear(); - Check(!ShaderRecompiler::IR::LowerProgram(decoded, cfg, ShaderType::Compute, - 64u, ir, &error), - "memory-family opcode bypassed specialized memory lowering"); - Check(Common::ContainsStr(error, "memory-family opcode") && - Common::ContainsStr(error, "specialized IR lowering"), - "memory-family lowering bypass did not report an explicit error"); + Check(!ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, + &error), + "memory-family opcode bypassed specialized memory translation"); + Check(Common::ContainsStr(error, "does not belong to decoded family"), + "memory-family mismatch did not report an explicit error"); + + auto &store = decoded.instructions.front(); + store.opcode = ShaderRecompiler::Decoder::Opcode::BUFFER_STORE_DWORD; + store.data_dwords = 1u; + store.data_bits = 32u; + store.offen = true; + store.src_count = 3u; + store.src0.kind = ShaderRecompiler::Decoder::OperandKind::Vgpr; + store.src1.kind = ShaderRecompiler::Decoder::OperandKind::Sgpr; + store.src2.kind = + ShaderRecompiler::Decoder::OperandKind::IntegerInlineConstant; + store.dst.kind = ShaderRecompiler::Decoder::OperandKind::Unknown; + error.clear(); + Check(!ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, + &error), + "buffer store accepted an unreadable data operand"); + Check(Common::ContainsStr(error, + "decoded operand cannot be used as an IR register"), + "buffer store data rejection did not report an explicit error"); + + store = {}; + store.family = ShaderRecompiler::Decoder::Family::MUBUF; + store.opcode = ShaderRecompiler::Decoder::Opcode::UNSUPPORTED; + store.unsupported_reason = "focused unsupported MUBUF diagnostic"; + error.clear(); + Check(!ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, + &error), + "unsupported memory instruction unexpectedly translated"); + Check(Common::ContainsStr(error, "focused unsupported MUBUF diagnostic"), + "unsupported memory instruction lost its decoder diagnostic"); + + store = {}; + store.family = ShaderRecompiler::Decoder::Family::VOP1; + store.opcode = ShaderRecompiler::Decoder::Opcode::V_MOVRELS_B32; + store.dst.kind = ShaderRecompiler::Decoder::OperandKind::Sgpr; + store.src0.kind = ShaderRecompiler::Decoder::OperandKind::Vgpr; + store.src_count = 1u; + error.clear(); + Check(!ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, + &error), + "invalid V_MOVRELS_B32 operands unexpectedly translated"); + Check(Common::ContainsStr(error, + "requires VGPR source and destination"), + "invalid V_MOVRELS_B32 operands did not fail recoverably"); + + store = {}; + store.family = ShaderRecompiler::Decoder::Family::SOP1; + store.opcode = ShaderRecompiler::Decoder::Opcode::S_GETPC_B64; + store.dst.kind = ShaderRecompiler::Decoder::OperandKind::Sgpr; + store.dst.reg = 105u; + error.clear(); + Check(ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, + &error), + error.c_str()); + bool getpc_wrote_vcc_lo = false; + for (const auto *typed_block : ir.blocks) { + for (const auto &typed_inst : *typed_block) { + getpc_wrote_vcc_lo |= + typed_inst.GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::SetVccLo; + } + } + Check(getpc_wrote_vcc_lo, + "S_GETPC_B64 did not advance s105 through the VCC alias"); + + const ShaderRecompiler::Decoder::OperandKind canonical_zero_masks[] = { + ShaderRecompiler::Decoder::OperandKind::Null, + ShaderRecompiler::Decoder::OperandKind::PopsExitingWaveId, + ShaderRecompiler::Decoder::OperandKind::VccZ, + ShaderRecompiler::Decoder::OperandKind::ExecZ, + }; + for (const auto kind : canonical_zero_masks) { + store = {}; + store.family = ShaderRecompiler::Decoder::Family::SOP1; + store.opcode = ShaderRecompiler::Decoder::Opcode::S_NOT_B64; + store.dst.kind = ShaderRecompiler::Decoder::OperandKind::Sgpr; + store.src0.kind = kind; + store.src_count = 1u; + error.clear(); + Check(ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, + &error), + error.c_str()); + bool preserved_mask_tag = false; + for (const auto *typed_block : ir.blocks) { + for (const auto &typed_inst : *typed_block) { + if (typed_inst.GetOpcode() != + ShaderRecompiler::IR::ValueOpcode::SetScalarMaskTag) { + continue; + } + const auto valid = typed_inst.Arg(1).Resolve(); + preserved_mask_tag |= valid.IsImmediate() && valid.U1(); + } + } + Check(preserved_mask_tag, + "canonical zero/condition operand lost scalar mask validity"); + } } -void TestNewShaderRecompilerMemoryFamilyLowering() { +void TestNewShaderRecompilerMemoryFamilyTranslation() { const uint32_t shader[] = { EncodeSmem0(0x00, 0, 4), 0u, // s_load_dword s0 @@ -4159,8 +4268,8 @@ void TestNewShaderRecompilerMemoryFamilyLowering() { Check(Common::ContainsStr(result.decoded_dump, "image_sample"), "new decoder did not decode MIMG sample"); Check( - !Common::ContainsStr(result.decoded_dump, "lowering is not implemented"), - "implemented memory decode still reports unsupported lowering"); + !Common::ContainsStr(result.decoded_dump, "translation is not implemented"), + "implemented memory decode still reports unsupported translation"); Check(Common::ContainsStr(result.ir_dump, "SLoadDword s0"), "SMEM load did not lower to IR"); Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s1"), @@ -4198,16 +4307,16 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { ShaderRecompiler::IR::ValueOpcode opcode, ShaderRecompiler::IR::ResourceKind kind) { size_t count = 0; - for (const auto *block : program.values->blocks) { + for (const auto *block : program.blocks) { for (const auto &inst : *block) { if (inst.GetOpcode() != opcode) { continue; } const auto index = inst.Flags().index; - Check(index < program.values->memory_info.size(), + Check(index < program.memory_info.size(), "typed scalar memory operation has invalid metadata"); - const auto &memory = program.values->memory_info[index]; + const auto &memory = program.memory_info[index]; Check(memory.kind == kind && memory.resource == 0u && !memory.planning_only, "typed scalar memory operation has the wrong live domain"); @@ -4317,7 +4426,7 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { CheckSpirvBinaryValidates(buffer.spirv); } -void TestNewShaderRecompilerImageQueryLowering() { +void TestNewShaderRecompilerImageQueryTranslation() { const uint32_t shader[] = { EncodeMimg0(0x60, 0x3), EncodeMimg1(6, 0, 0, 1), // image_get_lod @@ -5152,7 +5261,7 @@ void TestNewShaderRecompilerImageLoad2DMsaa() { "2D-MSAA load incorrectly emitted its fragment ID as a mip level"); } -void TestNewShaderRecompilerImageStoreLowering() { +void TestNewShaderRecompilerImageStoreTranslation() { const uint32_t shader[] = { EncodeMimg0(0x08, 0xf), EncodeMimg1(20, 0, 0, 4), // image_store @@ -5226,17 +5335,21 @@ void TestNewShaderRecompilerStorageImage3DDescriptorVariant() { error.c_str()); Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), "MIMG store did not decode the RDNA2 3D instruction dimension"); - const auto has_3d_store = std::any_of( - result.program.blocks.begin(), result.program.blocks.end(), - [](const auto &block) { - return std::any_of( - block.instructions.begin(), block.instructions.end(), - [](const auto &inst) { - return inst.op == ShaderRecompiler::IR::Opcode::ImageStore && - inst.memory.image_dimension == - ShaderRecompiler::Decoder::ImageDimension::Dim3D && - inst.memory.image_address_components == 3; - }); + const auto has_3d_store = std::ranges::any_of( + result.program.blocks, [&](const auto *block) { + return std::ranges::any_of(*block, [&](const auto &inst) { + if (inst.GetOpcode() != + ShaderRecompiler::IR::ValueOpcode::ImageWrite) { + return false; + } + const auto index = + inst.template Flags().index; + return index < result.program.memory_info.size() && + result.program.memory_info[index].image_dimension == + ShaderRecompiler::Decoder::ImageDimension::Dim3D && + result.program.memory_info[index].image_address_components == + 3u; + }); }); Check(has_3d_store, "3D storage image store did not preserve three address components"); @@ -5296,7 +5409,7 @@ void TestNewShaderRecompilerStorageImage2DDescriptorOverridesMimg3D() { "binding"); } -void TestNewShaderRecompilerImageAtomicLowering() { +void TestNewShaderRecompilerImageAtomicTranslation() { const uint32_t shader[] = { EncodeMimg0(0x11, 0x1, true), EncodeMimg1(52, 0, 0, 1), // image_atomic_add @@ -5369,7 +5482,7 @@ void TestNewShaderRecompilerImageAtomicLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerVintrpLowering() { +void TestNewShaderRecompilerVintrpTranslation() { const uint32_t shader[] = { EncodeVintrp(0, 10, 1, 2, 4), // v_interp_p1_f32 v10, v4, attr1.z EncodeVintrp(1, 11, 1, 2, 4), // v_interp_p2_f32 v11, v4, attr1.z @@ -5510,7 +5623,7 @@ void TestNewShaderRecompilerVintrpLowering() { CheckSpirvBinaryValidates(no_perspective_result.spirv); } -void TestCustomVintrpMovLowering() { +void TestCustomVintrpMovTranslation() { const uint32_t shader[] = { EncodeVintrp(2, 12, 0, 3, 2), EncodeVintrp(2, 13, 0, 3, 0), EncodeVintrp(2, 14, 0, 3, 1), EncodeVop2(0x03, 15, 12 + 256, 0), @@ -5755,34 +5868,66 @@ void TestNewShaderRecompilerNativeWideScalarMemoryIr() { ShaderRecompiler::Decoder::Program decoded; ShaderRecompiler::CFG::Graph graph; ShaderRecompiler::IR::Program ir; + ShaderComputeInputInfo compute{}; + ShaderRecompiler::Frontend::TranslateOptions translate_options{}; + translate_options.stage = ShaderType::Compute; + translate_options.wave_size = 64u; + translate_options.compute = &compute; std::string error; Check(ShaderRecompiler::Decoder::DecodeProgram(shader, decoded, &error) && ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) && - ShaderRecompiler::IR::LowerProgram( - decoded, graph, ShaderType::Compute, 64, ir, &error), + ShaderRecompiler::Frontend::TranslateProgram( + decoded, graph, translate_options, ir, &error), error.c_str()); std::vector address_widths; std::vector buffer_widths; - bool buffer_x2_targets_vcc = false; - for (const auto &block : ir.blocks) { - for (const auto &inst : block.instructions) { - if (inst.op == ShaderRecompiler::IR::Opcode::SLoadDword) { - address_widths.push_back(inst.memory.data_dwords); - } else if (inst.op == ShaderRecompiler::IR::Opcode::SBufferLoadDword) { - buffer_widths.push_back(inst.memory.data_dwords); - if (inst.memory.data_dwords == 2u) { - buffer_x2_targets_vcc = - inst.dst.reg.file == ShaderRecompiler::IR::RegisterFile::Vcc && - inst.dst.reg.index == 0u; - } - } + for (const auto &memory : ir.memory_info) { + if (memory.component_index != 0u) { + continue; + } + if (memory.kind == ShaderRecompiler::IR::ResourceKind::ScalarAddress) { + address_widths.push_back(memory.component_count); + } else if (memory.kind == ShaderRecompiler::IR::ResourceKind::ScalarBuffer) { + buffer_widths.push_back(memory.component_count); } } const std::vector expected{1u, 2u, 4u, 8u, 16u}; Check(address_widths == expected && buffer_widths == expected, - "SMEM x1/x2/x4/x8/x16 did not remain one width-bearing ShaderIR " - "operation each"); - Check(buffer_x2_targets_vcc, + "SMEM x1/x2/x4/x8/x16 lost their typed component spans"); + + std::array buffer_x2_targets_vcc{}; + for (const auto *block : ir.blocks) { + for (const auto &inst : *block) { + uint32_t component = UINT32_MAX; + if (inst.GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::SetVccLo) { + component = 0u; + } else if (inst.GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::SetVccHi) { + component = 1u; + } + const auto *load = component < buffer_x2_targets_vcc.size() + ? inst.Arg(0).Resolve().TryInstruction() + : nullptr; + if (load == nullptr || + load->GetOpcode() != + ShaderRecompiler::IR::ValueOpcode::ReadConstBuffer) { + continue; + } + const auto flags = + load->Flags(); + if (flags.index >= ir.memory_info.size()) { + continue; + } + const auto &memory = ir.memory_info[flags.index]; + buffer_x2_targets_vcc[component] = + memory.kind == + ShaderRecompiler::IR::ResourceKind::ScalarBuffer && + memory.component_count == 2u && + memory.component_index == component; + } + } + Check(buffer_x2_targets_vcc[0] && buffer_x2_targets_vcc[1], "s_buffer_load_dwordx2 did not preserve its VCC destination span"); const auto reject = [&](uint32_t opcode, uint32_t dst, uint32_t sbase, @@ -5797,9 +5942,9 @@ void TestNewShaderRecompilerNativeWideScalarMemoryIr() { &invalid_error) && ShaderRecompiler::CFG::BuildGraph(invalid_decoded, invalid_graph, &invalid_error) && - !ShaderRecompiler::IR::LowerProgram( - invalid_decoded, invalid_graph, ShaderType::Compute, 64, - invalid_ir, &invalid_error) && + !ShaderRecompiler::Frontend::TranslateProgram( + invalid_decoded, invalid_graph, translate_options, invalid_ir, + &invalid_error) && Common::ContainsStr(invalid_error, reason), "invalid SMEM register span was not rejected"); }; @@ -5833,7 +5978,7 @@ void TestNewShaderRecompilerNativeWideBufferIr() { Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); std::array counts{}; - for (const auto *block : result.program.values->blocks) { + for (const auto *block : result.program.blocks) { for (const auto &inst : *block) { switch (inst.GetOpcode()) { case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32: @@ -5864,11 +6009,11 @@ void TestNewShaderRecompilerNativeWideBufferIr() { } } Check(counts == std::array{0u, 1u, 1u, 1u, 1u, 1u, 1u}, - "native-wide buffer lowering retained scalar siblings or lost a width"); + "native-wide buffer translation retained scalar siblings or lost a width"); CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerBufferSignedLoadLowering() { +void TestNewShaderRecompilerBufferSignedLoadTranslation() { const uint32_t shader[] = { EncodeMubuf0(0x09, 3), EncodeMubuf1(46, 0, 1), // buffer_load_sbyte EncodeMubuf0(0x0b, 6), EncodeMubuf1(47, 0, 1), // buffer_load_sshort @@ -5908,7 +6053,7 @@ void TestNewShaderRecompilerBufferSignedLoadLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerBufferSubDwordStoreLowering() { +void TestNewShaderRecompilerBufferSubDwordStoreTranslation() { const uint32_t shader[] = { EncodeMubuf0(0x18, 2), EncodeMubuf1(48, 0, 1), // buffer_store_byte EncodeMubuf0(0x1a, 4), EncodeMubuf1(49, 0, 1), // buffer_store_short @@ -5949,7 +6094,7 @@ void TestNewShaderRecompilerBufferSubDwordStoreLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerMubufFormatLowering() { +void TestNewShaderRecompilerMubufFormatTranslation() { const uint32_t shader[] = { EncodeMubuf0(0x00, 4), EncodeMubuf1(84, 0, 1), // buffer_load_format_x @@ -6038,7 +6183,7 @@ void TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly() { "formatted store SPIR-V baked descriptor NumRecords into a store guard"); } -void TestNewShaderRecompilerTypedBufferLowering() { +void TestNewShaderRecompilerTypedBufferTranslation() { const uint32_t shader[] = { EncodeMtbuf0(0x00, 14, 7, 4), EncodeMtbuf1(0x00, 60, 0, 1), // tbuffer_load_format_x @@ -6099,7 +6244,7 @@ void TestNewShaderRecompilerTypedBufferLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerFlatOldBackedLowering() { +void TestNewShaderRecompilerFlatOldBackedTranslation() { const uint32_t shader[] = { EncodeFlat0(0x08, 0, 4), EncodeFlat1(9, 0x7d, 0, 1), // flat_load_ubyte @@ -6150,7 +6295,7 @@ void TestNewShaderRecompilerUnbasedFlatRequiresTranslator() { "unbased FLAT compiled without an explicit translator"); } -void TestNewShaderRecompilerFlatSignedLoadLowering() { +void TestNewShaderRecompilerFlatSignedLoadTranslation() { const uint32_t shader[] = { EncodeFlat0(0x09, 0, 4), EncodeFlat1(10, 0x7d, 0, 1), // flat_load_sbyte @@ -6199,7 +6344,7 @@ void TestNewShaderRecompilerFlatSignedLoadLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerFlatStoreLowering() { +void TestNewShaderRecompilerFlatStoreTranslation() { const uint32_t shader[] = { EncodeFlat0(0x18, 0, 2), EncodeFlat1(0, 0x7d, 72, 1), // flat_store_byte @@ -6278,7 +6423,7 @@ void TestNewShaderRecompilerFlatStoreLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerAtomicLowering() { +void TestNewShaderRecompilerAtomicTranslation() { const uint32_t shader[] = { EncodeMubuf0(0x30, 4, true, true), EncodeMubuf1(0, 0, 1), // buffer_atomic_swap @@ -6457,7 +6602,7 @@ void TestNewShaderRecompilerAtomicLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerDsReadWrite2Lowering() { +void TestNewShaderRecompilerDsReadWrite2Translation() { const uint32_t shader[] = { EncodeDs0(0x0e, (3u << 8u) | 1u), EncodeDs1Ex(0, 61, 60, 1), @@ -6472,28 +6617,18 @@ void TestNewShaderRecompilerDsReadWrite2Lowering() { ShaderRecompiler::Decoder::Program decoded; ShaderRecompiler::CFG::Graph graph; - ShaderRecompiler::IR::Program lowered; + ShaderRecompiler::IR::Program typed; + ShaderComputeInputInfo compute{}; + ShaderRecompiler::Frontend::TranslateOptions translate_options{}; + translate_options.stage = ShaderType::Compute; + translate_options.wave_size = 64u; + translate_options.compute = &compute; std::string error; Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, - &error) && + &error) && ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) && - ShaderRecompiler::IR::LowerProgram( - decoded, graph, ShaderType::Compute, 64, lowered, &error), - error.c_str()); - uint32_t pair_reads = 0; - uint32_t pair_writes = 0; - for (const auto &block : lowered.blocks) { - for (const auto &inst : block.instructions) { - pair_reads += inst.op == ShaderRecompiler::IR::Opcode::DsRead2B32; - pair_writes += inst.op == ShaderRecompiler::IR::Opcode::DsWrite2B32; - } - } - Check(pair_reads == 3u && pair_writes == 1u, - "DS read/write2 was split before typed SSA"); - ShaderRecompiler::IR::ValueProgram typed; - ShaderComputeInputInfo compute{}; - Check(ShaderRecompiler::Frontend::TranslateProgram(lowered, typed, nullptr, - nullptr, &compute, &error), + ShaderRecompiler::Frontend::TranslateProgram( + decoded, graph, translate_options, typed, &error), error.c_str()); uint32_t scalar_loads = 0; uint32_t vector_loads = 0; @@ -6552,7 +6687,7 @@ void TestNewShaderRecompilerDsReadWrite2Lowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerDsSubDwordLowering() { +void TestNewShaderRecompilerDsSubDwordTranslation() { const uint32_t shader[] = { EncodeDs0(0x1e, 1), EncodeDs1(0, 40, 1), // ds_write_b8 v40, v1 EncodeDs0(0x1f, 2), EncodeDs1(0, 41, 1), // ds_write_b16 v41, v1 @@ -6622,7 +6757,7 @@ void TestNewShaderRecompilerDsSubDwordLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerDsWideAndAtomicLowering() { +void TestNewShaderRecompilerDsWideAndAtomicTranslation() { const uint32_t shader[] = { EncodeDs0(0x4d, 4), EncodeDs1(0, 10, 1), // ds_write_b64 v[10:11], v1 EncodeDs0(0xde, 8), EncodeDs1(0, 12, 1), // ds_write_b96 v[12:14], v1 @@ -6638,28 +6773,18 @@ void TestNewShaderRecompilerDsWideAndAtomicLowering() { ShaderRecompiler::Decoder::Program decoded; ShaderRecompiler::CFG::Graph graph; - ShaderRecompiler::IR::Program lowered; + ShaderRecompiler::IR::Program typed; + ShaderComputeInputInfo compute{}; + ShaderRecompiler::Frontend::TranslateOptions translate_options{}; + translate_options.stage = ShaderType::Compute; + translate_options.wave_size = 64u; + translate_options.compute = &compute; std::string error; Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, - &error) && + &error) && ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) && - ShaderRecompiler::IR::LowerProgram( - decoded, graph, ShaderType::Compute, 64, lowered, &error), - error.c_str()); - uint32_t native_reads = 0; - uint32_t native_writes = 0; - for (const auto &block : lowered.blocks) { - for (const auto &inst : block.instructions) { - native_reads += inst.op == ShaderRecompiler::IR::Opcode::DsReadB32; - native_writes += inst.op == ShaderRecompiler::IR::Opcode::DsWriteB32; - } - } - Check(native_reads == 3u && native_writes == 3u, - "wide DS transfers were split before typed SSA"); - ShaderRecompiler::IR::ValueProgram typed; - ShaderComputeInputInfo compute{}; - Check(ShaderRecompiler::Frontend::TranslateProgram(lowered, typed, nullptr, - nullptr, &compute, &error), + ShaderRecompiler::Frontend::TranslateProgram( + decoded, graph, translate_options, typed, &error), error.c_str()); std::array load_widths{}; std::array store_widths{}; @@ -6721,7 +6846,7 @@ void TestNewShaderRecompilerDsWideAndAtomicLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerDsSwizzleLowering() { +void TestNewShaderRecompilerDsSwizzleTranslation() { const uint32_t shader[] = { EncodeDs0(0x35, 0x001f), EncodeDs1(8, 0, 5), // ds_swizzle_b32 v8, v5 @@ -6771,7 +6896,7 @@ void TestNewShaderRecompilerDsSwizzleLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerDsAddtidLowering() { +void TestNewShaderRecompilerDsAddtidTranslation() { const uint32_t shader[] = { EncodeSMovB32(124, 132), // m0 = 4 EncodeDs0(0xb0, 8), @@ -6819,7 +6944,7 @@ void TestNewShaderRecompilerDsAddtidLowering() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerDsFloatMinMaxLowering() { +void TestNewShaderRecompilerDsFloatMinMaxTranslation() { const uint32_t shader[] = { EncodeDs0(0x12, 4), EncodeDs1Ex(0, 9, 7, 1), // ds_min_f32 v7, v9, v1 EncodeDs0(0x13, 8), EncodeDs1Ex(0, 10, 8, 1), // ds_max_f32 v8, v10, v1 @@ -6952,9 +7077,9 @@ void TestNewShaderRecompilerStructuredU64Phi() { Check(!result.program.dispatcher_fallback, "U64 Phi fixture did not select structured mode"); - auto program = result.program; + auto program = std::move(result.program); IR::Block *join = nullptr; - for (auto *block : program.values->blocks) { + for (auto *block : program.blocks) { if (block->ImmPredecessors().size() == 2u) { join = block; break; @@ -7822,14 +7947,14 @@ void TestNewShaderRecompilerCfgExecSccSharedArm() { original_coverage, "EXEC/SCC shared-arm structurization changed semantic coverage"); - ShaderRecompiler::IR::Program native_ir; - Check(ShaderRecompiler::IR::LowerProgram(decoded, graph, ShaderType::Compute, - 64, native_ir, &error), - error.c_str()); - ShaderRecompiler::IR::ValueProgram value_ir; + ShaderRecompiler::IR::Program value_ir; ShaderComputeInputInfo compute_info{}; + ShaderRecompiler::Frontend::TranslateOptions translate_options{}; + translate_options.stage = ShaderType::Compute; + translate_options.wave_size = 64u; + translate_options.compute = &compute_info; Check(ShaderRecompiler::Frontend::TranslateProgram( - native_ir, value_ir, nullptr, nullptr, &compute_info, &error), + decoded, graph, translate_options, value_ir, &error), error.c_str()); uint32_t goto_sets = 0; uint32_t goto_gets = 0; @@ -8319,16 +8444,16 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { std::string error; Check(TryRecompile(shader, options, result, &error), error.c_str()); Check(result.program.dispatcher_fallback && - result.program.values->blocks.size() >= 3u, + result.program.blocks.size() >= 3u, "U32x3 spill fixture did not select dispatcher mode"); const auto before = MeasureSpirv(result.spirv); { - auto prologue_program = result.program; - IR::IREmitter definition(prologue_program.values->blocks[0]); + auto prologue_program = std::move(result.program); + IR::IREmitter definition(prologue_program.blocks[0]); const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU32x3, {IR::Value(1u), IR::Value(2u), IR::Value(3u)}); - IR::IREmitter use(prologue_program.values->blocks[1]); + IR::IREmitter use(prologue_program.blocks[1]); use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)}); Check(Spirv::AnalyzeProgramRequirements(prologue_program, &error), error.c_str()); @@ -8343,19 +8468,21 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { "dispatcher spilled a value defined by its dominating prologue"); } - auto program = result.program; - IR::IREmitter definition(program.values->blocks[1]); + CompileResult spill_result; + Check(TryRecompile(shader, options, spill_result, &error), error.c_str()); + auto program = std::move(spill_result.program); + IR::IREmitter definition(program.blocks[1]); const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU32x3, {IR::Value(1u), IR::Value(2u), IR::Value(3u)}); - IR::IREmitter use(program.values->blocks[2]); + IR::IREmitter use(program.blocks[2]); use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)}); use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(1u)}); Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str()); std::vector spirv; - Check(Spirv::EmitProgram(program, result.resources, options.input_info, spirv, - &error), + Check(Spirv::EmitProgram(program, spill_result.resources, + options.input_info, spirv, &error), error.c_str()); CheckSpirvBinaryValidates(spirv); const auto after = MeasureSpirv(spirv); @@ -8365,7 +8492,7 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { "dispatcher did not use one canonical U32x3 spill slot"); } -void TestNewShaderRecompilerU64PairLowering() { +void TestNewShaderRecompilerU64PairTranslation() { using namespace ShaderRecompiler; auto options = MakeCompileOptions(ShaderType::Compute); @@ -8394,22 +8521,6 @@ void TestNewShaderRecompilerU64PairLowering() { Check(Common::ContainsStr(result.decoded_dump, "V_LSHRREV_B64 v1, v12, vcc_lo"), "captured VOP3 V_LSHRREV_B64 was not present in the decoded dump"); - Decoder::Program decoded_lshrrev_b64_program; - CFG::Graph graph_lshrrev_b64; - IR::Program native_lshrrev_b64; - Check(Decoder::DecodeProgram(lshrrev_b64_shader, - decoded_lshrrev_b64_program, &error), - error.c_str()); - Check(CFG::BuildGraph(decoded_lshrrev_b64_program, graph_lshrrev_b64, - &error) && - CFG::Structurize(graph_lshrrev_b64, &error), - error.c_str()); - Check(IR::LowerProgram(decoded_lshrrev_b64_program, graph_lshrrev_b64, - ShaderType::Compute, 64u, native_lshrrev_b64, &error), - error.c_str()); - Check(Common::ContainsStr(IR::ProgramToString(native_lshrrev_b64), - "ShiftRightLogicalU64 v1, vcc_lo, v12"), - "V_LSHRREV_B64 did not reverse its encoded operands for pair-U64 IR"); CheckSpirvBinaryValidates(result.spirv); const uint32_t lshlrev_b64_shader[] = { @@ -8435,22 +8546,6 @@ void TestNewShaderRecompilerU64PairLowering() { Check(Common::ContainsStr(result.decoded_dump, "V_LSHLREV_B64 v33, v3, 1"), "captured VOP3 V_LSHLREV_B64 was not present in the decoded dump"); - Decoder::Program decoded_lshlrev_b64_program; - CFG::Graph graph_lshlrev_b64; - IR::Program native_lshlrev_b64; - Check(Decoder::DecodeProgram(lshlrev_b64_shader, - decoded_lshlrev_b64_program, &error), - error.c_str()); - Check(CFG::BuildGraph(decoded_lshlrev_b64_program, graph_lshlrev_b64, - &error) && - CFG::Structurize(graph_lshlrev_b64, &error), - error.c_str()); - Check(IR::LowerProgram(decoded_lshlrev_b64_program, graph_lshlrev_b64, - ShaderType::Compute, 64u, native_lshlrev_b64, &error), - error.c_str()); - Check(Common::ContainsStr(IR::ProgramToString(native_lshlrev_b64), - "ShiftLeftLogicalU64 v33, 0x00000001, v3"), - "V_LSHLREV_B64 did not reverse its encoded operands for pair-U64 IR"); CheckSpirvBinaryValidates(result.spirv); const uint32_t cmpx_i64_shader[] = { @@ -8471,20 +8566,6 @@ void TestNewShaderRecompilerU64PairLowering() { Check(Common::ContainsStr(result.decoded_dump, "V_CMPX_NE_I64 exec_lo, 0, v7"), "captured VOP3 V_CMPX_NE_I64 was not present in the decoded dump"); - Decoder::Program decoded_i64_program; - CFG::Graph graph_i64; - IR::Program native_i64; - Check(Decoder::DecodeProgram(cmpx_i64_shader, decoded_i64_program, &error), - error.c_str()); - Check(CFG::BuildGraph(decoded_i64_program, graph_i64, &error) && - CFG::Structurize(graph_i64, &error), - error.c_str()); - Check(IR::LowerProgram(decoded_i64_program, graph_i64, ShaderType::Compute, - 64u, native_i64, &error), - error.c_str()); - Check(Common::ContainsStr(IR::ProgramToString(native_i64), - "CompareNeU64 exec_lo, 0x00000000, v7"), - "V_CMPX_NE_I64 did not lower to masked 64-bit inequality"); CheckSpirvBinaryValidates(result.spirv); const uint32_t cmpx_shader[] = { @@ -8501,20 +8582,6 @@ void TestNewShaderRecompilerU64PairLowering() { decoded_cmpx.src1.reg == 7u, "decoder rejected captured V_CMPX_NE_U64 fields"); Check(TryRecompile(cmpx_shader, options, result, &error), error.c_str()); - Decoder::Program decoded_program; - CFG::Graph graph; - IR::Program native; - Check(Decoder::DecodeProgram(cmpx_shader, decoded_program, &error), - error.c_str()); - Check(CFG::BuildGraph(decoded_program, graph, &error) && - CFG::Structurize(graph, &error), - error.c_str()); - Check(IR::LowerProgram(decoded_program, graph, ShaderType::Compute, 64u, - native, &error), - error.c_str()); - Check(Common::ContainsStr(IR::ProgramToString(native), - "CompareNeU64 exec_lo, 0x00000000, v7"), - "V_CMPX_NE_U64 did not lower to masked 64-bit inequality"); CheckSpirvBinaryValidates(result.spirv); const uint32_t cmpx_v2_shader[] = { @@ -8540,8 +8607,8 @@ void TestNewShaderRecompilerU64PairLowering() { const uint32_t empty_shader[] = {EncodeSopp(0x01)}; Check(TryRecompile(empty_shader, options, result, &error), error.c_str()); - auto program = result.program; - IR::IREmitter ir(program.values->blocks.front()); + auto program = std::move(result.program); + IR::IREmitter ir(program.blocks.front()); const auto lane = ir.Emit(IR::ValueOpcode::LaneId); const auto high = ir.Emit(IR::ValueOpcode::IAdd32, {lane, IR::Value(1u)}); const auto base = @@ -8573,7 +8640,7 @@ void TestNewShaderRecompilerU64PairLowering() { const auto source = DisassembleSpirvBinary(spirv); Check(!Common::ContainsStr(source, "OpCapability Int64") && !Common::ContainsStr(source, "OpTypeInt 64"), - "portable pair-U64 lowering introduced native shader Int64"); + "portable pair-U64 translation introduced native shader Int64"); Check(SpirvInstructionOpcodeCount(spirv, 149u) == 1u, "pair-U64 addition did not use exactly one carry instruction"); Check(SpirvInstructionOpcodeCount(spirv, 154u) == 1u && @@ -8584,7 +8651,7 @@ void TestNewShaderRecompilerU64PairLowering() { direct_metrics.words <= 940u && direct_metrics.instructions <= 199u && direct_metrics.type_vectors == 2u && direct_metrics.phis == 0u && direct_metrics.function_variables == 0u, - "portable pair-U64 lowering exceeded its declaration/code-size ratchet"); + "portable pair-U64 translation exceeded its declaration/code-size ratchet"); const uint32_t dispatcher_shader[] = { EncodeSopp(0x05, 2), EncodeSopp(0x02, 0), EncodeSopp(0x05, 0xfffeu), @@ -8593,13 +8660,13 @@ void TestNewShaderRecompilerU64PairLowering() { Check(TryRecompile(dispatcher_shader, options, result, &error), error.c_str()); Check(result.program.dispatcher_fallback && - result.program.values->blocks.size() >= 3u, + result.program.blocks.size() >= 3u, "U64 spill fixture did not select dispatcher mode"); - program = result.program; - IR::IREmitter definition(program.values->blocks[1]); + program = std::move(result.program); + IR::IREmitter definition(program.blocks[1]); const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU64, {IR::Value(1u), IR::Value(2u)}); - IR::IREmitter use(program.values->blocks[2]); + IR::IREmitter use(program.blocks[2]); use.Emit(IR::ValueOpcode::CompositeExtractU64, {vector, IR::Value(1u)}); Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str()); spirv.clear(); @@ -8801,11 +8868,11 @@ void TestNewShaderRecompilerSetpcJumpTable() { Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), "S_SETPC_B64 jump table did not select dispatcher fallback"); const auto jump = std::find_if( - result.program.values->block_info.begin(), - result.program.values->block_info.end(), [](const auto &block) { + result.program.block_info.begin(), + result.program.block_info.end(), [](const auto &block) { return !block.terminator.indirect_targets.empty(); }); - Check(jump != result.program.values->block_info.end() && + Check(jump != result.program.block_info.end() && jump->terminator.indirect_targets.size() == 2, "S_SETPC_B64 jump table did not retain both targets"); Check(jump->terminator.indirect_selector_values.size() == 2, @@ -8856,18 +8923,25 @@ void TestNewShaderRecompilerPrunesUnreachableSetpcMetadata() { "unreachable S_SETPC retained jump-table metadata or dispatcher mode"); ShaderRecompiler::IR::Program ir; - Check(ShaderRecompiler::IR::LowerProgram(decoded, graph, ShaderType::Compute, - 64u, ir, &error), + ShaderComputeInputInfo compute{}; + ShaderRecompiler::Frontend::TranslateOptions translate_options{}; + translate_options.stage = ShaderType::Compute; + translate_options.wave_size = 64u; + translate_options.compute = &compute; + Check(ShaderRecompiler::Frontend::TranslateProgram( + decoded, graph, translate_options, ir, &error), error.c_str()); - const auto scalar_loads = - std::ranges::count_if(ir.blocks, [](const auto &block) { - return std::ranges::any_of(block.instructions, [](const auto &inst) { - return inst.op == ShaderRecompiler::IR::Opcode::SLoadDword && - inst.pc == 0x18u; - }); - }); - Check(scalar_loads == 1u, - "unreachable S_SETPC metadata suppressed a reachable scalar load"); + size_t scalar_loads = 0; + for (const auto *block : ir.blocks) { + scalar_loads += std::ranges::count_if(*block, [](const auto &inst) { + return inst.GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::LoadAddressU32 && + inst.template Flags().pc == + 0x18u; + }); + } + Check(scalar_loads == 2u, + "unreachable S_SETPC metadata suppressed a reachable scalar load component"); } void TestNewShaderRecompilerSetpcDwordJumpTable() { @@ -8906,19 +8980,19 @@ void TestNewShaderRecompilerSetpcDwordJumpTable() { Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), "subtractive S_SETPC_B64 table did not select dispatcher fallback"); const auto jump = std::find_if( - result.program.values->block_info.begin(), - result.program.values->block_info.end(), [](const auto &block) { + result.program.block_info.begin(), + result.program.block_info.end(), [](const auto &block) { return !block.terminator.indirect_targets.empty(); }); const auto block_pc = [&](uint32_t id) { const auto block = - std::find_if(result.program.values->block_info.begin(), - result.program.values->block_info.end(), + std::find_if(result.program.block_info.begin(), + result.program.block_info.end(), [=](const auto &info) { return info.id == id; }); - return block != result.program.values->block_info.end() ? block->start_pc + return block != result.program.block_info.end() ? block->start_pc : UINT32_MAX; }; - Check(jump != result.program.values->block_info.end() && + Check(jump != result.program.block_info.end() && jump->terminator.indirect_targets.size() == 2 && jump->terminator.indirect_target_pcs == std::vector({0x38u, 0x40u}) && @@ -8929,7 +9003,7 @@ void TestNewShaderRecompilerSetpcDwordJumpTable() { block_pc(jump->terminator.indirect_selector_targets[1]) == 0x40u && block_pc(jump->terminator.indirect_selector_targets[2]) == 0x38u && std::ranges::none_of( - result.program.values->block_info, + result.program.block_info, [](const auto &info) { return info.start_pc == 0x04u; }), "subtractive S_SETPC_B64 table targets or selector mapping changed"); Check(!Common::ContainsStr(result.ir_dump, "SLoadDword"), @@ -9214,18 +9288,22 @@ void TestTypedEntryStateIsMinimal() { using namespace ShaderRecompiler; const auto check = [](uint32_t wave_size) { - IR::Program source; - source.stage = ShaderType::Compute; - source.wave_size = wave_size; - source.user_data_count = 0; - source.blocks.emplace_back(); - source.blocks.front().id = 0; - source.blocks.front().terminator.kind = CFG::TerminatorKind::Return; - - IR::ValueProgram values; + Decoder::Program decoded; + CFG::Graph graph; + CFG::BasicBlock source_block; + source_block.id = 0; + source_block.terminator.kind = CFG::TerminatorKind::Return; + graph.blocks.push_back(std::move(source_block)); + graph.entry_block = 0; + IR::Program values; ShaderComputeInputInfo compute{}; + Frontend::TranslateOptions translate_options{}; + translate_options.stage = ShaderType::Compute; + translate_options.wave_size = wave_size; + translate_options.user_data_count = 0; + translate_options.compute = &compute; std::string error; - Check(Frontend::TranslateProgram(source, values, nullptr, nullptr, &compute, + Check(Frontend::TranslateProgram(decoded, graph, translate_options, values, &error), error.c_str()); @@ -9279,7 +9357,7 @@ void TestTypedEntryStateIsMinimal() { IR::RewriteToSsa(values.blocks); IR::RemoveIdentities(values.blocks); IR::EliminateDeadCode(values.blocks); - Check(IR::ValidateValueProgram(values, true, &error), error.c_str()); + Check(IR::ValidateProgram(values, true, &error), error.c_str()); }; check(32u); @@ -9290,7 +9368,7 @@ void TestFinalSsaRejectsRegisterStatePseudos() { using namespace ShaderRecompiler::IR; const auto check_rejected = [](bool setter) { - ValueProgram program; + Program program; program.block_storage.push_back(std::make_unique()); program.blocks.push_back(program.block_storage.back().get()); program.block_info.emplace_back(); @@ -9301,7 +9379,7 @@ void TestFinalSsaRejectsRegisterStatePseudos() { ir.Emit(ValueOpcode::ReferenceU32, {ir.GetScalarReg(ScalarReg{0})}); } std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, "register-state pseudo"), "final SSA accepted a register-state pseudo operation"); }; @@ -9324,7 +9402,7 @@ void TestValuePhiValidation() { }; const auto validate = [](InvalidPhi invalid, const char *expected_error) { - ValueProgram program; + Program program; const auto add_block = [&](uint32_t id) { program.block_storage.push_back(std::make_unique()); auto *block = program.block_storage.back().get(); @@ -9372,7 +9450,7 @@ void TestValuePhiValidation() { } std::string error; - const bool valid = ValidateValueProgram(program, true, &error); + const bool valid = ValidateProgram(program, true, &error); if (invalid == InvalidPhi::None) { Check(valid, error.c_str()); } else { @@ -9390,7 +9468,7 @@ void TestValuePhiValidation() { validate(InvalidPhi::AfterInstruction, "after a non-Phi"); const auto validate_cfg = [](bool duplicate_id) { - ValueProgram program; + Program program; for (uint32_t id = 0; id < 2; id++) { program.block_storage.push_back(std::make_unique()); auto *block = program.block_storage.back().get(); @@ -9403,7 +9481,7 @@ void TestValuePhiValidation() { program.block_info[0].terminator.true_block = 1; } std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, duplicate_id ? "id is duplicated" : "successor graph disagree"), @@ -9412,7 +9490,7 @@ void TestValuePhiValidation() { validate_cfg(true); validate_cfg(false); { - ValueProgram program; + Program program; for (uint32_t id = 0; id < 2; id++) { program.block_storage.push_back(std::make_unique()); program.blocks.push_back(program.block_storage.back().get()); @@ -9427,12 +9505,12 @@ void TestValuePhiValidation() { program.blocks[0]->AddBranch(program.blocks[1]); program.blocks[1]->AddBranch(program.blocks[0]); std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, "entry block has a predecessor"), "value IR accepted a backedge to its synthetic entry block"); } { - ValueProgram program; + Program program; for (uint32_t id = 0; id < 3; id++) { program.block_storage.push_back(std::make_unique()); program.blocks.push_back(program.block_storage.back().get()); @@ -9451,24 +9529,24 @@ void TestValuePhiValidation() { program.blocks[0]->AddBranch(program.blocks[1]); program.blocks[1]->AddBranch(program.blocks[1]); std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, "selector target is not a CFG successor"), "value IR accepted an indirect selector target outside its target " "set"); } { - ValueProgram program; + Program program; program.block_storage.push_back(std::make_unique()); program.blocks.push_back(program.block_storage.back().get()); program.block_info.push_back({.id = UINT32_MAX}); std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, "reserved exit id"), "value IR accepted the dispatcher exit sentinel as a block id"); } { - ValueProgram program; + Program program; for (uint32_t id = 0; id < 2; id++) { program.block_storage.push_back(std::make_unique()); program.blocks.push_back(program.block_storage.back().get()); @@ -9485,24 +9563,24 @@ void TestValuePhiValidation() { program.blocks[0]->AddBranch(program.blocks[1]); program.blocks[1]->AddBranch(program.blocks[1]); std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, "target is duplicated"), "value IR accepted duplicate indirect branch targets"); } { - ValueProgram program; + Program program; for (uint32_t id = 0; id < 2; id++) { program.block_storage.push_back(std::make_unique()); program.blocks.push_back(program.block_storage.back().get()); program.block_info.push_back({.id = id}); } std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, "unreachable block"), "value IR accepted an unreachable block"); } { - ValueProgram program; + Program program; program.block_storage.push_back(std::make_unique()); auto *block = program.block_storage.back().get(); program.blocks.push_back(block); @@ -9513,12 +9591,12 @@ void TestValuePhiValidation() { block->AppendNewInst(ValueOpcode::IAdd32, {Value(3u), Value(4u)}); use.SetArg(0, Value(&definition)); std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, "same-block definition"), "value IR accepted a same-block forward use"); } { - ValueProgram program; + Program program; for (uint32_t id = 0; id < 4; id++) { program.block_storage.push_back(std::make_unique()); program.blocks.push_back(program.block_storage.back().get()); @@ -9543,12 +9621,12 @@ void TestValuePhiValidation() { program.blocks[2]->AppendNewInst(ValueOpcode::IAdd32, {Value(&definition), Value(3u)}); std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, "does not dominate its use"), "value IR accepted a non-dominating ordinary definition"); } { - ValueProgram program; + Program program; for (uint32_t id = 0; id < 4; id++) { program.block_storage.push_back(std::make_unique()); program.blocks.push_back(program.block_storage.back().get()); @@ -9575,12 +9653,12 @@ void TestValuePhiValidation() { phi.AddPhiOperand(program.blocks[1], Value(3u)); phi.AddPhiOperand(program.blocks[2], Value(&definition)); std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, "does not dominate its edge"), "value IR accepted an unavailable Phi-edge definition"); } { - ValueProgram program; + Program program; for (uint32_t id = 0; id < 5; id++) { program.block_storage.push_back(std::make_unique()); program.blocks.push_back(program.block_storage.back().get()); @@ -9611,7 +9689,7 @@ void TestValuePhiValidation() { {Value(1u), Value(2u)}); program.block_info[2].condition = Value(&condition); std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, "branch condition definition"), "value IR accepted a non-dominating branch condition"); } @@ -9622,7 +9700,7 @@ void TestWqmMaskSignatureAndU64ShiftConstantPropagation() { const auto check_invalid_signature = [](ValueOpcode opcode, Value valid, Value invalid) { - ValueProgram malformed; + Program malformed; malformed.block_storage.push_back(std::make_unique()); malformed.blocks.push_back(malformed.block_storage.back().get()); malformed.block_info.emplace_back(); @@ -9630,14 +9708,14 @@ void TestWqmMaskSignatureAndU64ShiftConstantPropagation() { auto value = malformed_ir.Emit(opcode, {valid}); value.TryInstruction()->SetArg(0, invalid); std::string error; - Check(!ValidateValueProgram(malformed, false, &error) && + Check(!ValidateProgram(malformed, false, &error) && Common::ContainsStr(error, "argument 0 has type"), "value IR accepted a WQM operand that violates opcode metadata"); }; check_invalid_signature(ValueOpcode::WqmMask, Value(true), Value(uint64_t{1})); - ValueProgram shifts; + Program shifts; shifts.block_storage.push_back(std::make_unique()); shifts.blocks.push_back(shifts.block_storage.back().get()); shifts.block_info.emplace_back(); @@ -9676,7 +9754,7 @@ void TestNativeWideValueValidation() { using namespace ShaderRecompiler::IR; const auto make_program = [] { - ValueProgram program; + Program program; program.block_storage.push_back(std::make_unique()); program.blocks.push_back(program.block_storage.back().get()); program.block_info.emplace_back(); @@ -9685,7 +9763,7 @@ void TestNativeWideValueValidation() { ShaderRecompiler::CFG::TerminatorKind::Return; return program; }; - const auto append_load = [](ValueProgram &program, ValueOpcode opcode, + const auto append_load = [](Program &program, ValueOpcode opcode, MemoryFlags flags) { IREmitter ir(program.blocks.front()); const auto resource = ir.Emit(ValueOpcode::GetBufferResource, @@ -9693,7 +9771,7 @@ void TestNativeWideValueValidation() { ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(0u), Value(true)}, flags); }; - const auto append_scalar_read = [](ValueProgram &program, ValueOpcode opcode, + const auto append_scalar_read = [](Program &program, ValueOpcode opcode, MemoryFlags flags) { IREmitter ir(program.blocks.front()); if (opcode == ValueOpcode::ReadConstBuffer) { @@ -9707,10 +9785,10 @@ void TestNativeWideValueValidation() { ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(true)}, flags); } }; - const auto check_rejected = [](const ValueProgram &program, + const auto check_rejected = [](const Program &program, const char *expected) { std::string error; - Check(!ValidateValueProgram(program, true, &error) && + Check(!ValidateProgram(program, true, &error) && Common::ContainsStr(error, expected), "malformed native-wide value operation was not rejected"); }; @@ -9829,7 +9907,7 @@ void TestNativeWideValueValidation() { ir.Emit(ValueOpcode::CompositeExtractU64, {vector, dynamic_index}); check_rejected(program, "invalid component index"); } - const auto append_shared = [](ValueProgram &program, ValueOpcode opcode, + const auto append_shared = [](Program &program, ValueOpcode opcode, MemoryFlags flags) { IREmitter ir(program.blocks.front()); ir.Emit(opcode, {Value(0u), Value(true)}, flags); @@ -9866,7 +9944,7 @@ void TestNativeWideValueValidation() { append_shared(program, ValueOpcode::LoadSharedU32x3, {}); check_rejected(program, "inconsistent shared-memory width"); } - const auto append_address = [](ValueProgram &program, ValueOpcode opcode, + const auto append_address = [](Program &program, ValueOpcode opcode, MemoryFlags flags) { IREmitter ir(program.blocks.front()); const auto resource = @@ -9900,7 +9978,7 @@ void TestNativeWideValueValidation() { append_address(program, ValueOpcode::LoadAddressU8, {}); check_rejected(program, "inconsistent address-memory metadata"); } - const auto append_image = [](ValueProgram &program, ValueOpcode opcode, + const auto append_image = [](Program &program, ValueOpcode opcode, MemoryFlags flags) { IREmitter ir(program.blocks.front()); const auto image = ir.Emit(ValueOpcode::GetImageResource, @@ -10634,7 +10712,12 @@ void TestNewShaderRecompilerNativeBindingPlan() { "SPIR-V resources do not all use descriptor set zero"); CheckSpirvBinaryValidates(result.spirv); - auto malformed = result.program; + ShaderRecompiler::CompileResult malformed_result; + std::string malformed_compile_error; + Check(ShaderRecompiler::TryRecompile(shader, options, malformed_result, + &malformed_compile_error), + malformed_compile_error.c_str()); + auto malformed = std::move(malformed_result.program); auto malformed_buffers = std::find_if( malformed.bindings.descriptors.begin(), malformed.bindings.descriptors.end(), @@ -10646,7 +10729,8 @@ void TestNewShaderRecompilerNativeBindingPlan() { const auto rejected_before = rejected_spirv; std::string rejected_error; Check(!ShaderRecompiler::Spirv::EmitProgram( - malformed, result.resources, options.input_info, rejected_spirv, + malformed, malformed_result.resources, options.input_info, + rejected_spirv, &rejected_error) && rejected_spirv == rejected_before && rejected_error.find("topology") != std::string::npos, @@ -10685,7 +10769,7 @@ void TestNewShaderRecompilerNativeBindingPlan() { "unsupported runtime image type bypassed specialization validation"); ShaderRecompiler::IR::Inst *buffer_handle = nullptr; - for (auto *block : result.program.values->blocks) { + for (auto *block : result.program.blocks) { const auto found = std::ranges::find_if(*block, [](const auto &inst) { return inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::GetBufferResource; @@ -10710,54 +10794,49 @@ void TestNewShaderRecompilerNativeBindingPlan() { "invalid typed buffer handle was not rejected transactionally"); } -bool LowerTypedPlanning(const uint32_t *code, uint32_t words, - ShaderRecompiler::IR::Program &ir, std::string *error) { +bool BuildTypedPlan(const uint32_t *code, uint32_t words, + ShaderRecompiler::IR::Program &ir, std::string *error) { ShaderRecompiler::Decoder::Program decoded; if (!ShaderRecompiler::Decoder::DecodeProgram(std::span{code, words}, decoded, error)) { return false; } ShaderRecompiler::CFG::Graph cfg; - if (!ShaderRecompiler::CFG::BuildGraph(decoded, cfg, error) || - !ShaderRecompiler::IR::LowerProgram(decoded, cfg, ShaderType::Compute, 64, - ir, error)) { - return false; - } - ShaderVertexInputInfo vertex; - ShaderPixelInputInfo pixel; ShaderComputeInputInfo compute; - ir.values = std::make_shared(); - if (!ShaderRecompiler::Frontend::TranslateProgram(ir, *ir.values, &vertex, - &pixel, &compute, error)) { + ShaderRecompiler::Frontend::TranslateOptions options{}; + options.stage = ShaderType::Compute; + options.wave_size = 64u; + options.compute = &compute; + if (!ShaderRecompiler::CFG::BuildGraph(decoded, cfg, error) || + !ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, + error)) { return false; } - ir.blocks.clear(); - ShaderRecompiler::IR::RewriteToSsa(ir.values->blocks); - ShaderRecompiler::IR::ConstantPropagationPass(ir.values->blocks); - ShaderRecompiler::IR::ResolveControlFlowIdentities(*ir.values); - ShaderRecompiler::IR::RemoveIdentities(ir.values->blocks); - ShaderRecompiler::IR::EliminateDeadCode(ir.values->blocks); + ShaderRecompiler::IR::RewriteToSsa(ir.blocks); + ShaderRecompiler::IR::ConstantPropagationPass(ir.blocks); + ShaderRecompiler::IR::ResolveControlFlowIdentities(ir); + ShaderRecompiler::IR::RemoveIdentities(ir.blocks); + ShaderRecompiler::IR::EliminateDeadCode(ir.blocks); const auto read_lane = - ShaderRecompiler::IR::EliminateReadLane(*ir.values, ir.wave_size); + ShaderRecompiler::IR::EliminateReadLane(ir, ir.wave_size); if (read_lane.rewritten_reads != 0) { - ShaderRecompiler::IR::ConstantPropagationPass(ir.values->blocks); - ShaderRecompiler::IR::ResolveControlFlowIdentities(*ir.values); - ShaderRecompiler::IR::RemoveIdentities(ir.values->blocks); - ShaderRecompiler::IR::EliminateDeadCode(ir.values->blocks); + ShaderRecompiler::IR::ConstantPropagationPass(ir.blocks); + ShaderRecompiler::IR::ResolveControlFlowIdentities(ir); + ShaderRecompiler::IR::RemoveIdentities(ir.blocks); + ShaderRecompiler::IR::EliminateDeadCode(ir.blocks); } if (!ShaderRecompiler::IR::BuildSrtPlan(ir, error)) { return false; } - ShaderRecompiler::IR::EliminateDeadCode(ir.values->blocks); + ShaderRecompiler::IR::EliminateDeadCode(ir.blocks); return true; } const ShaderRecompiler::IR::DescriptorSource * TypedDescriptorSource(const ShaderRecompiler::IR::Program &program, uint32_t source) { - return program.values != nullptr && - source < program.values->descriptor_sources.size() - ? &program.values->descriptor_sources[source] + return source < program.descriptor_sources.size() + ? &program.descriptor_sources[source] : nullptr; } @@ -10793,7 +10872,7 @@ bool ValueDependsOn(ShaderRecompiler::IR::Value value, void CheckFlattenedReadSlots(const ShaderRecompiler::IR::Program &program, uint32_t expected, const char *message) { std::vector slots; - for (const auto *block : program.values->blocks) { + for (const auto *block : program.blocks) { for (const auto &inst : *block) { if (inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::ReadConst) { const auto slot = inst.Arg(1).Resolve(); @@ -10839,7 +10918,7 @@ bool ReadSrtHostRangeDword(void *userdata, uint64_t address, uint32_t *value) { return true; } -void TestTypedDescriptorRealWideMoveLowering() { +void TestTypedDescriptorRealWideMoveTranslation() { const uint32_t shader[] = { EncodeSop1(0x04, 0, 20), // s_mov_b64 s[0:1], s[20:21] EncodeSop1(0x04, 2, 22), // s_mov_b64 s[2:3], s[22:23] @@ -10850,9 +10929,9 @@ void TestTypedDescriptorRealWideMoveLowering() { std::string error; ShaderRecompiler::IR::Program ir; - const auto lowered = LowerTypedPlanning( + const auto translated = BuildTypedPlan( shader, static_cast(std::size(shader)), ir, &error); - Check(lowered, error.c_str()); + Check(translated, error.c_str()); Check(ShaderRecompiler::IR::TrackResources(ir, &error), error.c_str()); Check(ir.info.buffers.size() == 1, "real wide-move shader did not track one buffer use"); @@ -10866,7 +10945,7 @@ void TestTypedDescriptorRealWideMoveLowering() { ShaderRecompiler::IR::ValueOpcode::GetUserData && ShaderRecompiler::IR::RegIndex(value->Arg(0).ScalarRegister()) == 20 + i, - "real s_mov_b64 lowering did not copy both descriptor pairs"); + "real s_mov_b64 translation did not copy both descriptor pairs"); } } @@ -10883,12 +10962,12 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { }; std::string error; ShaderRecompiler::IR::Program carry_ir; - const auto carry_lowered = LowerTypedPlanning( + const auto carry_translated = BuildTypedPlan( carry_shader, static_cast(std::size(carry_shader)), carry_ir, &error); - Check(carry_lowered, error.c_str()); + Check(carry_translated, error.c_str()); const ShaderRecompiler::IR::Inst *carry_handle = nullptr; - for (const auto *block : carry_ir.values->blocks) { + for (const auto *block : carry_ir.blocks) { for (const auto &inst : *block) { if (inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::StoreBufferU32) { @@ -10907,8 +10986,8 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { carry_descriptor.dwords[index] = carry_handle->Arg(index).Resolve(); } const auto carry_source_index = - static_cast(carry_ir.values->descriptor_sources.size()); - carry_ir.values->descriptor_sources.push_back(carry_descriptor); + static_cast(carry_ir.descriptor_sources.size()); + carry_ir.descriptor_sources.push_back(carry_descriptor); const auto *carry_source = TypedDescriptorSource(carry_ir, carry_source_index); Check(carry_source != nullptr && carry_source->dword_count == 4 && @@ -10918,7 +10997,7 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { ShaderRecompiler::IR::ValueOpcode::GetShaderBase) && ValueDependsOn(carry_source->dwords[1], ShaderRecompiler::IR::ValueOpcode::IAddCarry32), - "real s_add_u32/s_addc_u32 lowering lost SCC carry provenance"); + "real s_add_u32/s_addc_u32 translation lost SCC carry provenance"); std::array carry_user_data{}; const uint64_t shader_base = 0x00000012fffffffbull; const uint64_t expected_pc = shader_base + 5u; @@ -10944,16 +11023,16 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { EncodeSopp(0x01), }; ShaderRecompiler::IR::Program load_ir; - const auto load_lowered = LowerTypedPlanning( + const auto load_translated = BuildTypedPlan( load_shader, static_cast(std::size(load_shader)), load_ir, &error); - Check(load_lowered, error.c_str()); - Check(load_ir.values->srt_reads.size() == 8 && - load_ir.values->dynamic_reads.empty(), + Check(load_translated, error.c_str()); + Check(load_ir.srt_reads.size() == 8 && + load_ir.dynamic_reads.empty(), "real scalar loads did not build eight flattened reads"); uint32_t address_reads = 0; uint32_t buffer_reads = 0; - for (const auto &read : load_ir.values->srt_reads) { + for (const auto &read : load_ir.srt_reads) { const auto *raw = read.value.ResolveInstruction(); Check(raw != nullptr, "real scalar-load SRT entry lost its raw typed value"); @@ -10993,7 +11072,7 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { }; ShaderRecompiler::IR::Program inline_sampler_ir; error.clear(); - Check(LowerTypedPlanning( + Check(BuildTypedPlan( inline_sampler_shader, static_cast(std::size(inline_sampler_shader)), inline_sampler_ir, &error) && @@ -11023,7 +11102,7 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { error.c_str()); } -void TestSrtWalkerRealSmemLowering() { +void TestSrtWalkerRealSmemTranslation() { const uint32_t shader[] = { EncodeSMovB32(124, 130), // m0 = 2 EncodeSmem0(0x02, 0, 4), @@ -11035,11 +11114,11 @@ void TestSrtWalkerRealSmemLowering() { }; std::string error; ShaderRecompiler::IR::Program ir; - const auto lowered = LowerTypedPlanning( + const auto translated = BuildTypedPlan( shader, static_cast(std::size(shader)), ir, &error); - Check(lowered, error.c_str()); - Check(ir.values->srt_reads.size() == 4 && ir.values->dynamic_reads.empty(), - "real SMEM lowering did not build four compact SRT reads"); + Check(translated, error.c_str()); + Check(ir.srt_reads.size() == 4 && ir.dynamic_reads.empty(), + "real SMEM translation did not build four compact SRT reads"); const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; @@ -11058,7 +11137,7 @@ void TestSrtWalkerRealSmemLowering() { CheckFlattenedReadSlots(ir, 4, "real SMEM patch used the wrong flat offsets"); } -void TestSrtWalkerVccBaseLowering() { +void TestSrtWalkerVccBaseTranslation() { const uint32_t shader[] = { EncodeSMovB32(106, 27), EncodeSMovB32(107, 28), EncodeSmem0(0x02, 0, 53), 125u << 25u, @@ -11067,11 +11146,11 @@ void TestSrtWalkerVccBaseLowering() { }; std::string error; ShaderRecompiler::IR::Program ir; - Check(LowerTypedPlanning(shader, static_cast(std::size(shader)), ir, - &error), + Check(BuildTypedPlan(shader, static_cast(std::size(shader)), ir, + &error), error.c_str()); - Check(ir.values->srt_reads.size() == 4, - "VCC-based SMEM lowering did not build four SRT reads"); + Check(ir.srt_reads.size() == 4, + "VCC-based SMEM translation did not build four SRT reads"); const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; @@ -11089,7 +11168,7 @@ void TestSrtWalkerVccBaseLowering() { "typed SSA lost an SMEM base copied through VCC"); } -void TestSrtWalkerRealSBufferLowering() { +void TestSrtWalkerRealSBufferTranslation() { const uint32_t shader[] = { EncodeSMovB32(124, 130), // m0 = 2 EncodeSmem0(0x0a, 0, 4), @@ -11098,11 +11177,11 @@ void TestSrtWalkerRealSBufferLowering() { }; std::string error; ShaderRecompiler::IR::Program ir; - const auto lowered = LowerTypedPlanning( + const auto translated = BuildTypedPlan( shader, static_cast(std::size(shader)), ir, &error); - Check(lowered, error.c_str()); - Check(ir.values->srt_reads.size() == 4 && ir.values->dynamic_reads.empty(), - "real S_BUFFER_LOAD lowering did not build four compact reads"); + Check(translated, error.c_str()); + Check(ir.srt_reads.size() == 4 && ir.dynamic_reads.empty(), + "real S_BUFFER_LOAD translation did not build four compact reads"); const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u}; @@ -11137,10 +11216,10 @@ void TestSrtWalkerRealSBufferLowering() { EncodeMubuf0(0x1c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01), }; ShaderRecompiler::IR::Program negative_ir; - const auto negative_lowered = LowerTypedPlanning( + const auto negative_translated = BuildTypedPlan( negative_shader, static_cast(std::size(negative_shader)), negative_ir, &error); - Check(negative_lowered, error.c_str()); + Check(negative_translated, error.c_str()); user_data[10] = sizeof(table); Check(!ShaderRecompiler::IR::WalkSrt(negative_ir, runtime, flat, &error) && error.find("negative immediate") != std::string::npos, @@ -11163,10 +11242,10 @@ void TestScalarMemorySourcesCapturedBeforeWrites() { EncodeSopp(0x01)}); std::string error; ShaderRecompiler::IR::Program ir; - Check(LowerTypedPlanning(shader.data(), + Check(BuildTypedPlan(shader.data(), static_cast(shader.size()), ir, &error), error.c_str()); - Check(ir.values->srt_reads.size() == 4 && ir.values->dynamic_reads.empty(), + Check(ir.srt_reads.size() == 4 && ir.dynamic_reads.empty(), opcode == 0x02 ? "overlapping S_LOAD operands were evaluated after a " "component write" : "overlapping S_BUFFER_LOAD operands were evaluated " @@ -11212,10 +11291,10 @@ void TestScalarMemoryLoadCrossesIntoVcc() { }; std::string error; ShaderRecompiler::IR::Program ir; - Check(LowerTypedPlanning(shader, static_cast(std::size(shader)), ir, - &error), + Check(BuildTypedPlan(shader, static_cast(std::size(shader)), ir, + &error), error.c_str()); - Check(ir.values->srt_reads.size() == 4 && ir.values->dynamic_reads.empty(), + Check(ir.srt_reads.size() == 4 && ir.dynamic_reads.empty(), "wide SMEM destination crossing into VCC lost scalar provenance"); CheckFlattenedReadSlots( ir, 4, "wide SMEM destination crossing into VCC used wrong flat offsets"); @@ -11245,11 +11324,11 @@ void TestScalarMemoryUnusedTailDce() { }; ShaderRecompiler::IR::Program ir; std::string error; - Check(LowerTypedPlanning(shader, static_cast(std::size(shader)), ir, - &error), + Check(BuildTypedPlan(shader, static_cast(std::size(shader)), ir, + &error), error.c_str()); uint32_t reads = 0; - for (const auto *block : ir.values->blocks) { + for (const auto *block : ir.blocks) { for (const auto &inst : *block) { if (inst.GetOpcode() != ShaderRecompiler::IR::ValueOpcode::ReadConstBuffer) { @@ -11257,9 +11336,9 @@ void TestScalarMemoryUnusedTailDce() { } reads++; const auto flags = inst.Flags(); - Check(flags.index < ir.values->memory_info.size() && - ir.values->memory_info[flags.index].component_index == 0u && - ir.values->memory_info[flags.index].component_count == 16u, + Check(flags.index < ir.memory_info.size() && + ir.memory_info[flags.index].component_index == 0u && + ir.memory_info[flags.index].component_count == 16u, "live x16 SMEM component lost its native-width metadata"); } } @@ -11284,9 +11363,9 @@ void TestResourceTrackingRealDensePatching() { }; std::string error; ShaderRecompiler::IR::Program ir; - const auto lowered = LowerTypedPlanning( + const auto translated = BuildTypedPlan( shader, static_cast(std::size(shader)), ir, &error); - Check(lowered, error.c_str()); + Check(translated, error.c_str()); const auto tracked = ShaderRecompiler::IR::TrackResources(ir, &error); Check(tracked, error.c_str()); Check(ir.info.buffers.size() == 2 && ir.info.images.size() == 2 && @@ -11295,7 +11374,7 @@ void TestResourceTrackingRealDensePatching() { uint32_t buffer_use = 0; uint32_t image_use = 0; - for (const auto *block : ir.values->blocks) { + for (const auto *block : ir.blocks) { for (const auto &inst : *block) { const auto op = inst.GetOpcode(); switch (op) { @@ -11312,7 +11391,7 @@ void TestResourceTrackingRealDensePatching() { case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32x3: case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32x4: { const auto &memory = - ir.values->memory_info + ir.memory_info [inst.Flags().index]; const uint32_t expected = buffer_use++ < 2 ? 0 : 1; const auto *handle = inst.Arg(0).ResolveInstruction(); @@ -11324,7 +11403,7 @@ void TestResourceTrackingRealDensePatching() { case ShaderRecompiler::IR::ValueOpcode::ImageSampleRaw: case ShaderRecompiler::IR::ValueOpcode::ImageWrite: { const auto &memory = - ir.values->memory_info + ir.memory_info [inst.Flags().index]; const auto expected = image_use++; const auto *handle = inst.Arg(0).ResolveInstruction(); @@ -11349,12 +11428,12 @@ void TestResourceTrackingRealDensePatching() { "real tracked resource access facts were incomplete"); } -void TestLowerProgramResetsAnalysisState() { +void TestDirectTranslationResetsAnalysisState() { const uint32_t first_shader[] = {EncodeMubuf0(0x1c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01)}; std::string error; ShaderRecompiler::IR::Program ir; - Check(LowerTypedPlanning(first_shader, + Check(BuildTypedPlan(first_shader, static_cast(std::size(first_shader)), ir, &error), error.c_str()); @@ -11369,17 +11448,17 @@ void TestLowerProgramResetsAnalysisState() { ir.shader_hash = 0xdeadbeef; const uint32_t second_shader[] = {EncodeSopp(0x01)}; - Check(LowerTypedPlanning(second_shader, + Check(BuildTypedPlan(second_shader, static_cast(std::size(second_shader)), ir, &error), error.c_str()); Check(!ir.resource_tracking_complete && !ir.shader_info_complete && - ir.srt_plan_complete && ir.values != nullptr && - ir.values->srt_reads.empty() && ir.shader_hash == 0 && + ir.srt_plan_complete && + ir.srt_reads.empty() && ir.shader_hash == 0 && ir.info.buffers.empty() && ir.info.images.empty() && ir.info.samplers.empty() && ir.info.sampled_pairs.empty() && ir.info.inputs.empty() && ir.info.outputs.empty(), - "LowerProgram reused stale provenance/resource/interface state"); + "direct translation reused stale provenance/resource/interface state"); } void TestNewShaderRecompilerStageInputInfo() { @@ -11693,7 +11772,7 @@ void TestSharedMemoryBarrierSafety() { using namespace ShaderRecompiler::IR; const auto make_program = [](size_t block_count) { - ValueProgram program; + Program program; program.memory_info.push_back({.kind = ResourceKind::Lds}); for (size_t index = 0; index < block_count; index++) { program.block_storage.push_back(std::make_unique()); @@ -11719,7 +11798,7 @@ void TestSharedMemoryBarrierSafety() { return inst.GetOpcode() == ValueOpcode::Barrier; }); }; - const auto divergent_condition = [](ValueProgram& program, size_t block) { + const auto divergent_condition = [](Program& program, size_t block) { IREmitter ir(program.blocks[block]); const auto local_id = U32(ir.Emit( ValueOpcode::GetBuiltin, @@ -11962,7 +12041,7 @@ void TestNewShaderRecompilerFlatUserPointerProvenance() { Check(result.program.info.addresses.size() == 1 && !result.program.info.addresses[0].unbased && result.program.info.addresses[0].source < - result.program.values->descriptor_sources.size(), + result.program.descriptor_sources.size(), "FLAT user pointer provenance was not attached"); Check(result.resources.addresses.size() == 1 && result.resources.addresses[0].guest_base == 0x0000001234567000ull && @@ -12113,7 +12192,7 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { uint32_t wide_stores = 0; uint32_t scalar_loads = 0; uint32_t scalar_stores = 0; - for (const auto *block : wide_result.program.values->blocks) { + for (const auto *block : wide_result.program.blocks) { for (const auto &inst : *block) { switch (inst.GetOpcode()) { case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32x4: @@ -12192,7 +12271,7 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { const auto count_shared = [](const ShaderRecompiler::CompileResult &result, ShaderRecompiler::IR::ValueOpcode opcode) { uint32_t count = 0; - for (const auto *block : result.program.values->blocks) { + for (const auto *block : result.program.blocks) { count += std::ranges::count(*block, opcode, &ShaderRecompiler::IR::Inst::GetOpcode); } @@ -12283,16 +12362,16 @@ int main() { TestNewShaderRecompilerAuxPositionExports(); TestNewShaderRecompilerNativeWideScalarMemoryIr(); TestNewShaderRecompilerNativeWideBufferIr(); - TestNewShaderRecompilerScalarB64LaneLowering(); - TestNewShaderRecompilerMubufFormatLowering(); - TestNewShaderRecompilerTypedBufferLowering(); - TestNewShaderRecompilerDsReadWrite2Lowering(); - TestNewShaderRecompilerDsWideAndAtomicLowering(); + TestNewShaderRecompilerScalarB64LaneTranslation(); + TestNewShaderRecompilerMubufFormatTranslation(); + TestNewShaderRecompilerTypedBufferTranslation(); + TestNewShaderRecompilerDsReadWrite2Translation(); + TestNewShaderRecompilerDsWideAndAtomicTranslation(); TestNewShaderRecompilerCapturedVop1SdwaByteConvert(); TestNewShaderRecompilerScalarMemoryBindingDomains(); - // Opcode semantics and optimized direct SPIR-V are exercised by - // ShaderRecompilerComputeTests. The pre-SSA register-IR shape checks above - // remain as historical decoder fixtures only. + // Opcode semantics and optimized SPIR-V are exercised by + // ShaderRecompilerComputeTests; keep the distinct decoder contract checks + // here. TestNewShaderDecoderArchitecture(); TestNewShaderRecompilerRejectsDppOn64BitCompares(); TestNewShaderRecompilerIrLookupMissFailsExplicitly(); @@ -12336,7 +12415,7 @@ int main() { TestNewShaderRecompilerCfgPrunesUnreachableSelectionEntry(); TestNewShaderRecompilerCfgIrreducibleDispatcher(); TestNewShaderRecompilerDispatcherSpillsU32x3(); - TestNewShaderRecompilerU64PairLowering(); + TestNewShaderRecompilerU64PairTranslation(); TestComputeDispatchWaveSize(); TestNewShaderRecompilerBufferLoadsGuardedByExec(); TestNewShaderRecompilerBufferAtomicsGuardedByBounds(); @@ -12359,19 +12438,19 @@ int main() { TestNewShaderRecompilerExpPixelOutputs(); TestRenderTargetReverseExportMapping(); TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled(); - TestTypedDescriptorRealWideMoveLowering(); + TestTypedDescriptorRealWideMoveTranslation(); TestTypedDescriptorRealCarryAndScalarLoads(); - TestSrtWalkerRealSmemLowering(); - TestSrtWalkerVccBaseLowering(); - TestSrtWalkerRealSBufferLowering(); + TestSrtWalkerRealSmemTranslation(); + TestSrtWalkerVccBaseTranslation(); + TestSrtWalkerRealSBufferTranslation(); TestScalarMemorySourcesCapturedBeforeWrites(); TestScalarMemoryLoadCrossesIntoVcc(); TestScalarMemoryUnusedTailDce(); TestResourceTrackingRealDensePatching(); - TestLowerProgramResetsAnalysisState(); + TestDirectTranslationResetsAnalysisState(); TestNewShaderRecompilerNativeBindingPlan(); TestNewShaderRecompilerStageInputInfo(); - TestCustomVintrpMovLowering(); + TestCustomVintrpMovTranslation(); TestGraphicsCreateInterpolantMapping(); TestNewShaderRecompilerPixelPipelineEntry(); TestComputeLdsAllocationIdentity();