diff --git a/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp b/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp index 19316eb7a..b02980596 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp @@ -57,46 +57,50 @@ bool Translator::TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32 return true; } -bool Translator::TranslateAttributeOperation(const Decoder::Instruction& inst, std::string* error) { +void Translator::V_INTERP_P1_F32() {} + +void Translator::V_INTERP_P2_F32(const Decoder::Instruction& inst) { + const auto value = ir.Emit(IR::ValueOpcode::GetAttribute, + {IR::Value(inst.src1.value), IR::Value(inst.src2.value)}); + WriteOperand(inst.dst, value); +} + +bool Translator::V_INTERP_MOV_F32(const Decoder::Instruction& inst, std::string* error) { + if (inst.src0.value >= 3u) { + return ShaderError::Fail(error, + fmt::format("v_interp_mov_f32 mode {} is reserved at pc 0x{:08x}", + inst.src0.value, inst.pc)); + } + const auto value = ir.Emit( + IR::ValueOpcode::GetInterpolationParameter, + {IR::Value(inst.src1.value), IR::Value(inst.src2.value), IR::Value(inst.src0.value)}); + WriteOperand(inst.dst, value); + return true; +} + +bool Translator::EXP(const Decoder::Instruction& inst, std::string* error) { + uint32_t index = 0; + if (ExportTargetKindFromTarget(inst.exp.target, index) == IR::ExportTargetKind::Unknown) { + return ShaderError::Fail(error, + fmt::format("unsupported EXP target 0x{:02x} at pc 0x{:08x}", + inst.exp.target, inst.pc)); + } + std::array components {IR::Value(0u), IR::Value(0u), IR::Value(0u), + IR::Value(0u)}; + for (uint32_t source = 0; source < std::min(inst.src_count, 4u); source++) { + components[source] = ReadRawU32(PlainOperand(SourceAt(inst, source))); + } + const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4, + {components[0], components[1], components[2], components[3]}); + ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst)); + return true; +} + +bool Translator::EmitInterpolation(const Decoder::Instruction& inst, std::string* error) { switch (inst.opcode) { - case Decoder::Opcode::V_INTERP_P1_F32: return true; - case Decoder::Opcode::V_INTERP_P2_F32: - case Decoder::Opcode::V_INTERP_MOV_F32: { - if (inst.opcode == Decoder::Opcode::V_INTERP_MOV_F32 && inst.src0.value >= 3u) { - return ShaderError::Fail( - error, fmt::format("v_interp_mov_f32 mode {} is reserved at pc 0x{:08x}", - inst.src0.value, inst.pc)); - } - const auto interpolation_mode = - inst.opcode == Decoder::Opcode::V_INTERP_MOV_F32 ? inst.src0.value : 3u; - const auto value = - interpolation_mode < 3u - ? ir.Emit(IR::ValueOpcode::GetInterpolationParameter, - {IR::Value(inst.src1.value), IR::Value(inst.src2.value), - IR::Value(interpolation_mode)}) - : ir.Emit(IR::ValueOpcode::GetAttribute, - {IR::Value(inst.src1.value), IR::Value(inst.src2.value)}); - WriteOperand(inst.dst, value); - return true; - } - case Decoder::Opcode::EXP: { - uint32_t index = 0; - if (ExportTargetKindFromTarget(inst.exp.target, index) == - IR::ExportTargetKind::Unknown) { - return ShaderError::Fail( - error, fmt::format("unsupported EXP target 0x{:02x} at pc 0x{:08x}", - inst.exp.target, inst.pc)); - } - std::array components {IR::Value(0u), IR::Value(0u), IR::Value(0u), - IR::Value(0u)}; - for (uint32_t source = 0; source < std::min(inst.src_count, 4u); source++) { - components[source] = ReadRawU32(PlainOperand(SourceAt(inst, source))); - } - const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4, - {components[0], components[1], components[2], components[3]}); - ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst)); - return true; - } + case Decoder::Opcode::V_INTERP_P1_F32: V_INTERP_P1_F32(); return true; + case Decoder::Opcode::V_INTERP_P2_F32: V_INTERP_P2_F32(inst); return true; + case Decoder::Opcode::V_INTERP_MOV_F32: return V_INTERP_MOV_F32(inst, error); default: return false; } } diff --git a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp index 9827856a9..8d6f8707d 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp @@ -2,221 +2,64 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend { -bool Translator::TranslateIntegerCompare(const Decoder::Instruction& inst) { - IR::ValueOpcode opcode {}; - IR::Type type = IR::Type::U32; - switch (inst.opcode) { - case Decoder::Opcode::V_CMP_F_I32: - case Decoder::Opcode::V_CMP_F_U32: - WriteCompareResult(inst.dst, IR::U1(IR::Value(false))); - return true; - case Decoder::Opcode::V_CMP_T_I32: - case Decoder::Opcode::V_CMP_T_U32: - WriteCompareResult(inst.dst, IR::U1(IR::Value(true))); - return true; - case Decoder::Opcode::S_CMP_EQ_U32: - case Decoder::Opcode::V_CMP_EQ_U32: - case Decoder::Opcode::V_CMPX_EQ_U32: - case Decoder::Opcode::S_CMP_EQ_I32: - case Decoder::Opcode::V_CMP_EQ_I32: - case Decoder::Opcode::V_CMPX_EQ_I32: opcode = IR::ValueOpcode::IEqual32; break; - case Decoder::Opcode::S_CMP_LG_U32: - case Decoder::Opcode::V_CMP_NE_U32: - case Decoder::Opcode::V_CMPX_NE_U32: - case Decoder::Opcode::S_CMP_LG_I32: - case Decoder::Opcode::V_CMP_NE_I32: - case Decoder::Opcode::V_CMPX_NE_I32: opcode = IR::ValueOpcode::INotEqual32; break; - case Decoder::Opcode::S_CMP_GT_U32: - case Decoder::Opcode::V_CMP_GT_U32: - case Decoder::Opcode::V_CMPX_GT_U32: opcode = IR::ValueOpcode::UGreaterThan32; break; - case Decoder::Opcode::S_CMP_GE_U32: - case Decoder::Opcode::V_CMP_GE_U32: - case Decoder::Opcode::V_CMPX_GE_U32: opcode = IR::ValueOpcode::UGreaterThanEqual32; break; - case Decoder::Opcode::S_CMP_LT_U32: - case Decoder::Opcode::V_CMP_LT_U32: - case Decoder::Opcode::V_CMPX_LT_U32: opcode = IR::ValueOpcode::ULessThan32; break; - case Decoder::Opcode::S_CMP_LE_U32: - case Decoder::Opcode::V_CMP_LE_U32: - case Decoder::Opcode::V_CMPX_LE_U32: opcode = IR::ValueOpcode::ULessThanEqual32; break; - case Decoder::Opcode::S_CMP_GT_I32: - case Decoder::Opcode::V_CMP_GT_I32: - case Decoder::Opcode::V_CMPX_GT_I32: opcode = IR::ValueOpcode::SGreaterThan32; break; - case Decoder::Opcode::S_CMP_GE_I32: - case Decoder::Opcode::V_CMP_GE_I32: - case Decoder::Opcode::V_CMPX_GE_I32: opcode = IR::ValueOpcode::SGreaterThanEqual32; break; - case Decoder::Opcode::S_CMP_LT_I32: - case Decoder::Opcode::V_CMP_LT_I32: - case Decoder::Opcode::V_CMPX_LT_I32: opcode = IR::ValueOpcode::SLessThan32; break; - case Decoder::Opcode::S_CMP_LE_I32: - case Decoder::Opcode::V_CMP_LE_I32: - case Decoder::Opcode::V_CMPX_LE_I32: opcode = IR::ValueOpcode::SLessThanEqual32; break; - case Decoder::Opcode::S_CMP_EQ_U64: - case Decoder::Opcode::V_CMP_EQ_I64: - opcode = IR::ValueOpcode::IEqual64; - type = IR::Type::U64; - break; - case Decoder::Opcode::V_CMP_GT_U64: - opcode = IR::ValueOpcode::UGreaterThan64; - type = IR::Type::U64; - break; - case Decoder::Opcode::S_CMP_LG_U64: - case Decoder::Opcode::V_CMP_NE_U64: - case Decoder::Opcode::V_CMPX_NE_I64: - case Decoder::Opcode::V_CMPX_NE_U64: - opcode = IR::ValueOpcode::INotEqual64; - type = IR::Type::U64; - break; - default: return false; +void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar, + bool cmpx) { + if (scalar) { + ir.SetScc(value); + return; } + const auto masked = ir.LogicalAnd(ir.GetExec(), value); + if (cmpx) { + const auto mask = BallotMask(masked); + ir.SetExec(masked); + ir.SetExecLo(mask[0]); + ir.SetExecHi(mask[1]); + return; + } + WriteMask(inst.dst, masked); +} + +void Translator::EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar, + bool cmpx) { + EmitCompareResult(inst, IR::U1(IR::Value(value)), scalar, cmpx); +} + +void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + IR::Type type, bool scalar, bool cmpx) { const auto lhs = ReadOperand(inst.src0, type); const auto rhs = ReadOperand(inst.src1, type); - WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs}))); - return true; + EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), scalar, cmpx); } -bool Translator::TranslateInteger16Compare(const Decoder::Instruction& inst) { - IR::ValueOpcode opcode {}; - bool signed_value = false; - switch (inst.opcode) { - case Decoder::Opcode::V_CMP_EQ_U16: - case Decoder::Opcode::V_CMP_EQ_I16: opcode = IR::ValueOpcode::IEqual32; break; - case Decoder::Opcode::V_CMP_NE_U16: - case Decoder::Opcode::V_CMP_NE_I16: opcode = IR::ValueOpcode::INotEqual32; break; - case Decoder::Opcode::V_CMP_GT_U16: opcode = IR::ValueOpcode::UGreaterThan32; break; - case Decoder::Opcode::V_CMP_GE_U16: opcode = IR::ValueOpcode::UGreaterThanEqual32; break; - case Decoder::Opcode::V_CMP_LT_U16: opcode = IR::ValueOpcode::ULessThan32; break; - case Decoder::Opcode::V_CMP_LE_U16: opcode = IR::ValueOpcode::ULessThanEqual32; break; - case Decoder::Opcode::V_CMP_GT_I16: - opcode = IR::ValueOpcode::SGreaterThan32; - signed_value = true; - break; - case Decoder::Opcode::V_CMP_GE_I16: - opcode = IR::ValueOpcode::SGreaterThanEqual32; - signed_value = true; - break; - case Decoder::Opcode::V_CMP_LT_I16: - opcode = IR::ValueOpcode::SLessThan32; - signed_value = true; - break; - case Decoder::Opcode::V_CMP_LE_I16: - opcode = IR::ValueOpcode::SLessThanEqual32; - signed_value = true; - break; - default: return false; - } - if (inst.opcode == Decoder::Opcode::V_CMP_EQ_I16 || - inst.opcode == Decoder::Opcode::V_CMP_NE_I16) { - signed_value = true; - } - WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {ReadU16AsU32(inst.src0, signed_value), - ReadU16AsU32(inst.src1, signed_value)}))); - return true; +void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool signed_value, bool cmpx) { + const auto lhs = ReadU16AsU32(inst.src0, signed_value); + const auto rhs = ReadU16AsU32(inst.src1, signed_value); + EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx); } -bool Translator::TranslateFloatCompare(const Decoder::Instruction& inst) { - IR::ValueOpcode opcode {}; - bool half = false; - switch (inst.opcode) { - case Decoder::Opcode::V_CMP_F_F32: - WriteCompareResult(inst.dst, IR::U1(IR::Value(false))); - return true; - case Decoder::Opcode::V_CMP_TRU_F32: - WriteCompareResult(inst.dst, IR::U1(IR::Value(true))); - return true; - case Decoder::Opcode::V_CMP_EQ_F32: - case Decoder::Opcode::V_CMPX_EQ_F32: opcode = IR::ValueOpcode::FPOrdEqual32; break; - case Decoder::Opcode::V_CMP_LG_F32: - case Decoder::Opcode::V_CMPX_LG_F32: opcode = IR::ValueOpcode::FPOrdNotEqual32; break; - case Decoder::Opcode::V_CMP_GT_F32: - case Decoder::Opcode::V_CMPX_GT_F32: opcode = IR::ValueOpcode::FPOrdGreaterThan32; break; - case Decoder::Opcode::V_CMP_GE_F32: - case Decoder::Opcode::V_CMPX_GE_F32: - opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32; - break; - case Decoder::Opcode::V_CMP_LT_F32: - case Decoder::Opcode::V_CMPX_LT_F32: opcode = IR::ValueOpcode::FPOrdLessThan32; break; - case Decoder::Opcode::V_CMP_LE_F32: - case Decoder::Opcode::V_CMPX_LE_F32: opcode = IR::ValueOpcode::FPOrdLessThanEqual32; break; - case Decoder::Opcode::V_CMP_NLG_F32: - case Decoder::Opcode::V_CMPX_NLG_F32: opcode = IR::ValueOpcode::FPUnordEqual32; break; - case Decoder::Opcode::V_CMP_NEQ_F32: - case Decoder::Opcode::V_CMPX_NEQ_F32: opcode = IR::ValueOpcode::FPUnordNotEqual32; break; - case Decoder::Opcode::V_CMP_NLE_F32: - case Decoder::Opcode::V_CMPX_NLE_F32: opcode = IR::ValueOpcode::FPUnordGreaterThan32; break; - case Decoder::Opcode::V_CMP_NLT_F32: - case Decoder::Opcode::V_CMPX_NLT_F32: - opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32; - break; - case Decoder::Opcode::V_CMP_NGE_F32: - case Decoder::Opcode::V_CMPX_NGE_F32: opcode = IR::ValueOpcode::FPUnordLessThan32; break; - case Decoder::Opcode::V_CMP_NGT_F32: - case Decoder::Opcode::V_CMPX_NGT_F32: - opcode = IR::ValueOpcode::FPUnordLessThanEqual32; - break; - case Decoder::Opcode::V_CMP_EQ_F16: - case Decoder::Opcode::V_CMPX_EQ_F16: - opcode = IR::ValueOpcode::FPOrdEqual32; - half = true; - break; - case Decoder::Opcode::V_CMP_LG_F16: - opcode = IR::ValueOpcode::FPOrdNotEqual32; - half = true; - break; - case Decoder::Opcode::V_CMP_GT_F16: - case Decoder::Opcode::V_CMPX_GT_F16: - opcode = IR::ValueOpcode::FPOrdGreaterThan32; - half = true; - break; - case Decoder::Opcode::V_CMP_GE_F16: - case Decoder::Opcode::V_CMPX_GE_F16: - opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32; - half = true; - break; - case Decoder::Opcode::V_CMP_LT_F16: - case Decoder::Opcode::V_CMPX_LT_F16: - opcode = IR::ValueOpcode::FPOrdLessThan32; - half = true; - break; - case Decoder::Opcode::V_CMP_LE_F16: - case Decoder::Opcode::V_CMPX_LE_F16: - opcode = IR::ValueOpcode::FPOrdLessThanEqual32; - half = true; - break; - case Decoder::Opcode::V_CMP_NEQ_F16: - case Decoder::Opcode::V_CMPX_NEQ_F16: - opcode = IR::ValueOpcode::FPUnordNotEqual32; - half = true; - break; - case Decoder::Opcode::V_CMPX_NLT_F16: - opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32; - half = true; - break; - case Decoder::Opcode::V_CMP_O_F32: - case Decoder::Opcode::V_CMP_U_F32: { - const auto lhs = IR::F32(ReadOperand(inst.src0, IR::Type::F32)); - const auto rhs = IR::F32(ReadOperand(inst.src1, IR::Type::F32)); - const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {lhs})), - IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {rhs}))); - const auto result = - inst.opcode == Decoder::Opcode::V_CMP_U_F32 ? unordered : ir.LogicalNot(unordered); - WriteCompareResult(inst.dst, result); - return true; - } - case Decoder::Opcode::V_CMP_CLASS_F32: { - const auto value = ReadOperand(inst.src0, IR::Type::F32); - const auto mask = ReadOperand(inst.src1, IR::Type::U32); - WriteCompareResult(inst.dst, - IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask}))); - return true; - } - default: return false; - } +void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool half, bool cmpx) { const auto lhs = half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32); const auto rhs = half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32); - WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs}))); - return true; + EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx); +} + +void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered) { + const auto lhs = IR::F32(ReadOperand(inst.src0, IR::Type::F32)); + const auto rhs = IR::F32(ReadOperand(inst.src1, IR::Type::F32)); + const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {lhs})), + IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {rhs}))); + EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, false); +} + +void Translator::V_CMP_CLASS_F32(const Decoder::Instruction& inst) { + const auto value = ReadOperand(inst.src0, IR::Type::F32); + const auto mask = ReadOperand(inst.src1, IR::Type::U32); + EmitCompareResult(inst, IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask})), false, + false); } } // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index 53bd79462..9b2d7eaa9 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -21,420 +21,350 @@ Decoder::Operand ConditionOperand(Decoder::OperandKind kind) { } // namespace -bool Translator::TranslateStateOperation(const Decoder::Instruction& inst) { - const auto zero = IR::U32(IR::Value(0u)); - switch (inst.opcode) { - case Decoder::Opcode::S_AND_SAVEEXEC_B32: - case Decoder::Opcode::S_ANDN1_SAVEEXEC_B32: - case Decoder::Opcode::S_AND_SAVEEXEC_B64: - case Decoder::Opcode::S_ANDN1_SAVEEXEC_B64: - case Decoder::Opcode::S_ORN2_SAVEEXEC_B64: { - const auto old = ir.GetExec(); - const auto src = ReadMask(inst.src0); - IR::U1 result; - switch (inst.opcode) { - case Decoder::Opcode::S_AND_SAVEEXEC_B32: - case Decoder::Opcode::S_AND_SAVEEXEC_B64: result = ir.LogicalAnd(old, src); break; - case Decoder::Opcode::S_ANDN1_SAVEEXEC_B32: - case Decoder::Opcode::S_ANDN1_SAVEEXEC_B64: - result = ir.LogicalAnd(old, ir.LogicalNot(src)); - break; - case Decoder::Opcode::S_ORN2_SAVEEXEC_B64: - result = ir.LogicalOr(ir.LogicalNot(old), src); - break; - default: EXIT("invalid saveexec opcode"); - } - if (inst.opcode == Decoder::Opcode::S_AND_SAVEEXEC_B64 || - inst.opcode == Decoder::Opcode::S_ANDN1_SAVEEXEC_B64 || - inst.opcode == Decoder::Opcode::S_ORN2_SAVEEXEC_B64) { - WriteMask64(inst.dst, old); - } else { - WriteMask(inst.dst, old); - } - const auto mask = BallotMask(result); - ir.SetExec(result); - ir.SetExecLo(mask[0]); - ir.SetExecHi(mask[1]); - ir.SetScc(result); - return true; - } - case Decoder::Opcode::S_ADD_U32: - case Decoder::Opcode::S_ADDC_U32: - case Decoder::Opcode::V_ADD_I32: - case Decoder::Opcode::V_ADDC_U32: { - const auto lhs = ReadU32(inst.src0); - const auto rhs = ReadU32(inst.src1); - const auto carry_in = inst.opcode == Decoder::Opcode::S_ADDC_U32 - ? ConditionBit(ConditionOperand(Decoder::OperandKind::Scc)) - : inst.opcode == Decoder::Opcode::V_ADDC_U32 - ? ConditionBit(inst.src2) - : zero; - const auto add0 = ir.Emit(IR::ValueOpcode::IAddCarry32, {lhs, rhs}); - const auto partial = ir.CompositeExtract(add0, 0); - const auto carry0 = ir.CompositeExtract(add0, 1); - const auto add1 = ir.Emit(IR::ValueOpcode::IAddCarry32, {partial, carry_in}); - const auto result = ir.CompositeExtract(add1, 0); - const auto carry1 = ir.CompositeExtract(add1, 1); - const auto carry = ir.INotEqual(ir.BitwiseOr(carry0, carry1), zero); - WriteOperand(DestinationOperand(inst), result); - if (inst.opcode == Decoder::Opcode::S_ADD_U32 || - inst.opcode == Decoder::Opcode::S_ADDC_U32) { - ir.SetScc(carry); - } else { - WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), carry)); - } - return true; - } - case Decoder::Opcode::S_SUB_U32: - case Decoder::Opcode::V_SUB_I32: - case Decoder::Opcode::V_SUBREV_I32: { - const bool reverse = inst.opcode == Decoder::Opcode::V_SUBREV_I32; - const auto lhs = ReadU32(reverse ? inst.src1 : inst.src0); - const auto rhs = ReadU32(reverse ? inst.src0 : inst.src1); - const auto result = ir.ISub(lhs, rhs); - const auto borrow = ir.UGreaterThan(rhs, lhs); - WriteOperand(DestinationOperand(inst), result); - if (inst.opcode == Decoder::Opcode::S_SUB_U32) { - ir.SetScc(borrow); - } else { - WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow)); - } - return true; - } - case Decoder::Opcode::S_SUBB_U32: - case Decoder::Opcode::V_SUBREV_CO_CI_U32: { - const bool vector = inst.opcode == Decoder::Opcode::V_SUBREV_CO_CI_U32; - const auto lhs = ReadU32(vector ? inst.src1 : inst.src0); - const auto rhs = ReadU32(vector ? inst.src0 : inst.src1); - const auto borrow_in = vector - ? ConditionBit(inst.src2) - : ConditionBit(ConditionOperand(Decoder::OperandKind::Scc)); - const auto partial = ir.ISub(lhs, rhs); - const auto result = ir.ISub(partial, borrow_in); - const auto borrow0 = ir.UGreaterThan(rhs, lhs); - const auto borrow1 = ir.UGreaterThan(borrow_in, partial); - const auto borrow = ir.LogicalOr(borrow0, borrow1); - WriteOperand(DestinationOperand(inst), result); - if (vector) { - WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow)); - } else { - ir.SetScc(borrow); - } - return true; - } - case Decoder::Opcode::S_ADD_I32: - case Decoder::Opcode::S_SUB_I32: { - const auto lhs = ReadU32(inst.src0); - const auto rhs = ReadU32(inst.src1); - const bool subtract = inst.opcode == Decoder::Opcode::S_SUB_I32; - const auto result = subtract ? ir.ISub(lhs, rhs) : ir.IAdd(lhs, rhs); - const auto shift = IR::U32(IR::Value(31u)); - const auto lhs_sign = ir.ShiftRightLogical(lhs, shift); - const auto rhs_sign = ir.ShiftRightLogical(rhs, shift); - const auto out_sign = ir.ShiftRightLogical(result, shift); - const auto inputs = - subtract ? ir.INotEqual(lhs_sign, rhs_sign) : ir.IEqual(lhs_sign, rhs_sign); - const auto changed = ir.INotEqual(lhs_sign, out_sign); - WriteOperand(DestinationOperand(inst), result); - ir.SetScc(ir.LogicalAnd(inputs, changed)); - return true; - } - case Decoder::Opcode::S_LSHL1_ADD_U32: - case Decoder::Opcode::S_LSHL2_ADD_U32: - case Decoder::Opcode::S_LSHL3_ADD_U32: - case Decoder::Opcode::S_LSHL4_ADD_U32: { - const uint32_t shift_amount = 1u + static_cast(inst.opcode) - - static_cast(Decoder::Opcode::S_LSHL1_ADD_U32); - const auto lhs = ReadU32(inst.src0); - const auto shift = IR::U32(IR::Value(shift_amount)); - const auto rhs = ReadU32(inst.src1); - const auto shifted = ir.ShiftLeftLogical(lhs, shift); - const auto result = ir.IAdd(shifted, rhs); - const auto add_carry = ir.ULessThan(result, shifted); - const auto inverse_shift = ir.ISub(IR::U32(IR::Value(32u)), shift); - const auto shifted_out = ir.ShiftRightLogical(lhs, inverse_shift); - const auto shift_carry = ir.INotEqual(shifted_out, zero); - WriteOperand(DestinationOperand(inst), result); - ir.SetScc(ir.LogicalOr(add_carry, shift_carry)); - return true; - } - case Decoder::Opcode::S_MIN_I32: - case Decoder::Opcode::S_MAX_I32: - case Decoder::Opcode::S_MIN_U32: - case Decoder::Opcode::S_MAX_U32: { - const auto lhs = ReadU32(inst.src0); - const auto rhs = ReadU32(inst.src1); - IR::ValueOpcode value_opcode; - IR::ValueOpcode compare_opcode; - switch (inst.opcode) { - case Decoder::Opcode::S_MIN_I32: - value_opcode = IR::ValueOpcode::SMin32; - compare_opcode = IR::ValueOpcode::SLessThan32; - break; - case Decoder::Opcode::S_MAX_I32: - value_opcode = IR::ValueOpcode::SMax32; - compare_opcode = IR::ValueOpcode::SGreaterThan32; - break; - case Decoder::Opcode::S_MIN_U32: - value_opcode = IR::ValueOpcode::UMin32; - compare_opcode = IR::ValueOpcode::ULessThan32; - break; - case Decoder::Opcode::S_MAX_U32: - value_opcode = IR::ValueOpcode::UMax32; - compare_opcode = IR::ValueOpcode::UGreaterThan32; - break; - default: EXIT("invalid scalar min/max opcode"); - } - WriteOperand(DestinationOperand(inst), ir.Emit(value_opcode, {lhs, rhs})); - ir.SetScc(IR::U1(ir.Emit(compare_opcode, {lhs, rhs}))); - return true; - } - default: return false; +void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation, + bool negate_exec, bool negate_source, bool write_64) { + const auto old = ir.GetExec(); + const auto src = ReadMask(inst.src0); + const auto lhs = negate_exec ? ir.LogicalNot(old) : old; + const auto rhs = negate_source ? ir.LogicalNot(src) : src; + const auto result = IR::U1(ir.Emit(operation, {lhs, rhs})); + if (write_64) { + WriteMask64(inst.dst, old); + } else { + WriteMask(inst.dst, old); + } + const auto mask = BallotMask(result); + ir.SetExec(result); + ir.SetExecLo(mask[0]); + ir.SetExecHi(mask[1]); + ir.SetScc(result); +} + +void Translator::ADD_U32(const Decoder::Instruction& inst, bool vector, bool use_carry_in) { + const auto zero = IR::U32(IR::Value(0u)); + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + const auto carry_in = !use_carry_in ? zero + : vector ? ConditionBit(inst.src2) + : ConditionBit(ConditionOperand(Decoder::OperandKind::Scc)); + const auto add0 = ir.Emit(IR::ValueOpcode::IAddCarry32, {lhs, rhs}); + const auto partial = ir.CompositeExtract(add0, 0); + const auto carry0 = ir.CompositeExtract(add0, 1); + const auto add1 = ir.Emit(IR::ValueOpcode::IAddCarry32, {partial, carry_in}); + const auto result = ir.CompositeExtract(add1, 0); + const auto carry1 = ir.CompositeExtract(add1, 1); + const auto carry = ir.INotEqual(ir.BitwiseOr(carry0, carry1), zero); + WriteOperand(DestinationOperand(inst), result); + if (vector) { + WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), carry)); + } else { + ir.SetScc(carry); } } -bool Translator::TranslateControlOperation(const Decoder::Instruction& inst) { - switch (inst.opcode) { - case Decoder::Opcode::V_NOP: return true; - case Decoder::Opcode::S_NOP: - case Decoder::Opcode::S_SETREG_B32: - case Decoder::Opcode::S_SLEEP: - case Decoder::Opcode::S_TRAP: ir.Emit(IR::ValueOpcode::ControlNop); return true; - case Decoder::Opcode::S_WAITCNT: - case Decoder::Opcode::S_WAITCNT_DEPCTR: ir.Emit(IR::ValueOpcode::Waitcnt); return true; - case Decoder::Opcode::S_BARRIER: ir.Emit(IR::ValueOpcode::Barrier); return true; - case Decoder::Opcode::S_SENDMSG: ir.Emit(IR::ValueOpcode::Sendmsg); return true; - case Decoder::Opcode::S_TTRACEDATA: ir.Emit(IR::ValueOpcode::TtraceData); return true; - case Decoder::Opcode::S_INST_PREFETCH: ir.Emit(IR::ValueOpcode::InstPrefetch); return true; - default: return false; +void Translator::SUB_U32(const Decoder::Instruction& inst, bool vector, bool reverse) { + const auto lhs = ReadU32(reverse ? inst.src1 : inst.src0); + const auto rhs = ReadU32(reverse ? inst.src0 : inst.src1); + const auto result = ir.ISub(lhs, rhs); + const auto borrow = ir.UGreaterThan(rhs, lhs); + WriteOperand(DestinationOperand(inst), result); + if (vector) { + WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow)); + } else { + ir.SetScc(borrow); } } -bool Translator::TranslateMove(const Decoder::Instruction& inst, std::string* error) { - switch (inst.opcode) { - case Decoder::Opcode::S_GETPC_B64: { - const auto base = IR::U64(ir.Emit(IR::ValueOpcode::GetShaderBase)); - const auto address = IR::U64(ir.Emit( - IR::ValueOpcode::IAdd64, {base, IR::Value(static_cast(inst.pc) + 4u)})); - if (inst.dst.kind == Decoder::OperandKind::Null) { - return true; - } - auto high = PlainOperand(inst.dst); - switch (inst.dst.kind) { - case Decoder::OperandKind::Sgpr: - if (inst.dst.reg < 105u) { - high.reg++; - } else if (inst.dst.reg == 105u) { - high.kind = Decoder::OperandKind::VccLo; - high.reg = 0u; - } else { - if (error != nullptr) { - *error = "S_GETPC_B64 destination does not name a valid scalar pair"; - } - return false; - } - break; - case Decoder::OperandKind::VccLo: high.kind = Decoder::OperandKind::VccHi; break; - case Decoder::OperandKind::M0: high.kind = Decoder::OperandKind::Null; break; - case Decoder::OperandKind::ExecLo: high.kind = Decoder::OperandKind::ExecHi; break; - default: - if (error != nullptr) { - *error = "S_GETPC_B64 destination does not name a valid scalar pair"; - } - return false; - } - const auto words = ExtractU64(address); - WriteOperand(inst.dst, words[0]); - WriteOperand(high, words[1]); - return true; - } - case Decoder::Opcode::S_CSELECT_B32: { - const auto result = ir.Select(ir.GetScc(), ReadU32(inst.src0), ReadU32(inst.src1)); - WriteOperand(DestinationOperand(inst), result); - return true; - } - case Decoder::Opcode::S_CSELECT_B64: { - const auto condition = ir.GetScc(); - const auto lhs = ReadU32Pair(inst.src0); - const auto rhs = ReadU32Pair(inst.src1); - const auto selected_mask = IR::U1(ir.Emit( - IR::ValueOpcode::SelectU1, {condition, ReadMask(inst.src0), ReadMask(inst.src1)})); - const auto selected_mask_valid = - IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, - {condition, ReadMaskValid(inst.src0), ReadMaskValid(inst.src1)})); - if (IsExecOrVcc(inst.dst)) { - WriteMask64(inst.dst, selected_mask); - return true; - } - WriteU32Pair(inst.dst, {ir.Select(condition, lhs[0], rhs[0]), - ir.Select(condition, lhs[1], rhs[1])}); - if (inst.dst.kind == Decoder::OperandKind::Sgpr) { - const auto dst = static_cast(inst.dst.reg); - ir.SetThreadBitScalarReg(dst, selected_mask); - ir.SetScalarMaskTag(dst, selected_mask_valid); - } - return true; - } - case Decoder::Opcode::S_MOV_B32: - case Decoder::Opcode::S_MOVK_I32: - case Decoder::Opcode::V_MOV_B32: - if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) { - WriteMask(inst.dst, ReadMask(inst.src0)); - } else if (inst.opcode == Decoder::Opcode::V_MOV_B32 && - (inst.src0.negate || inst.src0.absolute)) { - WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::F32)); +void Translator::SUBB_U32(const Decoder::Instruction& inst, bool vector) { + const auto lhs = ReadU32(vector ? inst.src1 : inst.src0); + const auto rhs = ReadU32(vector ? inst.src0 : inst.src1); + const auto borrow_in = vector ? ConditionBit(inst.src2) + : ConditionBit(ConditionOperand(Decoder::OperandKind::Scc)); + const auto partial = ir.ISub(lhs, rhs); + const auto result = ir.ISub(partial, borrow_in); + const auto borrow0 = ir.UGreaterThan(rhs, lhs); + const auto borrow1 = ir.UGreaterThan(borrow_in, partial); + const auto borrow = ir.LogicalOr(borrow0, borrow1); + WriteOperand(DestinationOperand(inst), result); + if (vector) { + WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow)); + } else { + ir.SetScc(borrow); + } +} + +void Translator::S_ADD_SUB_I32(const Decoder::Instruction& inst, bool subtract) { + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + const auto result = subtract ? ir.ISub(lhs, rhs) : ir.IAdd(lhs, rhs); + const auto shift = IR::U32(IR::Value(31u)); + const auto lhs_sign = ir.ShiftRightLogical(lhs, shift); + const auto rhs_sign = ir.ShiftRightLogical(rhs, shift); + const auto out_sign = ir.ShiftRightLogical(result, shift); + const auto inputs = subtract ? ir.INotEqual(lhs_sign, rhs_sign) : ir.IEqual(lhs_sign, rhs_sign); + const auto changed = ir.INotEqual(lhs_sign, out_sign); + WriteOperand(DestinationOperand(inst), result); + ir.SetScc(ir.LogicalAnd(inputs, changed)); +} + +void Translator::S_LSHL_ADD_U32(const Decoder::Instruction& inst, uint32_t shift_amount) { + const auto lhs = ReadU32(inst.src0); + const auto shift = IR::U32(IR::Value(shift_amount)); + const auto rhs = ReadU32(inst.src1); + const auto shifted = ir.ShiftLeftLogical(lhs, shift); + const auto result = ir.IAdd(shifted, rhs); + const auto add_carry = ir.ULessThan(result, shifted); + const auto inverse_shift = ir.ISub(IR::U32(IR::Value(32u)), shift); + const auto shifted_out = ir.ShiftRightLogical(lhs, inverse_shift); + const auto shift_carry = ir.INotEqual(shifted_out, IR::U32(IR::Value(0u))); + WriteOperand(DestinationOperand(inst), result); + ir.SetScc(ir.LogicalOr(add_carry, shift_carry)); +} + +void Translator::ScalarMinMax32(const Decoder::Instruction& inst, IR::ValueOpcode value_opcode, + IR::ValueOpcode compare_opcode) { + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + WriteOperand(DestinationOperand(inst), ir.Emit(value_opcode, {lhs, rhs})); + ir.SetScc(IR::U1(ir.Emit(compare_opcode, {lhs, rhs}))); +} + +void Translator::EmitControlNop() { + ir.Emit(IR::ValueOpcode::ControlNop); +} + +void Translator::EmitWaitcnt() { + ir.Emit(IR::ValueOpcode::Waitcnt); +} + +void Translator::S_BARRIER() { + ir.Emit(IR::ValueOpcode::Barrier); +} + +void Translator::S_SENDMSG() { + ir.Emit(IR::ValueOpcode::Sendmsg); +} + +void Translator::S_TTRACEDATA() { + ir.Emit(IR::ValueOpcode::TtraceData); +} + +void Translator::S_INST_PREFETCH() { + ir.Emit(IR::ValueOpcode::InstPrefetch); +} + +bool Translator::S_GETPC_B64(const Decoder::Instruction& inst, std::string* error) { + const auto base = IR::U64(ir.Emit(IR::ValueOpcode::GetShaderBase)); + const auto address = IR::U64( + ir.Emit(IR::ValueOpcode::IAdd64, {base, IR::Value(static_cast(inst.pc) + 4u)})); + if (inst.dst.kind == Decoder::OperandKind::Null) { + return true; + } + auto high = PlainOperand(inst.dst); + switch (inst.dst.kind) { + case Decoder::OperandKind::Sgpr: + if (inst.dst.reg < 105u) { + high.reg++; + } else if (inst.dst.reg == 105u) { + high.kind = Decoder::OperandKind::VccLo; + high.reg = 0u; } else { - WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::U32)); - } - return true; - case Decoder::Opcode::S_MOV_B64: { - if (IsExecOrVcc(inst.dst) || IsExecOrVcc(inst.src0)) { - WriteMask64(inst.dst, ReadMask(inst.src0)); - return true; - } - const bool scalar_copy = inst.dst.kind == Decoder::OperandKind::Sgpr && - inst.src0.kind == Decoder::OperandKind::Sgpr; - IR::U1 source_mask; - IR::U1 source_mask_valid; - if (scalar_copy) { - source_mask = ir.GetThreadBitScalarReg(static_cast(inst.src0.reg)); - source_mask_valid = ir.GetScalarMaskTag(static_cast(inst.src0.reg)); - } - WriteU32Pair(inst.dst, ReadU32Pair(inst.src0)); - if (scalar_copy) { - ir.SetThreadBitScalarReg(static_cast(inst.dst.reg), source_mask); - ir.SetScalarMaskTag(static_cast(inst.dst.reg), source_mask_valid); - } - return true; - } - case Decoder::Opcode::S_WQM_B64: { - if (!IsExecOrVcc(inst.dst) && !IsExecOrVcc(inst.src0)) { - const auto mask_valid = ReadMaskValid(inst.src0); - const auto invocation_result = - IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)})); - const auto result = IR::U64( - ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)})); - WriteOperand(DestinationOperand(inst), result); - if (inst.dst.kind == Decoder::OperandKind::Sgpr) { - const auto dst = static_cast(inst.dst.reg); - ir.SetThreadBitScalarReg(dst, invocation_result); - ir.SetScalarMaskTag(dst, mask_valid); - const auto raw_nonzero = IR::U1( - ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})); - ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, - {mask_valid, invocation_result, raw_nonzero}))); - } else { - ir.SetScc(IR::U1( - ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); + if (error != nullptr) { + *error = "S_GETPC_B64 destination does not name a valid scalar pair"; } - return true; + return false; } - const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)})); - WriteMask64(inst.dst, result); - ir.SetScc(result); - return true; - } - default: return false; + break; + case Decoder::OperandKind::VccLo: high.kind = Decoder::OperandKind::VccHi; break; + case Decoder::OperandKind::M0: high.kind = Decoder::OperandKind::Null; break; + case Decoder::OperandKind::ExecLo: high.kind = Decoder::OperandKind::ExecHi; break; + default: + if (error != nullptr) { + *error = "S_GETPC_B64 destination does not name a valid scalar pair"; + } + return false; + } + const auto words = ExtractU64(address); + WriteOperand(inst.dst, words[0]); + WriteOperand(high, words[1]); + return true; +} + +void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) { + const auto result = ir.Select(ir.GetScc(), ReadU32(inst.src0), ReadU32(inst.src1)); + WriteOperand(DestinationOperand(inst), result); +} + +void Translator::S_CSELECT_B64(const Decoder::Instruction& inst) { + const auto condition = ir.GetScc(); + const auto lhs = ReadU32Pair(inst.src0); + const auto rhs = ReadU32Pair(inst.src1); + const auto selected_mask = IR::U1( + ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMask(inst.src0), ReadMask(inst.src1)})); + const auto selected_mask_valid = + IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, + {condition, ReadMaskValid(inst.src0), ReadMaskValid(inst.src1)})); + if (IsExecOrVcc(inst.dst)) { + WriteMask64(inst.dst, selected_mask); + return; + } + WriteU32Pair(inst.dst, + {ir.Select(condition, lhs[0], rhs[0]), ir.Select(condition, lhs[1], rhs[1])}); + if (inst.dst.kind == Decoder::OperandKind::Sgpr) { + const auto dst = static_cast(inst.dst.reg); + ir.SetThreadBitScalarReg(dst, selected_mask); + ir.SetScalarMaskTag(dst, selected_mask_valid); } } -bool Translator::TranslateLaneOperation(const Decoder::Instruction& inst, std::string* error) { +void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers) { + if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) { + WriteMask(inst.dst, ReadMask(inst.src0)); + } else if (apply_float_modifiers && (inst.src0.negate || inst.src0.absolute)) { + WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::F32)); + } else { + WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::U32)); + } +} + +void Translator::S_MOV_B64(const Decoder::Instruction& inst) { + if (IsExecOrVcc(inst.dst) || IsExecOrVcc(inst.src0)) { + WriteMask64(inst.dst, ReadMask(inst.src0)); + return; + } + const bool scalar_copy = + inst.dst.kind == Decoder::OperandKind::Sgpr && inst.src0.kind == Decoder::OperandKind::Sgpr; + IR::U1 source_mask; + IR::U1 source_mask_valid; + if (scalar_copy) { + source_mask = ir.GetThreadBitScalarReg(static_cast(inst.src0.reg)); + source_mask_valid = ir.GetScalarMaskTag(static_cast(inst.src0.reg)); + } + WriteU32Pair(inst.dst, ReadU32Pair(inst.src0)); + if (scalar_copy) { + ir.SetThreadBitScalarReg(static_cast(inst.dst.reg), source_mask); + ir.SetScalarMaskTag(static_cast(inst.dst.reg), source_mask_valid); + } +} + +void Translator::S_WQM_B64(const Decoder::Instruction& inst) { + if (!IsExecOrVcc(inst.dst) && !IsExecOrVcc(inst.src0)) { + const auto mask_valid = ReadMaskValid(inst.src0); + const auto invocation_result = + IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)})); + const auto result = + IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)})); + WriteOperand(DestinationOperand(inst), result); + if (inst.dst.kind == Decoder::OperandKind::Sgpr) { + const auto dst = static_cast(inst.dst.reg); + ir.SetThreadBitScalarReg(dst, invocation_result); + ir.SetScalarMaskTag(dst, mask_valid); + const auto raw_nonzero = + IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})); + ir.SetScc(IR::U1( + ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero}))); + } else { + ir.SetScc( + IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); + } + return; + } + const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)})); + WriteMask64(inst.dst, result); + ir.SetScc(result); +} + +bool Translator::V_MOVRELS_B32(const Decoder::Instruction& inst, std::string* error) { + if (inst.dst.kind != Decoder::OperandKind::Vgpr || + inst.src0.kind != Decoder::OperandKind::Vgpr) { + if (error != nullptr) { + *error = "V_MOVRELS_B32 requires VGPR source and destination"; + } + return false; + } + if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp || + inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || inst.src0.absolute || + inst.src0.dpp) { + if (error != nullptr) { + *error = "V_MOVRELS_B32 modifiers are not implemented"; + } + return false; + } + const auto base = inst.src0.reg; + const auto m0 = ir.BitwiseAnd(ReadU32(ConditionOperand(Decoder::OperandKind::M0)), + IR::U32(IR::Value(0xffu))); + auto selected = ir.GetVectorReg(static_cast(base)); + for (uint32_t index = base + 1u; index < current_vector_limit; index++) { + const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base))); + selected = ir.Select(match, ir.GetVectorReg(static_cast(index)), selected); + } + WriteOperand(DestinationOperand(inst), selected); + return true; +} + +bool Translator::V_MOVRELD_B32(const Decoder::Instruction& inst, std::string* error) { + if (inst.dst.kind != Decoder::OperandKind::Vgpr) { + if (error != nullptr) { + *error = "V_MOVRELD_B32 requires VGPR destination"; + } + return false; + } + if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp || + inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || inst.src0.absolute || + inst.src0.dpp) { + if (error != nullptr) { + *error = "V_MOVRELD_B32 modifiers are not implemented"; + } + return false; + } + const auto base = inst.dst.reg; + const auto value = ReadU32(inst.src0); + const auto m0 = ir.BitwiseAnd(ReadU32(ConditionOperand(Decoder::OperandKind::M0)), + IR::U32(IR::Value(0xffu))); + for (uint32_t index = base; index < current_vector_limit; index++) { + const auto reg = static_cast(index); + const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base))); + const auto write = ir.LogicalAnd(ir.GetExec(), match); + ir.SetVectorReg(reg, ir.Select(write, value, ir.GetVectorReg(reg))); + } + return true; +} + +void Translator::V_READFIRSTLANE_B32(const Decoder::Instruction& inst) { + const auto result = ir.Emit(IR::ValueOpcode::ReadFirstLane, {ReadU32(inst.src0), ir.GetExec()}); + WriteOperand(DestinationOperand(inst), result); +} + +void Translator::V_READLANE_B32(const Decoder::Instruction& inst) { const auto lane_mask = IR::U32(IR::Value(current_wave_size == 32u ? 31u : 63u)); - switch (inst.opcode) { - case Decoder::Opcode::V_MOVRELS_B32: { - if (inst.dst.kind != Decoder::OperandKind::Vgpr || - inst.src0.kind != Decoder::OperandKind::Vgpr) { - if (error != nullptr) { - *error = "V_MOVRELS_B32 requires VGPR source and destination"; - } - return false; - } - if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp || - inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || - inst.src0.absolute || inst.src0.dpp) { - if (error != nullptr) { - *error = "V_MOVRELS_B32 modifiers are not implemented"; - } - return false; - } - const auto base = inst.src0.reg; - const auto m0 = ir.BitwiseAnd(ReadU32(ConditionOperand(Decoder::OperandKind::M0)), - IR::U32(IR::Value(0xffu))); - auto selected = ir.GetVectorReg(static_cast(base)); - for (uint32_t index = base + 1u; index < current_vector_limit; index++) { - const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base))); - selected = - ir.Select(match, ir.GetVectorReg(static_cast(index)), selected); - } - WriteOperand(DestinationOperand(inst), selected); - return true; - } - case Decoder::Opcode::V_MOVRELD_B32: { - if (inst.dst.kind != Decoder::OperandKind::Vgpr) { - if (error != nullptr) { - *error = "V_MOVRELD_B32 requires VGPR destination"; - } - return false; - } - if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp || - inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || - inst.src0.absolute || inst.src0.dpp) { - if (error != nullptr) { - *error = "V_MOVRELD_B32 modifiers are not implemented"; - } - return false; - } - const auto base = inst.dst.reg; - const auto value = ReadU32(inst.src0); - const auto m0 = ir.BitwiseAnd(ReadU32(ConditionOperand(Decoder::OperandKind::M0)), - IR::U32(IR::Value(0xffu))); - for (uint32_t index = base; index < current_vector_limit; index++) { - const auto reg = static_cast(index); - const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base))); - const auto write = ir.LogicalAnd(ir.GetExec(), match); - ir.SetVectorReg(reg, ir.Select(write, value, ir.GetVectorReg(reg))); - } - return true; - } - case Decoder::Opcode::V_READFIRSTLANE_B32: { - const auto result = - ir.Emit(IR::ValueOpcode::ReadFirstLane, {ReadU32(inst.src0), ir.GetExec()}); - WriteOperand(DestinationOperand(inst), result); - return true; - } - case Decoder::Opcode::V_READLANE_B32: { - const auto lane = ir.BitwiseAnd(ReadU32(inst.src1), lane_mask); - WriteOperand(DestinationOperand(inst), - ir.Emit(IR::ValueOpcode::ReadLane, {ReadU32(inst.src0), lane})); - return true; - } - case Decoder::Opcode::V_WRITELANE_B32: { - EXIT_IF(inst.dst.kind != Decoder::OperandKind::Vgpr); - const auto reg = static_cast(inst.dst.reg); - const auto lane = ir.BitwiseAnd(ReadU32(inst.src1), lane_mask); - const auto result = IR::U32(ir.Emit(IR::ValueOpcode::WriteLane, - {ir.GetVectorReg(reg), ReadU32(inst.src0), lane})); - ir.SetVectorReg(reg, result); - return true; - } - case Decoder::Opcode::V_PERMLANE16_B32: - case Decoder::Opcode::V_PERMLANEX16_B32: { - const IR::PermlaneFlags flags { - .x16 = inst.opcode == Decoder::Opcode::V_PERMLANEX16_B32, - .fetch_inactive = inst.dst.op_sel, - .bound_control = inst.dst.op_sel_hi, - }; - const auto result = ir.Emit( - IR::ValueOpcode::Permlane16U32, - {ReadU32(inst.src0), ReadU32(inst.src1), ReadU32(inst.src2), ir.GetExec()}, flags); - auto dst = DestinationOperand(inst); - dst.op_sel = false; - dst.op_sel_hi = false; - WriteOperand(dst, result); - return true; - } - default: return false; - } + const auto lane = ir.BitwiseAnd(ReadU32(inst.src1), lane_mask); + WriteOperand(DestinationOperand(inst), + ir.Emit(IR::ValueOpcode::ReadLane, {ReadU32(inst.src0), lane})); +} + +void Translator::V_WRITELANE_B32(const Decoder::Instruction& inst) { + EXIT_IF(inst.dst.kind != Decoder::OperandKind::Vgpr); + const auto lane_mask = IR::U32(IR::Value(current_wave_size == 32u ? 31u : 63u)); + const auto reg = static_cast(inst.dst.reg); + const auto lane = ir.BitwiseAnd(ReadU32(inst.src1), lane_mask); + const auto result = IR::U32( + ir.Emit(IR::ValueOpcode::WriteLane, {ir.GetVectorReg(reg), ReadU32(inst.src0), lane})); + ir.SetVectorReg(reg, result); +} + +void Translator::V_PERMLANE16_B32(const Decoder::Instruction& inst, bool x16) { + const IR::PermlaneFlags flags { + .x16 = x16, + .fetch_inactive = inst.dst.op_sel, + .bound_control = inst.dst.op_sel_hi, + }; + const auto result = + ir.Emit(IR::ValueOpcode::Permlane16U32, + {ReadU32(inst.src0), ReadU32(inst.src1), ReadU32(inst.src2), ir.GetExec()}, flags); + auto dst = DestinationOperand(inst); + dst.op_sel = false; + dst.op_sel_hi = false; + WriteOperand(dst, result); } } // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Convert.cpp b/src/graphics/shader/recompiler/frontend/translate/Convert.cpp index 30bd8a9b9..720290c86 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Convert.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Convert.cpp @@ -38,146 +38,136 @@ IR::U32 Translator::ConvertF32ToI32Saturated(IR::F32 value, float lower_bound, f return ir.Select(nan, IR::U32(IR::Value(0u)), clamped); } -bool Translator::TranslateConversion(const Decoder::Instruction& inst) { - const auto f32 = [&](uint32_t index) { - return IR::F32(ReadOperand(SourceAt(inst, index), IR::Type::F32)); - }; - const auto u32 = [&](uint32_t index) { return ReadU32(SourceAt(inst, index)); }; - const auto convert_u32 = [&](IR::F32 value) { - return ConvertF32ToU32Saturated(value, 4294967296.0f, 4294967040.0f, 0xffffffffu); - }; - const auto convert_i32 = [&](IR::F32 value) { - return ConvertF32ToI32Saturated(value, -2147483648.0f, 2147483648.0f, 2147483520.0f, - 0x80000000u, 0x7fffffffu); - }; +void Translator::V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index) { + const auto source = ReadU32(SourceAt(inst, 0)); + const auto byte = + ir.BitwiseAnd(ir.ShiftRightLogical(source, IR::U32(IR::Value(byte_index * 8u))), + IR::U32(IR::Value(0xffu))); + WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::ConvertF32U32, {byte})); +} - switch (inst.opcode) { - case Decoder::Opcode::V_CVT_F32_UBYTE0: - case Decoder::Opcode::V_CVT_F32_UBYTE1: - case Decoder::Opcode::V_CVT_F32_UBYTE2: - case Decoder::Opcode::V_CVT_F32_UBYTE3: { - const auto index = static_cast(inst.opcode) - - static_cast(Decoder::Opcode::V_CVT_F32_UBYTE0); - const auto byte = - ir.BitwiseAnd(ir.ShiftRightLogical(u32(0), IR::U32(IR::Value(index * 8u))), - IR::U32(IR::Value(0xffu))); - WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::ConvertF32U32, {byte})); - return true; - } - case Decoder::Opcode::V_CVT_F32_U32: - WriteOperand(DestinationOperand(inst), - ir.Emit(IR::ValueOpcode::ConvertF32U32, {u32(0)})); - return true; - case Decoder::Opcode::V_CVT_F32_I32: - WriteOperand(DestinationOperand(inst), - ir.Emit(IR::ValueOpcode::ConvertF32S32, {u32(0)})); - return true; - case Decoder::Opcode::V_CVT_U32_F32: - WriteOperand(DestinationOperand(inst), convert_u32(f32(0))); - return true; - case Decoder::Opcode::V_CVT_I32_F32: - WriteOperand(DestinationOperand(inst), convert_i32(f32(0))); - return true; - case Decoder::Opcode::V_CVT_F16_F32: - WriteF16(DestinationOperand(inst), f32(0)); - return true; - case Decoder::Opcode::V_CVT_F32_F16: - WriteOperand(DestinationOperand(inst), ReadF16AsF32(inst.src0)); - return true; - case Decoder::Opcode::V_CVT_F16_U16: { - const auto value = - IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32U32, {ReadU16AsU32(inst.src0, false)})); - WriteF16(DestinationOperand(inst), value); - return true; - } - case Decoder::Opcode::V_CVT_F16_I16: { - const auto value = - IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {ReadU16AsU32(inst.src0, true)})); - WriteF16(DestinationOperand(inst), value); - return true; - } - case Decoder::Opcode::V_CVT_U16_F16: { - const auto converted = - ConvertF32ToU32Saturated(ReadF16AsF32(inst.src0), 65536.0f, 65535.0f, 0xffffu); - WriteU16(DestinationOperand(inst), converted); - return true; - } - case Decoder::Opcode::V_CVT_I16_F16: { - const auto converted = ConvertF32ToI32Saturated( - ReadF16AsF32(inst.src0), -32768.0f, 32768.0f, 32767.0f, 0xffff8000u, 0x7fffu); - WriteU16(DestinationOperand(inst), - ir.BitwiseAnd(converted, IR::U32(IR::Value(0xffffu)))); - return true; - } - case Decoder::Opcode::V_CVT_RPI_I32_F32: { - const auto biased = - IR::F32(ir.Emit(IR::ValueOpcode::FPAdd32, {f32(0), IR::Value::F32(0.5f)})); - const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {biased})); - WriteOperand(DestinationOperand(inst), convert_i32(rounded)); - return true; - } - case Decoder::Opcode::V_CVT_FLR_I32_F32: { - const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {f32(0)})); - WriteOperand(DestinationOperand(inst), convert_i32(rounded)); - return true; - } - case Decoder::Opcode::V_FREXP_EXP_I32_F32: { - const auto bits = ir.BitCastU32(f32(0)); - const auto exponent = IR::U32( - ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)})); - const auto mantissa = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu))); - const auto normal = ir.ISub(exponent, IR::U32(IR::Value(126u))); - const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {mantissa})); - const auto subnormal = ir.ISub(msb, IR::U32(IR::Value(148u))); - const auto denormal = ir.Select(ir.INotEqual(mantissa, IR::U32(IR::Value(0u))), - subnormal, IR::U32(IR::Value(0u))); - const auto finite = ir.Select( - ir.INotEqual(exponent, IR::U32(IR::Value(0xffu))), - ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal, denormal), - IR::U32(IR::Value(0u))); - WriteOperand(DestinationOperand(inst), finite); - return true; - } - case Decoder::Opcode::V_CVT_OFF_F32_I4: { - const auto nibble = IR::U32( - ir.Emit(IR::ValueOpcode::BitFieldSExtract, {u32(0), IR::Value(0u), IR::Value(4u)})); - const auto value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {nibble})); - WriteOperand(DestinationOperand(inst), - ir.Emit(IR::ValueOpcode::FPMul32, {value, IR::Value::F32(1.0f / 16.0f)})); - return true; - } - case Decoder::Opcode::V_CVT_PKRTZ_F16_F32: - WriteOperand(DestinationOperand(inst), - ir.Emit(IR::ValueOpcode::PackFloat2x16Rtz, {f32(0), f32(1)})); - return true; - case Decoder::Opcode::V_CVT_PKNORM_I16_F32: - case Decoder::Opcode::V_CVT_PKNORM_U16_F32: { - const auto pair = ir.Emit(IR::ValueOpcode::CompositeConstructF32x2, {f32(0), f32(1)}); - const auto opcode = inst.opcode == Decoder::Opcode::V_CVT_PKNORM_I16_F32 - ? IR::ValueOpcode::PackSnorm2x16 - : IR::ValueOpcode::PackUnorm2x16; - WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {pair})); - return true; - } - case Decoder::Opcode::V_CVT_PK_U8_F32: { - const auto byte = ConvertF32ToU32Saturated(f32(0), 255.0f, 255.0f, 255u); - const auto index = ir.BitwiseAnd(u32(1), IR::U32(IR::Value(3u))); - const auto shift = ir.ShiftLeftLogical(index, IR::U32(IR::Value(3u))); - const auto mask = ir.ShiftLeftLogical(IR::U32(IR::Value(0xffu)), shift); - const auto base = ir.BitwiseAnd(u32(2), ir.BitwiseNot(mask)); - WriteOperand(DestinationOperand(inst), - ir.BitwiseOr(base, ir.ShiftLeftLogical(byte, shift))); - return true; - } - case Decoder::Opcode::V_PACK_B32_F16: { - const auto low = ReadF16LaneBits(inst.src0, false); - const auto high = - ir.ShiftLeftLogical(ReadF16LaneBits(inst.src1, false), IR::U32(IR::Value(16u))); - WriteOperand(DestinationOperand(inst), ir.BitwiseOr(low, high)); - return true; - } - default: return false; +void Translator::V_CVT_F32_U32(const Decoder::Instruction& inst) { + WriteOperand(DestinationOperand(inst), + ir.Emit(IR::ValueOpcode::ConvertF32U32, {ReadU32(SourceAt(inst, 0))})); +} + +void Translator::V_CVT_F32_I32(const Decoder::Instruction& inst) { + WriteOperand(DestinationOperand(inst), + ir.Emit(IR::ValueOpcode::ConvertF32S32, {ReadU32(SourceAt(inst, 0))})); +} + +void Translator::V_CVT_U32_F32(const Decoder::Instruction& inst) { + const auto value = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32)); + WriteOperand(DestinationOperand(inst), + ConvertF32ToU32Saturated(value, 4294967296.0f, 4294967040.0f, 0xffffffffu)); +} + +void Translator::V_CVT_I32_F32(const Decoder::Instruction& inst) { + const auto value = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32)); + WriteOperand(DestinationOperand(inst), + ConvertF32ToI32Saturated(value, -2147483648.0f, 2147483648.0f, 2147483520.0f, + 0x80000000u, 0x7fffffffu)); +} + +void Translator::V_CVT_F16_F32(const Decoder::Instruction& inst) { + WriteF16(DestinationOperand(inst), IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32))); +} + +void Translator::V_CVT_F32_F16(const Decoder::Instruction& inst) { + WriteOperand(DestinationOperand(inst), ReadF16AsF32(inst.src0)); +} + +void Translator::V_CVT_F16_16(const Decoder::Instruction& inst, bool signed_value) { + const auto value = IR::F32( + ir.Emit(signed_value ? IR::ValueOpcode::ConvertF32S32 : IR::ValueOpcode::ConvertF32U32, + {ReadU16AsU32(inst.src0, signed_value)})); + WriteF16(DestinationOperand(inst), value); +} + +void Translator::V_CVT_16_F16(const Decoder::Instruction& inst, bool signed_value) { + const auto value = ReadF16AsF32(inst.src0); + if (signed_value) { + const auto converted = + ConvertF32ToI32Saturated(value, -32768.0f, 32768.0f, 32767.0f, 0xffff8000u, 0x7fffu); + WriteU16(DestinationOperand(inst), ir.BitwiseAnd(converted, IR::U32(IR::Value(0xffffu)))); + return; } + WriteU16(DestinationOperand(inst), + ConvertF32ToU32Saturated(value, 65536.0f, 65535.0f, 0xffffu)); +} + +void Translator::V_CVT_RPI_I32_F32(const Decoder::Instruction& inst) { + const auto source = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32)); + const auto biased = IR::F32(ir.Emit(IR::ValueOpcode::FPAdd32, {source, IR::Value::F32(0.5f)})); + const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {biased})); + WriteOperand(DestinationOperand(inst), + ConvertF32ToI32Saturated(rounded, -2147483648.0f, 2147483648.0f, 2147483520.0f, + 0x80000000u, 0x7fffffffu)); +} + +void Translator::V_CVT_FLR_I32_F32(const Decoder::Instruction& inst) { + const auto source = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32)); + const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {source})); + WriteOperand(DestinationOperand(inst), + ConvertF32ToI32Saturated(rounded, -2147483648.0f, 2147483648.0f, 2147483520.0f, + 0x80000000u, 0x7fffffffu)); +} + +void Translator::V_FREXP_EXP_I32_F32(const Decoder::Instruction& inst) { + const auto source = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32)); + const auto bits = ir.BitCastU32(source); + const auto exponent = + IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)})); + const auto mantissa = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu))); + const auto normal = ir.ISub(exponent, IR::U32(IR::Value(126u))); + const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {mantissa})); + const auto subnormal = ir.ISub(msb, IR::U32(IR::Value(148u))); + const auto denormal = ir.Select(ir.INotEqual(mantissa, IR::U32(IR::Value(0u))), subnormal, + IR::U32(IR::Value(0u))); + const auto finite = + ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0xffu))), + ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal, denormal), + IR::U32(IR::Value(0u))); + WriteOperand(DestinationOperand(inst), finite); +} + +void Translator::V_CVT_OFF_F32_I4(const Decoder::Instruction& inst) { + const auto nibble = + IR::U32(ir.Emit(IR::ValueOpcode::BitFieldSExtract, + {ReadU32(SourceAt(inst, 0)), IR::Value(0u), IR::Value(4u)})); + const auto value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {nibble})); + WriteOperand(DestinationOperand(inst), + ir.Emit(IR::ValueOpcode::FPMul32, {value, IR::Value::F32(1.0f / 16.0f)})); +} + +void Translator::V_CVT_PKRTZ_F16_F32(const Decoder::Instruction& inst) { + const auto lhs = ReadOperand(SourceAt(inst, 0), IR::Type::F32); + const auto rhs = ReadOperand(SourceAt(inst, 1), IR::Type::F32); + WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::PackFloat2x16Rtz, {lhs, rhs})); +} + +void Translator::V_CVT_PKNORM_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { + const auto lhs = ReadOperand(SourceAt(inst, 0), IR::Type::F32); + const auto rhs = ReadOperand(SourceAt(inst, 1), IR::Type::F32); + const auto pair = ir.Emit(IR::ValueOpcode::CompositeConstructF32x2, {lhs, rhs}); + WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {pair})); +} + +void Translator::V_CVT_PK_U8_F32(const Decoder::Instruction& inst) { + const auto source = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32)); + const auto byte = ConvertF32ToU32Saturated(source, 255.0f, 255.0f, 255u); + const auto index = ir.BitwiseAnd(ReadU32(SourceAt(inst, 1)), IR::U32(IR::Value(3u))); + const auto shift = ir.ShiftLeftLogical(index, IR::U32(IR::Value(3u))); + const auto mask = ir.ShiftLeftLogical(IR::U32(IR::Value(0xffu)), shift); + const auto base = ir.BitwiseAnd(ReadU32(SourceAt(inst, 2)), ir.BitwiseNot(mask)); + WriteOperand(DestinationOperand(inst), ir.BitwiseOr(base, ir.ShiftLeftLogical(byte, shift))); +} + +void Translator::V_PACK_B32_F16(const Decoder::Instruction& inst) { + const auto low = ReadF16LaneBits(inst.src0, false); + const auto high = + ir.ShiftLeftLogical(ReadF16LaneBits(inst.src1, false), IR::U32(IR::Value(16u))); + WriteOperand(DestinationOperand(inst), ir.BitwiseOr(low, high)); } IR::U32 Translator::PackU16Lanes(IR::U32 low, IR::U32 high) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp index 4222efb1d..f102e5d6a 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp @@ -2,94 +2,6 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend { -bool Translator::TranslateU64MaskOperation(const Decoder::Instruction& inst) { - switch (inst.opcode) { - case Decoder::Opcode::S_AND_B64: - case Decoder::Opcode::S_ANDN2_B64: - case Decoder::Opcode::S_OR_B64: - case Decoder::Opcode::S_ORN2_B64: - case Decoder::Opcode::S_XOR_B64: - case Decoder::Opcode::S_NAND_B64: - case Decoder::Opcode::S_NOR_B64: - case Decoder::Opcode::S_XNOR_B64: - case Decoder::Opcode::S_NOT_B64: break; - default: return false; - } - const auto invocation_result = EvaluateU64Mask(inst); - const auto is_exec_or_vcc = [](const Decoder::Operand& operand) { - switch (operand.kind) { - case Decoder::OperandKind::ExecLo: - case Decoder::OperandKind::ExecHi: - case Decoder::OperandKind::VccLo: - case Decoder::OperandKind::VccHi: return true; - default: return false; - } - }; - if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) || - (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) { - WriteMask64(inst.dst, invocation_result); - ir.SetScc(invocation_result); - return true; - } - - const auto lhs = ReadU32Pair(inst.src0); - auto mask_valid = ReadMaskValid(inst.src0); - if (inst.src_count > 1u) { - mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src1)); - } - std::array result; - if (inst.opcode == Decoder::Opcode::S_NOT_B64) { - result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])}; - } else { - const auto rhs = ReadU32Pair(inst.src1); - for (uint32_t component = 0; component < 2u; component++) { - switch (inst.opcode) { - case Decoder::Opcode::S_AND_B64: - result[component] = ir.BitwiseAnd(lhs[component], rhs[component]); - break; - case Decoder::Opcode::S_ANDN2_B64: - result[component] = - ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component])); - break; - case Decoder::Opcode::S_OR_B64: - result[component] = ir.BitwiseOr(lhs[component], rhs[component]); - break; - case Decoder::Opcode::S_ORN2_B64: - result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component])); - break; - case Decoder::Opcode::S_XOR_B64: - result[component] = ir.BitwiseXor(lhs[component], rhs[component]); - break; - case Decoder::Opcode::S_NAND_B64: - result[component] = - ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component])); - break; - case Decoder::Opcode::S_NOR_B64: - result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component])); - break; - case Decoder::Opcode::S_XNOR_B64: - result[component] = - ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component])); - break; - default: return false; - } - } - } - WriteU32Pair(inst.dst, result); - if (inst.dst.kind == Decoder::OperandKind::Sgpr) { - const auto dst = static_cast(inst.dst.reg); - ir.SetThreadBitScalarReg(dst, invocation_result); - ir.SetScalarMaskTag(dst, mask_valid); - const auto raw_nonzero = - ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))); - ir.SetScc(IR::U1( - ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero}))); - } else { - ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)))); - } - return true; -} - bool Translator::TranslateInstruction(const Decoder::Instruction& inst, std::string* error) { current_opcode = inst.opcode; current_pc = inst.pc; @@ -106,49 +18,33 @@ bool Translator::TranslateInstruction(const Decoder::Instruction& inst, std::str *error = "unsupported decoded instruction: " + Decoder::InstructionToString(inst); } return false; - case Decoder::Opcode::S_GETPC_B64: return TranslateMove(inst, error); - case Decoder::Opcode::V_MOVRELS_B32: - case Decoder::Opcode::V_MOVRELD_B32: return TranslateLaneOperation(inst, error); default: break; } + bool translated = false; switch (inst.family) { + case Decoder::Family::SOP1: + case Decoder::Family::SOP2: + case Decoder::Family::SOPK: + case Decoder::Family::SOPC: + case Decoder::Family::SOPP: translated = EmitScalar(inst, error); break; + case Decoder::Family::VOP1: + case Decoder::Family::VOP2: + case Decoder::Family::VOP3: + case Decoder::Family::VOP3P: + case Decoder::Family::VOPC: translated = EmitVector(inst, error); break; case Decoder::Family::SMEM: case Decoder::Family::MUBUF: case Decoder::Family::MTBUF: case Decoder::Family::FLAT: case Decoder::Family::DS: - case Decoder::Family::MIMG: return TranslateMemoryOperation(inst, error); - case Decoder::Family::EXP: return TranslateAttributeOperation(inst, error); + case Decoder::Family::MIMG: translated = EmitMemory(inst, error); break; + case Decoder::Family::VINTRP: translated = EmitInterpolation(inst, error); break; + case Decoder::Family::EXP: translated = EXP(inst, error); break; default: break; } - switch (inst.opcode) { - case Decoder::Opcode::V_INTERP_P1_F32: - case Decoder::Opcode::V_INTERP_P2_F32: - case Decoder::Opcode::V_INTERP_MOV_F32: return TranslateAttributeOperation(inst, error); - case Decoder::Opcode::S_BRANCH: - case Decoder::Opcode::S_CBRANCH_SCC0: - case Decoder::Opcode::S_CBRANCH_SCC1: - case Decoder::Opcode::S_CBRANCH_VCCZ: - case Decoder::Opcode::S_CBRANCH_VCCNZ: - case Decoder::Opcode::S_CBRANCH_EXECZ: - case Decoder::Opcode::S_CBRANCH_EXECNZ: - case Decoder::Opcode::S_SETPC_B64: - case Decoder::Opcode::S_ENDPGM: return true; - default: break; - } - - const bool translated = TranslateControlOperation(inst) || TranslateStateOperation(inst) || - TranslateMove(inst, error) || TranslateLaneOperation(inst, error) || - TranslateIntegerCompare(inst) || TranslateInteger16Compare(inst) || - TranslateFloatCompare(inst) || TranslateConversion(inst) || - TranslateInteger16Operation(inst) || TranslatePackedInteger16(inst) || - TranslatePackedFloat16(inst) || TranslateFloat16Operation(inst) || - TranslateFloatOperation(inst) || TranslateU64MaskOperation(inst) || - TranslateSimpleInteger(inst) || TranslateComposedInteger(inst) || - TranslateExtendedInteger(inst); - if (!translated && error != nullptr) { + if (!translated && error != nullptr && error->empty()) { *error = "decoded opcode has no IR translation"; } return translated; diff --git a/src/graphics/shader/recompiler/frontend/translate/Float.cpp b/src/graphics/shader/recompiler/frontend/translate/Float.cpp index 9387f631d..6c754b7dd 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Float.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Float.cpp @@ -5,22 +5,13 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend { -bool Translator::TranslatePackedFloat16(const Decoder::Instruction& inst) { - IR::ValueOpcode opcode; - switch (inst.opcode) { - case Decoder::Opcode::V_PK_ADD_F16: opcode = IR::ValueOpcode::FPAdd32; break; - case Decoder::Opcode::V_PK_MUL_F16: opcode = IR::ValueOpcode::FPMul32; break; - case Decoder::Opcode::V_PK_MIN_F16: opcode = IR::ValueOpcode::FPMin32; break; - case Decoder::Opcode::V_PK_MAX_F16: opcode = IR::ValueOpcode::FPMax32; break; - case Decoder::Opcode::V_PK_FMA_F16: opcode = IR::ValueOpcode::FPFma32; break; - case Decoder::Opcode::V_PK_FMAC_F16: opcode = IR::ValueOpcode::FPFma32; break; - default: return false; - } +bool Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool accumulator, bool quiet_snan) { const auto translate_lane = [&](bool high) { const auto lhs = ReadF16LaneAsF32(inst.src0, high, true); const auto rhs = ReadF16LaneAsF32(inst.src1, high, true); IR::F32 result; - if (inst.opcode == Decoder::Opcode::V_PK_FMAC_F16) { + if (accumulator) { result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.dst, high, true)})); } else if (inst.src_count == 3u) { result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src2, high, true)})); @@ -33,9 +24,8 @@ bool Translator::TranslatePackedFloat16(const Decoder::Instruction& inst) { raw.omod = 0u; raw.clamp = false; auto result = PackHalf2x16(translate_lane(false), translate_lane(true)); - if (inst.opcode == Decoder::Opcode::V_PK_MIN_F16 || - inst.opcode == Decoder::Opcode::V_PK_MAX_F16) { - const auto quiet_snan = [&](const Decoder::Operand& operand, bool high) { + if (quiet_snan) { + const auto quiet_snan_lane = [&](const Decoder::Operand& operand, bool high) { const auto bits = ReadU16LaneAsU32(operand, high, false); const auto exponent = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x7c00u))); const auto payload = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x01ffu))); @@ -44,8 +34,8 @@ bool Translator::TranslatePackedFloat16(const Decoder::Instruction& inst) { return std::pair {snan, ir.BitwiseOr(bits, IR::U32(IR::Value(0x0200u)))}; }; const auto override_lane = [&](bool high) { - const auto [lhs_snan, lhs_quiet] = quiet_snan(inst.src0, high); - const auto [rhs_snan, rhs_quiet] = quiet_snan(inst.src1, high); + const auto [lhs_snan, lhs_quiet] = quiet_snan_lane(inst.src0, high); + const auto [rhs_snan, rhs_quiet] = quiet_snan_lane(inst.src1, high); const auto normal = high ? ir.ShiftRightLogical(result, IR::U32(IR::Value(16u))) : ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))); return ir.Select(lhs_snan, lhs_quiet, ir.Select(rhs_snan, rhs_quiet, normal)); @@ -56,176 +46,130 @@ bool Translator::TranslatePackedFloat16(const Decoder::Instruction& inst) { return true; } -bool Translator::TranslateFloat16Operation(const Decoder::Instruction& inst) { - IR::ValueOpcode opcode; - switch (inst.opcode) { - case Decoder::Opcode::V_ADD_F16: opcode = IR::ValueOpcode::FPAdd32; break; - case Decoder::Opcode::V_SUB_F16: opcode = IR::ValueOpcode::FPSub32; break; - case Decoder::Opcode::V_SUBREV_F16: opcode = IR::ValueOpcode::FPSub32; break; - case Decoder::Opcode::V_MUL_F16: opcode = IR::ValueOpcode::FPMul32; break; - case Decoder::Opcode::V_MIN_F16: opcode = IR::ValueOpcode::FPMin32; break; - case Decoder::Opcode::V_MAX_F16: opcode = IR::ValueOpcode::FPMax32; break; - case Decoder::Opcode::V_FMAC_F16: - case Decoder::Opcode::V_FMAMK_F16: - case Decoder::Opcode::V_FMAAK_F16: - case Decoder::Opcode::V_FMA_F16: - case Decoder::Opcode::V_MAD_MIXLO_F16: opcode = IR::ValueOpcode::FPFma32; break; - case Decoder::Opcode::V_MAD_MIXHI_F16: opcode = IR::ValueOpcode::FPFma32; break; - case Decoder::Opcode::V_RCP_F16: opcode = IR::ValueOpcode::FPRecip32; break; - case Decoder::Opcode::V_SQRT_F16: opcode = IR::ValueOpcode::FPSqrt; break; - case Decoder::Opcode::V_RSQ_F16: opcode = IR::ValueOpcode::FPRecipSqrt32; break; - case Decoder::Opcode::V_LOG_F16: opcode = IR::ValueOpcode::FPLog2; break; - case Decoder::Opcode::V_EXP_F16: opcode = IR::ValueOpcode::FPExp2; break; - case Decoder::Opcode::V_FLOOR_F16: opcode = IR::ValueOpcode::FPFloor32; break; - case Decoder::Opcode::V_CEIL_F16: opcode = IR::ValueOpcode::FPCeil32; break; - case Decoder::Opcode::V_TRUNC_F16: opcode = IR::ValueOpcode::FPTrunc32; break; - case Decoder::Opcode::V_RNDNE_F16: opcode = IR::ValueOpcode::FPRoundEven32; break; - case Decoder::Opcode::V_MIN3_F16: opcode = IR::ValueOpcode::FPMinTri32; break; - case Decoder::Opcode::V_MAX3_F16: opcode = IR::ValueOpcode::FPMaxTri32; break; - case Decoder::Opcode::V_MED3_F16: opcode = IR::ValueOpcode::FPMedTri32; break; - default: return false; - } - const bool accumulator = inst.opcode == Decoder::Opcode::V_FMAC_F16; - const uint32_t source_count = accumulator ? 3u : inst.src_count; - std::array args; - for (uint32_t index = 0; index < source_count; index++) { - auto operand = accumulator && index == 2u ? inst.dst : SourceAt(inst, index); - if (inst.opcode == Decoder::Opcode::V_SUBREV_F16 && index < 2u) { - operand = SourceAt(inst, 1u - index); - } - const bool mix = inst.opcode == Decoder::Opcode::V_MAD_MIXLO_F16 || - inst.opcode == Decoder::Opcode::V_MAD_MIXHI_F16; - args[index] = mix ? IR::Value(ReadMixF32(operand)) : IR::Value(ReadF16AsF32(operand)); - } - IR::F32 result; - switch (source_count) { - case 1: result = IR::F32(ir.Emit(opcode, {args[0]})); break; - case 2: result = IR::F32(ir.Emit(opcode, {args[0], args[1]})); break; - case 3: result = IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]})); break; - default: EXIT("invalid half-float source count: %u", inst.src_count); - } - if (inst.opcode == Decoder::Opcode::V_SQRT_F16 || inst.opcode == Decoder::Opcode::V_RSQ_F16 || - inst.opcode == Decoder::Opcode::V_LOG_F16) { - const auto negative = - IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {args[0], IR::Value::F32(0.0f)})); - result = ApplyF32ResultModifiers(inst.dst, result); - const auto bits = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f))); - const auto invalid = IR::U32(IR::Value(inst.dst.clamp ? 0u : 0xfe00u)); - WriteU16(DestinationOperand(inst), ir.Select(negative, invalid, bits)); +bool Translator::Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool invalid_negative) { + const auto argument = ReadF16AsF32(inst.src0); + auto result = IR::F32(ir.Emit(opcode, {argument})); + if (!invalid_negative) { + WriteF16(DestinationOperand(inst), result); return true; } - WriteF16(DestinationOperand(inst), result); + const auto negative = + IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {argument, IR::Value::F32(0.0f)})); + result = ApplyF32ResultModifiers(inst.dst, result); + const auto bits = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f))); + const auto invalid = IR::U32(IR::Value(inst.dst.clamp ? 0u : 0xfe00u)); + WriteU16(DestinationOperand(inst), ir.Select(negative, invalid, bits)); return true; } -bool Translator::TranslateFloatOperation(const Decoder::Instruction& inst) { - if (inst.opcode == Decoder::Opcode::V_FREXP_MANT_F32) { - const auto bits = ReadU32(inst.src0); - const auto exponent = IR::U32( - ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)})); - const auto mantissa = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu))); - const auto sign = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x80000000u))); - const auto base = ir.BitwiseOr(sign, IR::U32(IR::Value(0x3f000000u))); - const auto normal = ir.BitwiseOr(base, mantissa); - const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {mantissa})); - const auto shift = ir.ISub(IR::U32(IR::Value(23u)), msb); - const auto fraction = - ir.BitwiseAnd(ir.ShiftLeftLogical(mantissa, shift), IR::U32(IR::Value(0x007fffffu))); - const auto subnormal = ir.BitwiseOr(base, fraction); - const auto zero = ir.IEqual(mantissa, IR::U32(IR::Value(0u))); - const auto finite = ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal, - ir.Select(zero, bits, subnormal)); - const auto result = ir.Select(ir.IEqual(exponent, IR::U32(IR::Value(0xffu))), bits, finite); - WriteOperand(DestinationOperand(inst), ir.BitCastF32(result)); - return true; - } - IR::ValueOpcode opcode {}; - switch (inst.opcode) { - case Decoder::Opcode::V_RCP_F32: opcode = IR::ValueOpcode::FPRecip32; break; - case Decoder::Opcode::V_RCP_IFLAG_F32: opcode = IR::ValueOpcode::FPRecipIFlag32; break; - case Decoder::Opcode::V_FRACT_F32: opcode = IR::ValueOpcode::FPFract32; break; - case Decoder::Opcode::V_TRUNC_F32: opcode = IR::ValueOpcode::FPTrunc32; break; - case Decoder::Opcode::V_CEIL_F32: opcode = IR::ValueOpcode::FPCeil32; break; - case Decoder::Opcode::V_RNDNE_F32: opcode = IR::ValueOpcode::FPRoundEven32; break; - case Decoder::Opcode::V_FLOOR_F32: opcode = IR::ValueOpcode::FPFloor32; break; - case Decoder::Opcode::V_EXP_F32: opcode = IR::ValueOpcode::FPExp2; break; - case Decoder::Opcode::V_LOG_F32: opcode = IR::ValueOpcode::FPLog2; break; - case Decoder::Opcode::V_RSQ_F32: opcode = IR::ValueOpcode::FPRecipSqrt32; break; - case Decoder::Opcode::V_SQRT_F32: opcode = IR::ValueOpcode::FPSqrt; break; - case Decoder::Opcode::V_SIN_F32: opcode = IR::ValueOpcode::FPSin; break; - case Decoder::Opcode::V_COS_F32: opcode = IR::ValueOpcode::FPCos; break; - case Decoder::Opcode::V_ADD_F32: opcode = IR::ValueOpcode::FPAdd32; break; - case Decoder::Opcode::V_SUB_F32: opcode = IR::ValueOpcode::FPSub32; break; - case Decoder::Opcode::V_SUBREV_F32: opcode = IR::ValueOpcode::FPSub32; break; - case Decoder::Opcode::V_MUL_F32: opcode = IR::ValueOpcode::FPMul32; break; - case Decoder::Opcode::V_MIN_F32: opcode = IR::ValueOpcode::FPMin32; break; - case Decoder::Opcode::V_MAX_F32: opcode = IR::ValueOpcode::FPMax32; break; - case Decoder::Opcode::V_MAC_F32: opcode = IR::ValueOpcode::FPFma32; break; - case Decoder::Opcode::V_MADMK_F32: - case Decoder::Opcode::V_MADAK_F32: - case Decoder::Opcode::V_MAD_F32: - case Decoder::Opcode::V_FMA_F32: opcode = IR::ValueOpcode::FPFma32; break; - case Decoder::Opcode::V_MIN3_F32: opcode = IR::ValueOpcode::FPMinTri32; break; - case Decoder::Opcode::V_MAX3_F32: opcode = IR::ValueOpcode::FPMaxTri32; break; - case Decoder::Opcode::V_MED3_F32: opcode = IR::ValueOpcode::FPMedTri32; break; - case Decoder::Opcode::V_LDEXP_F32: opcode = IR::ValueOpcode::FPLdexp; break; - default: break; - } - if (opcode != IR::ValueOpcode {}) { - const bool accumulator = inst.opcode == Decoder::Opcode::V_MAC_F32; - const bool fused = accumulator || inst.opcode == Decoder::Opcode::V_MADMK_F32 || - inst.opcode == Decoder::Opcode::V_MADAK_F32 || - inst.opcode == Decoder::Opcode::V_MAD_F32 || - inst.opcode == Decoder::Opcode::V_FMA_F32; - const uint32_t source_count = accumulator ? 3u : inst.src_count; - std::array args; - for (uint32_t index = 0; index < source_count; index++) { - const auto arg_type = IR::ArgTypeOf(opcode, index); - auto operand = accumulator && index == 2u ? inst.dst : SourceAt(inst, index); - if (inst.opcode == Decoder::Opcode::V_SUBREV_F32 && index < 2u) { - operand = SourceAt(inst, 1u - index); - } - args[index] = arg_type == IR::Type::F32 ? (fused ? IR::Value(ReadMixF32(operand)) - : ReadOperand(operand, IR::Type::F32)) - : ReadOperand(operand, arg_type); - } - IR::Value result; - switch (source_count) { - case 1: result = ir.Emit(opcode, {args[0]}); break; - case 2: result = ir.Emit(opcode, {args[0], args[1]}); break; - case 3: result = ir.Emit(opcode, {args[0], args[1], args[2]}); break; - default: EXIT("invalid floating-point source count: %u", inst.src_count); - } - WriteOperand(DestinationOperand(inst), result); - return true; - } +bool Translator::Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool reverse) { + const auto lhs = ReadF16AsF32(reverse ? inst.src1 : inst.src0); + const auto rhs = ReadF16AsF32(reverse ? inst.src0 : inst.src1); + WriteF16(DestinationOperand(inst), IR::F32(ir.Emit(opcode, {lhs, rhs}))); + return true; +} +bool Translator::Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool accumulator, bool mix) { + std::array args; + for (uint32_t index = 0; index < args.size(); index++) { + const auto operand = accumulator && index == 2u ? inst.dst : SourceAt(inst, index); + args[index] = mix ? IR::Value(ReadMixF32(operand)) : IR::Value(ReadF16AsF32(operand)); + } + WriteF16(DestinationOperand(inst), IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]}))); + return true; +} + +bool Translator::FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { + const auto type = IR::ArgTypeOf(opcode, 0); + WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {ReadOperand(inst.src0, type)})); + return true; +} + +bool Translator::FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool reverse) { + std::array args; + for (uint32_t index = 0; index < args.size(); index++) { + const auto operand = SourceAt(inst, reverse ? 1u - index : index); + args[index] = ReadOperand(operand, IR::ArgTypeOf(opcode, index)); + } + WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {args[0], args[1]})); + return true; +} + +bool Translator::FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool accumulator, bool mix) { + std::array args; + for (uint32_t index = 0; index < args.size(); index++) { + const auto operand = accumulator && index == 2u ? inst.dst : SourceAt(inst, index); + const auto type = IR::ArgTypeOf(opcode, index); + args[index] = type == IR::Type::F32 && mix ? IR::Value(ReadMixF32(operand)) + : ReadOperand(operand, type); + } + WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {args[0], args[1], args[2]})); + return true; +} + +bool Translator::V_FREXP_MANT_F32(const Decoder::Instruction& inst) { + const auto bits = ReadU32(inst.src0); + const auto exponent = + IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)})); + const auto mantissa = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu))); + const auto sign = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x80000000u))); + const auto base = ir.BitwiseOr(sign, IR::U32(IR::Value(0x3f000000u))); + const auto normal = ir.BitwiseOr(base, mantissa); + const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {mantissa})); + const auto shift = ir.ISub(IR::U32(IR::Value(23u)), msb); + const auto fraction = + ir.BitwiseAnd(ir.ShiftLeftLogical(mantissa, shift), IR::U32(IR::Value(0x007fffffu))); + const auto subnormal = ir.BitwiseOr(base, fraction); + const auto zero = ir.IEqual(mantissa, IR::U32(IR::Value(0u))); + const auto finite = ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal, + ir.Select(zero, bits, subnormal)); + const auto result = ir.Select(ir.IEqual(exponent, IR::U32(IR::Value(0xffu))), bits, finite); + WriteOperand(DestinationOperand(inst), ir.BitCastF32(result)); + return true; +} + +bool Translator::V_DOT2C_F32_F16(const Decoder::Instruction& inst) { + auto a = inst.src0; + a.op_sel = false; + a.op_sel_hi = true; + auto b = inst.src1; + b.op_sel = false; + b.op_sel_hi = true; + const auto a_lo = ReadF16LaneAsF32(a, false); + const auto a_hi = ReadF16LaneAsF32(a, true); + const auto b_lo = ReadF16LaneAsF32(b, false); + const auto b_hi = ReadF16LaneAsF32(b, true); + const auto acc = ReadMixF32(inst.dst); + const auto lo = IR::F32(ir.Emit(IR::ValueOpcode::FPFma32, {a_lo, b_lo, acc})); + WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo})); + return true; +} + +bool Translator::V_CUBEID_F32(const Decoder::Instruction& inst) { + return FloatCube(inst, 0u); +} + +bool Translator::V_CUBESC_F32(const Decoder::Instruction& inst) { + return FloatCube(inst, 1u); +} + +bool Translator::V_CUBETC_F32(const Decoder::Instruction& inst) { + return FloatCube(inst, 2u); +} + +bool Translator::V_CUBEMA_F32(const Decoder::Instruction& inst) { + return FloatCube(inst, 3u); +} + +bool Translator::FloatCube(const Decoder::Instruction& inst, uint32_t result_kind) { const auto select_f32 = [&](IR::U1 condition, IR::F32 true_value, IR::F32 false_value) { return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value})); }; - if (inst.opcode == Decoder::Opcode::V_DOT2C_F32_F16) { - auto a = inst.src0; - a.op_sel = false; - a.op_sel_hi = true; - auto b = inst.src1; - b.op_sel = false; - b.op_sel_hi = true; - const auto a_lo = ReadF16LaneAsF32(a, false); - const auto a_hi = ReadF16LaneAsF32(a, true); - const auto b_lo = ReadF16LaneAsF32(b, false); - const auto b_hi = ReadF16LaneAsF32(b, true); - const auto acc = ReadMixF32(inst.dst); - const auto lo = IR::F32(ir.Emit(IR::ValueOpcode::FPFma32, {a_lo, b_lo, acc})); - WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo})); - return true; - } - if (inst.opcode != Decoder::Opcode::V_CUBEID_F32 && - inst.opcode != Decoder::Opcode::V_CUBESC_F32 && - inst.opcode != Decoder::Opcode::V_CUBETC_F32 && - inst.opcode != Decoder::Opcode::V_CUBEMA_F32) { - return false; - } - const auto x = ReadMixF32(inst.src0); const auto y = ReadMixF32(inst.src1); const auto z = ReadMixF32(inst.src2); @@ -246,27 +190,23 @@ bool Translator::TranslateFloatOperation(const Decoder::Instruction& inst) { return select_f32(z_face, z_value, select_f32(y_face, y_value, x_value)); }; IR::F32 result; - switch (inst.opcode) { - case Decoder::Opcode::V_CUBEID_F32: + switch (result_kind) { + case 0u: result = select_face( select_f32(x_neg, IR::F32(IR::Value::F32(1.0f)), IR::F32(IR::Value::F32(0.0f))), select_f32(y_neg, IR::F32(IR::Value::F32(3.0f)), IR::F32(IR::Value::F32(2.0f))), select_f32(z_neg, IR::F32(IR::Value::F32(5.0f)), IR::F32(IR::Value::F32(4.0f)))); break; - case Decoder::Opcode::V_CUBESC_F32: - result = select_face(select_f32(x_neg, z, nz), x, select_f32(z_neg, nx, x)); - break; - case Decoder::Opcode::V_CUBETC_F32: - result = select_face(ny, select_f32(y_neg, nz, z), ny); - break; - case Decoder::Opcode::V_CUBEMA_F32: { + case 1u: result = select_face(select_f32(x_neg, z, nz), x, select_f32(z_neg, nx, x)); break; + case 2u: result = select_face(ny, select_f32(y_neg, nz, z), ny); break; + case 3u: { const auto two = IR::F32(IR::Value::F32(2.0f)); result = select_face(IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {x, two})), IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {y, two})), IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {z, two}))); break; } - default: return false; + default: EXIT("invalid cube result kind"); } WriteOperand(DestinationOperand(inst), result); return true; diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp index a1aecd150..2fa32851d 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -4,232 +4,153 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend { -bool Translator::TranslateInteger16Operation(const Decoder::Instruction& inst) { - const auto mask16 = [&](IR::U32 value) { - return ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu))); - }; - IR::Value result; - switch (inst.opcode) { - case Decoder::Opcode::V_LSHLREV_B16: - case Decoder::Opcode::V_LSHRREV_B16: - case Decoder::Opcode::V_ASHRREV_I16: { - const bool arithmetic = inst.opcode == Decoder::Opcode::V_ASHRREV_I16; - const auto value = ReadU16AsU32(inst.src1, arithmetic); - const auto count = - ir.BitwiseAnd(ReadU16AsU32(inst.src0, false), IR::U32(IR::Value(15u))); - if (inst.opcode == Decoder::Opcode::V_LSHLREV_B16) { - result = ir.ShiftLeftLogical(value, count); - } else if (arithmetic) { - result = ir.ShiftRightArithmetic(value, count); - } else { - result = ir.ShiftRightLogical(value, count); - } - break; - } - case Decoder::Opcode::V_ADD_NC_U16: - case Decoder::Opcode::V_ADD_NC_I16: - result = ir.IAdd(ReadU16AsU32(inst.src0, false), ReadU16AsU32(inst.src1, false)); - break; - case Decoder::Opcode::V_SUB_NC_U16: - case Decoder::Opcode::V_SUB_NC_I16: - result = ir.ISub(ReadU16AsU32(inst.src0, false), ReadU16AsU32(inst.src1, false)); - break; - case Decoder::Opcode::V_MED3_I16: - result = ir.Emit(IR::ValueOpcode::SMedTri32, - {ReadU16AsU32(inst.src0, true), ReadU16AsU32(inst.src1, true), - ReadU16AsU32(inst.src2, true)}); - break; - case Decoder::Opcode::V_MIN_I16: - case Decoder::Opcode::V_MAX_I16: - case Decoder::Opcode::V_MIN_U16: - case Decoder::Opcode::V_MAX_U16: { - const bool sign = inst.opcode == Decoder::Opcode::V_MIN_I16 || - inst.opcode == Decoder::Opcode::V_MAX_I16; - IR::ValueOpcode opcode; - switch (inst.opcode) { - case Decoder::Opcode::V_MIN_I16: opcode = IR::ValueOpcode::SMin32; break; - case Decoder::Opcode::V_MAX_I16: opcode = IR::ValueOpcode::SMax32; break; - case Decoder::Opcode::V_MIN_U16: opcode = IR::ValueOpcode::UMin32; break; - default: opcode = IR::ValueOpcode::UMax32; break; - } - result = - ir.Emit(opcode, {ReadU16AsU32(inst.src0, sign), ReadU16AsU32(inst.src1, sign)}); - break; - } - default: return false; - } - WriteU16(DestinationOperand(inst), mask16(IR::U32(result))); +bool Translator::Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool arithmetic) { + const auto value = ReadU16AsU32(inst.src1, arithmetic); + const auto count = ir.BitwiseAnd(ReadU16AsU32(inst.src0, false), IR::U32(IR::Value(15u))); + const auto result = IR::U32(ir.Emit(opcode, {value, count})); + WriteU16(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu)))); return true; } -bool Translator::TranslatePackedInteger16(const Decoder::Instruction& inst) { - const auto lane = [&](const Decoder::Operand& operand, bool high, bool sign) { - return ReadU16LaneAsU32(operand, high, sign); - }; - const auto mask_count = [&](IR::U32 value) { - return ir.BitwiseAnd(value, IR::U32(IR::Value(15u))); - }; - const auto translate_lane = [&](bool high) -> IR::U32 { - if (inst.opcode == Decoder::Opcode::V_PK_LSHLREV_B16 || - inst.opcode == Decoder::Opcode::V_PK_LSHRREV_B16 || - inst.opcode == Decoder::Opcode::V_PK_ASHRREV_I16) { - const auto count = mask_count(lane(inst.src0, high, false)); - const bool arithmetic = inst.opcode == Decoder::Opcode::V_PK_ASHRREV_I16; - const auto value = lane(inst.src1, high, arithmetic); - if (inst.opcode == Decoder::Opcode::V_PK_LSHLREV_B16) { - return ir.ShiftLeftLogical(value, count); - } - return arithmetic ? ir.ShiftRightArithmetic(value, count) - : ir.ShiftRightLogical(value, count); - } - - const bool signed_minmax = inst.opcode == Decoder::Opcode::V_PK_MAX_I16 || - inst.opcode == Decoder::Opcode::V_PK_MIN_I16; - const auto lhs = lane(inst.src0, high, signed_minmax); - const auto rhs = lane(inst.src1, high, signed_minmax); - switch (inst.opcode) { - case Decoder::Opcode::V_PK_MAD_I16: - case Decoder::Opcode::V_PK_MAD_U16: - return ir.IAdd(ir.IMul(lhs, rhs), - lane(inst.src2, high, inst.opcode == Decoder::Opcode::V_PK_MAD_I16)); - case Decoder::Opcode::V_PK_MUL_LO_U16: return ir.IMul(lhs, rhs); - case Decoder::Opcode::V_PK_ADD_I16: - case Decoder::Opcode::V_PK_ADD_U16: return ir.IAdd(lhs, rhs); - case Decoder::Opcode::V_PK_SUB_I16: - case Decoder::Opcode::V_PK_SUB_U16: return ir.ISub(lhs, rhs); - case Decoder::Opcode::V_PK_MAX_I16: - return IR::U32(ir.Emit(IR::ValueOpcode::SMax32, {lhs, rhs})); - case Decoder::Opcode::V_PK_MIN_I16: - return IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {lhs, rhs})); - case Decoder::Opcode::V_PK_MAX_U16: - return IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs})); - case Decoder::Opcode::V_PK_MIN_U16: - return IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs})); - default: EXIT("invalid packed integer opcode"); - } - }; - - switch (inst.opcode) { - case Decoder::Opcode::V_PK_MAD_I16: - case Decoder::Opcode::V_PK_MUL_LO_U16: - case Decoder::Opcode::V_PK_ADD_I16: - case Decoder::Opcode::V_PK_SUB_I16: - case Decoder::Opcode::V_PK_LSHLREV_B16: - case Decoder::Opcode::V_PK_LSHRREV_B16: - case Decoder::Opcode::V_PK_ASHRREV_I16: - case Decoder::Opcode::V_PK_MAX_I16: - case Decoder::Opcode::V_PK_MIN_I16: - case Decoder::Opcode::V_PK_MAD_U16: - case Decoder::Opcode::V_PK_ADD_U16: - case Decoder::Opcode::V_PK_SUB_U16: - case Decoder::Opcode::V_PK_MAX_U16: - case Decoder::Opcode::V_PK_MIN_U16: - WriteOperand(DestinationOperand(inst), - PackU16Lanes(translate_lane(false), translate_lane(true))); - return true; - default: return false; - } +bool Translator::Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool sign) { + const auto lhs = ReadU16AsU32(inst.src0, sign); + const auto rhs = ReadU16AsU32(inst.src1, sign); + const auto result = IR::U32(ir.Emit(opcode, {lhs, rhs})); + WriteU16(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu)))); + return true; } -IR::U1 Translator::EvaluateU64Mask(const Decoder::Instruction& inst) { - const auto lhs = [&] { return ReadMask(inst.src0); }; - const auto rhs = [&] { return ReadMask(inst.src1); }; - switch (inst.opcode) { - case Decoder::Opcode::S_AND_B64: return ir.LogicalAnd(lhs(), rhs()); - case Decoder::Opcode::S_OR_B64: return ir.LogicalOr(lhs(), rhs()); - case Decoder::Opcode::S_XOR_B64: - return IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})); - case Decoder::Opcode::S_ANDN2_B64: return ir.LogicalAnd(lhs(), ir.LogicalNot(rhs())); - case Decoder::Opcode::S_ORN2_B64: return ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); - case Decoder::Opcode::S_NAND_B64: return ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); - case Decoder::Opcode::S_NOR_B64: return ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); - case Decoder::Opcode::S_XNOR_B64: - return ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}))); - case Decoder::Opcode::S_NOT_B64: return ir.LogicalNot(lhs()); - default: EXIT("invalid 64-bit mask opcode"); - } +bool Translator::V_MED3_I16(const Decoder::Instruction& inst) { + const auto result = IR::U32(ir.Emit( + IR::ValueOpcode::SMedTri32, {ReadU16AsU32(inst.src0, true), ReadU16AsU32(inst.src1, true), + ReadU16AsU32(inst.src2, true)})); + WriteU16(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu)))); + return true; } -bool Translator::TranslateSimpleInteger(const Decoder::Instruction& inst) { - IR::ValueOpcode opcode {}; - IR::Type type = IR::Type::U32; - switch (inst.opcode) { - case Decoder::Opcode::S_ABS_I32: opcode = IR::ValueOpcode::IAbs32; break; - case Decoder::Opcode::S_MUL_I32: - case Decoder::Opcode::S_MULK_I32: - case Decoder::Opcode::V_MUL_LO_U32: - case Decoder::Opcode::V_MUL_LO_I32: opcode = IR::ValueOpcode::IMul32; break; - case Decoder::Opcode::S_MUL_HI_U32: - case Decoder::Opcode::V_MUL_HI_U32: opcode = IR::ValueOpcode::UMulHi; break; - case Decoder::Opcode::V_MUL_HI_I32: opcode = IR::ValueOpcode::SMulHi; break; - case Decoder::Opcode::V_ADD_NC_U32: opcode = IR::ValueOpcode::IAdd32; break; - case Decoder::Opcode::V_SUB_NC_U32: - case Decoder::Opcode::V_SUBREV_NC_U32: opcode = IR::ValueOpcode::ISub32; break; - case Decoder::Opcode::V_MIN_I32: opcode = IR::ValueOpcode::SMin32; break; - case Decoder::Opcode::V_MAX_I32: opcode = IR::ValueOpcode::SMax32; break; - case Decoder::Opcode::V_MIN_U32: opcode = IR::ValueOpcode::UMin32; break; - case Decoder::Opcode::V_MAX_U32: opcode = IR::ValueOpcode::UMax32; break; - case Decoder::Opcode::V_MIN3_I32: opcode = IR::ValueOpcode::SMinTri32; break; - case Decoder::Opcode::V_MAX3_I32: opcode = IR::ValueOpcode::SMaxTri32; break; - case Decoder::Opcode::V_MED3_I32: opcode = IR::ValueOpcode::SMedTri32; break; - case Decoder::Opcode::V_MIN3_U32: opcode = IR::ValueOpcode::UMinTri32; break; - case Decoder::Opcode::V_MAX3_U32: opcode = IR::ValueOpcode::UMaxTri32; break; - case Decoder::Opcode::V_MED3_U32: opcode = IR::ValueOpcode::UMedTri32; break; - case Decoder::Opcode::S_AND_B32: - case Decoder::Opcode::V_AND_B32: opcode = IR::ValueOpcode::BitwiseAnd32; break; - case Decoder::Opcode::S_OR_B32: - case Decoder::Opcode::V_OR_B32: opcode = IR::ValueOpcode::BitwiseOr32; break; - case Decoder::Opcode::S_XOR_B32: - case Decoder::Opcode::V_XOR_B32: opcode = IR::ValueOpcode::BitwiseXor32; break; - case Decoder::Opcode::S_NOT_B32: - case Decoder::Opcode::V_NOT_B32: opcode = IR::ValueOpcode::BitwiseNot32; break; - case Decoder::Opcode::S_BREV_B32: - case Decoder::Opcode::V_BFREV_B32: opcode = IR::ValueOpcode::BitReverse32; break; - case Decoder::Opcode::S_BCNT1_I32_B32: opcode = IR::ValueOpcode::BitCount32; break; - case Decoder::Opcode::S_BCNT1_I32_B64: - opcode = IR::ValueOpcode::BitCount64; - type = IR::Type::U64; - break; - case Decoder::Opcode::S_FF1_I32_B32: - case Decoder::Opcode::V_FFBL_B32: opcode = IR::ValueOpcode::FindILsb32; break; - case Decoder::Opcode::S_LSHL_B32: - case Decoder::Opcode::V_LSHL_B32: - case Decoder::Opcode::V_LSHLREV_B32: opcode = IR::ValueOpcode::ShiftLeftLogical32; break; - case Decoder::Opcode::S_LSHR_B32: - case Decoder::Opcode::V_LSHR_B32: - case Decoder::Opcode::V_LSHRREV_B32: opcode = IR::ValueOpcode::ShiftRightLogical32; break; - case Decoder::Opcode::S_ASHR_I32: - case Decoder::Opcode::V_ASHR_I32: - case Decoder::Opcode::V_ASHRREV_I32: - opcode = IR::ValueOpcode::ShiftRightArithmetic32; - break; - case Decoder::Opcode::S_LSHL_B64: - case Decoder::Opcode::V_LSHLREV_B64: - opcode = IR::ValueOpcode::ShiftLeftLogical64; - type = IR::Type::U64; - break; - case Decoder::Opcode::S_LSHR_B64: - case Decoder::Opcode::V_LSHRREV_B64: - opcode = IR::ValueOpcode::ShiftRightLogical64; - type = IR::Type::U64; - break; - default: return false; +bool Translator::PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool arithmetic) { + const auto translate_lane = [&](bool high) { + const auto count = + ir.BitwiseAnd(ReadU16LaneAsU32(inst.src0, high, false), IR::U32(IR::Value(15u))); + const auto value = ReadU16LaneAsU32(inst.src1, high, arithmetic); + return IR::U32(ir.Emit(opcode, {value, count})); + }; + WriteOperand(DestinationOperand(inst), + PackU16Lanes(translate_lane(false), translate_lane(true))); + return true; +} + +bool Translator::PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { + const auto translate_lane = [&](bool high) { + const auto lhs = ReadU16LaneAsU32(inst.src0, high, false); + const auto rhs = ReadU16LaneAsU32(inst.src1, high, false); + return IR::U32(ir.Emit(opcode, {lhs, rhs})); + }; + WriteOperand(DestinationOperand(inst), + PackU16Lanes(translate_lane(false), translate_lane(true))); + return true; +} + +bool Translator::PackedInteger16Mad(const Decoder::Instruction& inst, bool sign) { + const auto translate_lane = [&](bool high) { + const auto lhs = ReadU16LaneAsU32(inst.src0, high, false); + const auto rhs = ReadU16LaneAsU32(inst.src1, high, false); + return ir.IAdd(ir.IMul(lhs, rhs), ReadU16LaneAsU32(inst.src2, high, sign)); + }; + WriteOperand(DestinationOperand(inst), + PackU16Lanes(translate_lane(false), translate_lane(true))); + return true; +} + +bool Translator::PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool sign) { + const auto translate_lane = [&](bool high) { + const auto lhs = ReadU16LaneAsU32(inst.src0, high, sign); + const auto rhs = ReadU16LaneAsU32(inst.src1, high, sign); + return IR::U32(ir.Emit(opcode, {lhs, rhs})); + }; + WriteOperand(DestinationOperand(inst), + PackU16Lanes(translate_lane(false), translate_lane(true))); + return true; +} + +IR::U1 Translator::U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool negate_rhs, bool negate_result) { + const auto lhs = ReadMask(inst.src0); + auto rhs = ReadMask(inst.src1); + if (negate_rhs) { + rhs = ir.LogicalNot(rhs); + } + auto result = IR::U1(ir.Emit(opcode, {lhs, rhs})); + return negate_result ? ir.LogicalNot(result) : result; +} + +bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode, + IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, + bool unary) { + const auto invocation_result = + unary ? ir.LogicalNot(ReadMask(inst.src0)) + : U64MaskBinary(inst, logical_opcode, negate_rhs, negate_result); + const auto is_exec_or_vcc = [](const Decoder::Operand& operand) { + switch (operand.kind) { + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: return true; + default: return false; + } + }; + if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) || + (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) { + WriteMask64(inst.dst, invocation_result); + ir.SetScc(invocation_result); + return true; } - const bool reverse = inst.opcode == Decoder::Opcode::V_SUBREV_NC_U32 || - inst.opcode == Decoder::Opcode::V_LSHLREV_B32 || - inst.opcode == Decoder::Opcode::V_LSHRREV_B32 || - inst.opcode == Decoder::Opcode::V_ASHRREV_I32 || - inst.opcode == Decoder::Opcode::V_LSHLREV_B64 || - inst.opcode == Decoder::Opcode::V_LSHRREV_B64; + const auto lhs = ReadU32Pair(inst.src0); + auto mask_valid = ReadMaskValid(inst.src0); + if (inst.src_count > 1u) { + mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src1)); + } + std::array result; + if (unary) { + result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])}; + } else { + const auto rhs = ReadU32Pair(inst.src1); + for (uint32_t component = 0; component < 2u; component++) { + auto value = ir.Emit( + bit_opcode, {lhs[component], negate_rhs ? IR::Value(ir.BitwiseNot(rhs[component])) + : IR::Value(rhs[component])}); + result[component] = negate_result ? ir.BitwiseNot(IR::U32(value)) : IR::U32(value); + } + } + WriteU32Pair(inst.dst, result); + if (inst.dst.kind == Decoder::OperandKind::Sgpr) { + const auto dst = static_cast(inst.dst.reg); + ir.SetThreadBitScalarReg(dst, invocation_result); + ir.SetScalarMaskTag(dst, mask_valid); + const auto raw_nonzero = + ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))); + ir.SetScc(IR::U1( + ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero}))); + } else { + ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)))); + } + return true; +} + +bool Translator::SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + IR::Type type, bool reverse, bool mask_shift_count, + bool update_scc) { std::array args; for (uint32_t index = 0; index < inst.src_count; index++) { const auto arg_type = IR::ArgTypeOf(opcode, index); const auto operand = SourceAt(inst, reverse && index < 2u ? 1u - index : index); args[index] = ReadOperand(operand, arg_type == IR::Type::Void ? type : arg_type); - if (index == 1u && (opcode == IR::ValueOpcode::ShiftLeftLogical32 || - opcode == IR::ValueOpcode::ShiftRightLogical32 || - opcode == IR::ValueOpcode::ShiftRightArithmetic32)) { + if (mask_shift_count && index == 1u) { args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(31u))); } } @@ -241,398 +162,398 @@ bool Translator::TranslateSimpleInteger(const Decoder::Instruction& inst) { default: EXIT("invalid simple integer source count: %u", inst.src_count); } WriteOperand(DestinationOperand(inst), result); - switch (inst.opcode) { - case Decoder::Opcode::S_ABS_I32: - case Decoder::Opcode::S_BCNT1_I32_B32: - case Decoder::Opcode::S_BCNT1_I32_B64: - case Decoder::Opcode::S_AND_B32: - case Decoder::Opcode::S_OR_B32: - case Decoder::Opcode::S_XOR_B32: - case Decoder::Opcode::S_NOT_B32: - case Decoder::Opcode::S_LSHL_B32: - case Decoder::Opcode::S_LSHR_B32: - case Decoder::Opcode::S_ASHR_I32: - ir.SetScc(ir.INotEqual(IR::U32(result), IR::U32(IR::Value(0u)))); - break; - case Decoder::Opcode::S_LSHL_B64: - case Decoder::Opcode::S_LSHR_B64: + if (update_scc) { + if (IR::TypeOf(opcode) == IR::Type::U64) { ir.SetScc( IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); - break; - default: break; + } else { + ir.SetScc(ir.INotEqual(IR::U32(result), IR::U32(IR::Value(0u)))); + } } return true; } -bool Translator::TranslateComposedInteger(const Decoder::Instruction& inst) { - const auto binary_u32 = [&](auto operation) { - const auto lhs = ReadU32(inst.src0); - const auto rhs = ReadU32(inst.src1); - return operation(lhs, rhs); - }; - - IR::Value result; - switch (inst.opcode) { - case Decoder::Opcode::S_ANDN2_B32: - result = binary_u32( - [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseAnd(lhs, ir.BitwiseNot(rhs)); }); - break; - case Decoder::Opcode::S_ORN2_B32: - result = binary_u32( - [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseOr(lhs, ir.BitwiseNot(rhs)); }); - break; - case Decoder::Opcode::S_NAND_B32: - result = binary_u32( - [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseAnd(lhs, rhs)); }); - break; - case Decoder::Opcode::S_NOR_B32: - result = binary_u32( - [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseOr(lhs, rhs)); }); - break; - case Decoder::Opcode::S_XNOR_B32: - case Decoder::Opcode::V_XNOR_B32: - result = binary_u32( - [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); }); - break; - case Decoder::Opcode::V_AND_OR_B32: { - const auto lhs = ReadU32(inst.src0); - const auto rhs = ReadU32(inst.src1); - const auto add = ReadU32(inst.src2); - result = ir.BitwiseOr(ir.BitwiseAnd(lhs, rhs), add); - break; - } - case Decoder::Opcode::V_OR3_B32: - result = ir.BitwiseOr(ir.BitwiseOr(ReadU32(inst.src0), ReadU32(inst.src1)), - ReadU32(inst.src2)); - break; - case Decoder::Opcode::V_XOR3_B32: - result = ir.BitwiseXor(ir.BitwiseXor(ReadU32(inst.src0), ReadU32(inst.src1)), - ReadU32(inst.src2)); - break; - case Decoder::Opcode::S_FF1_I32_B64: { - const auto source = ExtractU64(ReadU64(inst.src0)); - const auto low_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]})); - const auto high_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]})); - const auto high_position = ir.IAdd(high_lsb, IR::U32(IR::Value(32u))); - result = ir.Select(ir.INotEqual(source[0], IR::U32(IR::Value(0u))), low_lsb, - ir.Select(ir.INotEqual(source[1], IR::U32(IR::Value(0u))), - high_position, IR::U32(IR::Value(0xffffffffu)))); - break; - } - case Decoder::Opcode::V_FFBH_U32: { - const auto source = ReadU32(inst.src0); - const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {source})); - const auto position = ir.ISub(IR::U32(IR::Value(31u)), msb); - result = ir.Select(ir.INotEqual(source, IR::U32(IR::Value(0u))), position, - IR::U32(IR::Value(0xffffffffu))); - break; - } - case Decoder::Opcode::S_FLBIT_I32_B64: { - const auto source = ReadU64(inst.src0); - const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb64, {source})); - const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb); - const auto nonzero = IR::U1( - ir.Emit(IR::ValueOpcode::INotEqual64, - {source, ir.ConstructU64(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))})); - result = ir.Select(nonzero, position, IR::U32(IR::Value(0xffffffffu))); - break; - } - default: return false; +bool Translator::ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool negate_rhs, bool negate_result, bool update_scc) { + const auto lhs = ReadU32(inst.src0); + auto rhs = ReadU32(inst.src1); + if (negate_rhs) { + rhs = ir.BitwiseNot(rhs); + } + auto result = IR::U32(ir.Emit(opcode, {lhs, rhs})); + if (negate_result) { + result = ir.BitwiseNot(result); } WriteOperand(DestinationOperand(inst), result); - switch (inst.opcode) { - case Decoder::Opcode::S_ANDN2_B32: - case Decoder::Opcode::S_ORN2_B32: - case Decoder::Opcode::S_NAND_B32: - case Decoder::Opcode::S_NOR_B32: - case Decoder::Opcode::S_XNOR_B32: - ir.SetScc(ir.INotEqual(IR::U32(result), IR::U32(IR::Value(0u)))); - break; - default: break; + if (update_scc) { + ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u)))); } return true; } -bool Translator::TranslateExtendedInteger(const Decoder::Instruction& inst) { - const auto imm = [](uint32_t value) { return IR::U32(IR::Value(value)); }; - const auto mask = [&](IR::U32 value, uint32_t bits) { return ir.BitwiseAnd(value, imm(bits)); }; - const auto extract = [&](IR::U32 value, IR::U32 offset, IR::U32 width, bool sign) { +bool Translator::V_AND_OR_B32(const Decoder::Instruction& inst) { + const auto result = + ir.BitwiseOr(ir.BitwiseAnd(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2)); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::V_OR3_B32(const Decoder::Instruction& inst) { + const auto result = + ir.BitwiseOr(ir.BitwiseOr(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2)); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::V_XOR3_B32(const Decoder::Instruction& inst) { + const auto result = + ir.BitwiseXor(ir.BitwiseXor(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2)); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::S_FF1_I32_B64(const Decoder::Instruction& inst) { + const auto source = ExtractU64(ReadU64(inst.src0)); + const auto low_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]})); + const auto high_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]})); + const auto high_position = ir.IAdd(high_lsb, IR::U32(IR::Value(32u))); + const auto result = ir.Select(ir.INotEqual(source[0], IR::U32(IR::Value(0u))), low_lsb, + ir.Select(ir.INotEqual(source[1], IR::U32(IR::Value(0u))), + high_position, IR::U32(IR::Value(0xffffffffu)))); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::V_FFBH_U32(const Decoder::Instruction& inst) { + const auto source = ReadU32(inst.src0); + const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {source})); + const auto position = ir.ISub(IR::U32(IR::Value(31u)), msb); + const auto result = ir.Select(ir.INotEqual(source, IR::U32(IR::Value(0u))), position, + IR::U32(IR::Value(0xffffffffu))); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::S_FLBIT_I32_B64(const Decoder::Instruction& inst) { + const auto source = ReadU64(inst.src0); + const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb64, {source})); + const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb); + const auto nonzero = + IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, + {source, ir.ConstructU64(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))})); + const auto result = ir.Select(nonzero, position, IR::U32(IR::Value(0xffffffffu))); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool addend) { + const auto extract24 = [&](IR::U32 value) { return IR::U32( ir.Emit(sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract, - {value, offset, width})); + {value, IR::Value(0u), IR::Value(24u)})); }; - const auto right_mask32 = [&](IR::U32 count) { - return IR::U32( - ir.Emit(IR::ValueOpcode::BitFieldInsert, {imm(0), imm(0xffffffffu), imm(0), count})); - }; - const auto right_mask64 = [&](IR::U32 count) { - const auto below32 = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {count, imm(32)})); - const auto above32 = IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThan32, {count, imm(32)})); - const auto low_count = ir.Select(below32, count, imm(32)); - const auto high_count = ir.Select(above32, ir.ISub(count, imm(32)), imm(0)); - return ir.ConstructU64(right_mask32(low_count), right_mask32(high_count)); - }; - - IR::Value result; - switch (inst.opcode) { - case Decoder::Opcode::V_MAD_I32_I24: - case Decoder::Opcode::V_MAD_U32_U24: - case Decoder::Opcode::V_MUL_I32_I24: - case Decoder::Opcode::V_MUL_U32_U24: { - const bool sign = inst.opcode == Decoder::Opcode::V_MAD_I32_I24 || - inst.opcode == Decoder::Opcode::V_MUL_I32_I24; - const auto lhs = extract(ReadU32(inst.src0), imm(0), imm(24), sign); - const auto rhs = extract(ReadU32(inst.src1), imm(0), imm(24), sign); - auto value = ir.IMul(lhs, rhs); - if (inst.src_count == 3) { - value = ir.IAdd(value, ReadU32(inst.src2)); - } - result = value; - break; - } - case Decoder::Opcode::V_MAD_U64_U32: { - const auto lhs = ReadU32(inst.src0); - const auto rhs = ReadU32(inst.src1); - const auto add = ExtractU64(ReadU64(inst.src2)); - const auto mul_low = ir.IMul(lhs, rhs); - const auto mul_high = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})); - const auto low = ir.IAdd(mul_low, add[0]); - const auto carry_low = ir.ULessThan(low, mul_low); - const auto high0 = ir.IAdd(mul_high, add[1]); - const auto carry0 = ir.ULessThan(high0, mul_high); - const auto high = ir.IAdd(high0, ir.Select(carry_low, imm(1), imm(0))); - const auto carry1 = ir.ULessThan(high, high0); - WriteOperand(DestinationOperand(inst), ir.ConstructU64(low, high)); - if (inst.dst2.kind != Decoder::OperandKind::Null && - inst.dst2.kind != Decoder::OperandKind::Unknown) { - WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1)); - } - return true; - } - case Decoder::Opcode::V_SAD_U32: { - const auto lhs = ReadU32(inst.src0); - const auto rhs = ReadU32(inst.src1); - const auto lo = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs})); - const auto hi = IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs})); - result = ir.IAdd(ir.ISub(hi, lo), ReadU32(inst.src2)); - break; - } - case Decoder::Opcode::V_ADD3_U32: - result = ir.IAdd(ir.IAdd(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2)); - break; - case Decoder::Opcode::S_BITSET0_B32: - case Decoder::Opcode::S_BITSET1_B32: { - const auto bit = ir.ShiftLeftLogical(imm(1), mask(ReadU32(inst.src0), 31u)); - result = inst.opcode == Decoder::Opcode::S_BITSET0_B32 - ? IR::Value(ir.BitwiseAnd(ReadU32(inst.dst), ir.BitwiseNot(bit))) - : IR::Value(ir.BitwiseOr(ReadU32(inst.dst), bit)); - break; - } - case Decoder::Opcode::V_BCNT_U32_B32: - result = ir.IAdd(IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {ReadU32(inst.src0)})), - ReadU32(inst.src1)); - break; - case Decoder::Opcode::V_MBCNT_LO_U32_B32: - case Decoder::Opcode::V_MBCNT_HI_U32_B32: { - const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); - const auto local = mask(lane, 31u); - const auto below = ir.ISub(ir.ShiftLeftLogical(imm(1), local), imm(1)); - const auto high_lane = - IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, imm(32)})); - const auto thread_mask = inst.opcode == Decoder::Opcode::V_MBCNT_LO_U32_B32 - ? ir.Select(high_lane, imm(0xffffffffu), below) - : ir.Select(high_lane, below, imm(0)); - const auto active = ir.BitwiseAnd(ReadU32(inst.src0), thread_mask); - result = ir.IAdd(IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {active})), - ReadU32(inst.src1)); - break; - } - case Decoder::Opcode::S_BITREPLICATE_B64_B32: { - const auto replicate = [&](IR::U32 value) { - auto bits = ir.BitwiseOr(value, ir.ShiftLeftLogical(value, imm(8))); - bits = ir.BitwiseAnd(bits, imm(0x00ff00ffu)); - bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(4))); - bits = ir.BitwiseAnd(bits, imm(0x0f0f0f0fu)); - bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(2))); - bits = ir.BitwiseAnd(bits, imm(0x33333333u)); - bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(1))); - bits = ir.BitwiseAnd(bits, imm(0x55555555u)); - return ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(1))); - }; - const auto source = ReadU32(inst.src0); - result = ir.ConstructU64(replicate(mask(source, 0xffffu)), - replicate(ir.ShiftRightLogical(source, imm(16)))); - break; - } - case Decoder::Opcode::S_QUADMASK_B64: { - const auto compact = [&](IR::U32 value) { - auto bits = ir.BitwiseOr(value, ir.ShiftRightLogical(value, imm(1))); - bits = ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(2))); - bits = mask(bits, 0x11111111u); - bits = mask(ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(3))), 0x03030303u); - bits = mask(ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(6))), 0x000f000fu); - return mask(ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(12))), 0xffu); - }; - const auto source = ReadU32Pair(inst.src0); - const auto quads = - ir.BitwiseOr(compact(source[0]), ir.ShiftLeftLogical(compact(source[1]), imm(8))); - result = ir.ConstructU64(quads, imm(0)); - break; - } - case Decoder::Opcode::S_BFM_B32: - case Decoder::Opcode::V_BFM_B32: { - const auto count = mask(ReadU32(inst.src0), 31u); - const auto offset = mask(ReadU32(inst.src1), 31u); - result = - ir.Emit(IR::ValueOpcode::BitFieldInsert, {imm(0), imm(0xffffffffu), offset, count}); - break; - } - case Decoder::Opcode::S_BFM_B64: { - const auto count = mask(ReadU32(inst.src0), 63u); - const auto offset = mask(ReadU32(inst.src1), 63u); - result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {right_mask64(count), offset}); - break; - } - case Decoder::Opcode::S_BFE_U32: - case Decoder::Opcode::S_BFE_I32: { - const auto source = ReadU32(inst.src0); - const auto field = ReadU32(inst.src1); - const auto offset = extract(field, imm(0), imm(5), false); - const auto count = - IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {extract(field, imm(16), imm(7), false), - ir.ISub(imm(32), offset)})); - const auto opcode = inst.opcode == Decoder::Opcode::S_BFE_I32 - ? IR::ValueOpcode::BitFieldSExtract - : IR::ValueOpcode::BitFieldUExtract; - result = ir.Emit(opcode, {source, offset, count}); - break; - } - case Decoder::Opcode::S_BFE_U64: { - const auto source = ReadU64(inst.src0); - const auto field = ReadU32(inst.src1); - const auto offset = extract(field, imm(0), imm(6), false); - const auto raw_count = extract(field, imm(16), imm(7), false); - const auto available = ir.ISub(imm(64), offset); - const auto count = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {raw_count, available})); - const auto shifted = - IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {source, offset})); - result = ir.Emit(IR::ValueOpcode::BitwiseAnd64, {shifted, right_mask64(count)}); - break; - } - case Decoder::Opcode::V_BFE_U32: - case Decoder::Opcode::V_BFE_I32: { - const auto source = ReadU32(inst.src0); - const auto offset = mask(ReadU32(inst.src1), 31u); - const auto count = - IR::U32(ir.Emit(IR::ValueOpcode::UMin32, - {mask(ReadU32(inst.src2), 31u), ir.ISub(imm(32), offset)})); - const auto opcode = inst.opcode == Decoder::Opcode::V_BFE_I32 - ? IR::ValueOpcode::BitFieldSExtract - : IR::ValueOpcode::BitFieldUExtract; - result = ir.Emit(opcode, {source, offset, count}); - break; - } - case Decoder::Opcode::V_BFI_B32: { - const auto bits = ReadU32(inst.src0); - const auto insert = ReadU32(inst.src1); - const auto base = ReadU32(inst.src2); - result = - ir.BitwiseOr(ir.BitwiseAnd(bits, insert), ir.BitwiseAnd(ir.BitwiseNot(bits), base)); - break; - } - case Decoder::Opcode::S_BITCMP0_B32: - case Decoder::Opcode::S_BITCMP1_B32: { - const auto value = ReadU32(inst.src0); - const auto bit = extract(value, mask(ReadU32(inst.src1), 31u), imm(1), false); - const auto expected = imm(inst.opcode == Decoder::Opcode::S_BITCMP1_B32 ? 1u : 0u); - WriteCompareResult(inst.dst, ir.IEqual(bit, expected)); - return true; - } - case Decoder::Opcode::V_ALIGNBIT_B32: { - const auto hi = ReadU32(inst.src0); - const auto lo = ReadU32(inst.src1); - const auto shift = mask(ReadU32(inst.src2), 31u); - const auto lo_part = ir.ShiftRightLogical(lo, shift); - const auto hi_part_raw = ir.ShiftLeftLogical(hi, mask(ir.ISub(imm(32), shift), 31u)); - const auto hi_part = ir.Select(ir.INotEqual(shift, imm(0)), hi_part_raw, imm(0)); - result = ir.BitwiseOr(lo_part, hi_part); - break; - } - case Decoder::Opcode::V_ALIGNBYTE_B32: { - const auto hi = ReadU32(inst.src0); - const auto lo = ReadU32(inst.src1); - const auto byte_offset = mask(ReadU32(inst.src2), 31u); - const auto bit_offset = ir.ShiftLeftLogical(byte_offset, imm(3)); - const auto concatenated = ir.ConstructU64(lo, hi); - const auto shifted = IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, - {concatenated, mask(bit_offset, 63u)})); - const auto in_range = - IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {byte_offset, imm(8)})); - result = ir.Select(in_range, ExtractU64(shifted)[0], imm(0)); - break; - } - case Decoder::Opcode::V_LSHL_ADD_U32: - result = ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src0), mask(ReadU32(inst.src1), 31u)), - ReadU32(inst.src2)); - break; - case Decoder::Opcode::V_ADD_LSHL_U32: - result = ir.ShiftLeftLogical(ir.IAdd(ReadU32(inst.src0), ReadU32(inst.src1)), - mask(ReadU32(inst.src2), 31u)); - break; - case Decoder::Opcode::V_XAD_U32: - result = - ir.IAdd(ir.BitwiseXor(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2)); - break; - case Decoder::Opcode::V_LSHL_OR_B32: - result = - ir.BitwiseOr(ir.ShiftLeftLogical(ReadU32(inst.src0), mask(ReadU32(inst.src1), 31u)), - ReadU32(inst.src2)); - break; - case Decoder::Opcode::V_CNDMASK_B32: { - Decoder::Operand mask_operand; - mask_operand.kind = Decoder::OperandKind::VccLo; - if (inst.src_count >= 3u) { - mask_operand = inst.src2; - } - const auto condition = ReadMask(mask_operand); - if (inst.src0.negate || inst.src0.absolute || inst.src1.negate || inst.src1.absolute) { - result = ir.Emit(IR::ValueOpcode::SelectF32, - {condition, ReadOperand(inst.src1, IR::Type::F32), - ReadOperand(inst.src0, IR::Type::F32)}); - } else { - result = ir.Select(condition, ReadU32(inst.src1), ReadU32(inst.src0)); - } - break; - } - case Decoder::Opcode::S_PACK_LL_B32_B16: - case Decoder::Opcode::S_PACK_LH_B32_B16: - case Decoder::Opcode::S_PACK_HH_B32_B16: - case Decoder::Opcode::V_CVT_PK_U16_U32: - case Decoder::Opcode::V_CVT_PK_I16_I32: { - const bool high0 = inst.opcode == Decoder::Opcode::S_PACK_HH_B32_B16; - const bool high1 = inst.opcode == Decoder::Opcode::S_PACK_LH_B32_B16 || high0; - const auto lo = - high0 ? ir.ShiftRightLogical(ReadU32(inst.src0), imm(16)) : ReadU32(inst.src0); - const auto hi = - high1 ? ir.ShiftRightLogical(ReadU32(inst.src1), imm(16)) : ReadU32(inst.src1); - result = - ir.BitwiseOr(mask(lo, 0xffffu), ir.ShiftLeftLogical(mask(hi, 0xffffu), imm(16))); - break; - } - default: return false; + const auto lhs = extract24(ReadU32(inst.src0)); + const auto rhs = extract24(ReadU32(inst.src1)); + auto result = ir.IMul(lhs, rhs); + if (addend) { + result = ir.IAdd(result, ReadU32(inst.src2)); } WriteOperand(DestinationOperand(inst), result); - switch (inst.opcode) { - case Decoder::Opcode::S_BFE_U32: - case Decoder::Opcode::S_BFE_I32: ir.SetScc(ir.INotEqual(IR::U32(result), imm(0u))); break; - case Decoder::Opcode::S_BFE_U64: - case Decoder::Opcode::S_QUADMASK_B64: - ir.SetScc( - IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); - break; - default: break; + return true; +} + +bool Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) { + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + const auto add = ExtractU64(ReadU64(inst.src2)); + const auto mul_low = ir.IMul(lhs, rhs); + const auto mul_high = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})); + const auto low = ir.IAdd(mul_low, add[0]); + const auto carry_low = ir.ULessThan(low, mul_low); + const auto high0 = ir.IAdd(mul_high, add[1]); + const auto carry0 = ir.ULessThan(high0, mul_high); + const auto high = + ir.IAdd(high0, ir.Select(carry_low, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)))); + const auto carry1 = ir.ULessThan(high, high0); + WriteOperand(DestinationOperand(inst), ir.ConstructU64(low, high)); + if (inst.dst2.kind != Decoder::OperandKind::Null && + inst.dst2.kind != Decoder::OperandKind::Unknown) { + WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1)); } return true; } +bool Translator::V_SAD_U32(const Decoder::Instruction& inst) { + const auto lhs = ReadU32(inst.src0); + const auto rhs = ReadU32(inst.src1); + const auto lo = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs})); + const auto hi = IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs})); + const auto result = ir.IAdd(ir.ISub(hi, lo), ReadU32(inst.src2)); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::V_ADD3_U32(const Decoder::Instruction& inst) { + const auto result = + ir.IAdd(ir.IAdd(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2)); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::S_BITSET_B32(const Decoder::Instruction& inst, bool set) { + const auto offset = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(31u))); + const auto bit = ir.ShiftLeftLogical(IR::U32(IR::Value(1u)), offset); + const auto old = ReadU32(inst.dst); + const auto result = set ? ir.BitwiseOr(old, bit) : ir.BitwiseAnd(old, ir.BitwiseNot(bit)); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::V_BCNT_U32_B32(const Decoder::Instruction& inst) { + const auto count = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {ReadU32(inst.src0)})); + WriteOperand(DestinationOperand(inst), ir.IAdd(count, ReadU32(inst.src1))); + return true; +} + +bool Translator::V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low) { + const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); + const auto local = ir.BitwiseAnd(lane, IR::U32(IR::Value(31u))); + const auto below = + ir.ISub(ir.ShiftLeftLogical(IR::U32(IR::Value(1u)), local), IR::U32(IR::Value(1u))); + const auto high_lane = + IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, IR::Value(32u)})); + const auto thread_mask = low ? ir.Select(high_lane, IR::U32(IR::Value(0xffffffffu)), below) + : ir.Select(high_lane, below, IR::U32(IR::Value(0u))); + const auto active = ir.BitwiseAnd(ReadU32(inst.src0), thread_mask); + const auto count = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {active})); + WriteOperand(DestinationOperand(inst), ir.IAdd(count, ReadU32(inst.src1))); + return true; +} + +bool Translator::S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst) { + const auto replicate = [&](IR::U32 value) { + auto bits = ir.BitwiseOr(value, ir.ShiftLeftLogical(value, IR::U32(IR::Value(8u)))); + bits = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x00ff00ffu))); + bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, IR::U32(IR::Value(4u)))); + bits = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x0f0f0f0fu))); + bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, IR::U32(IR::Value(2u)))); + bits = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x33333333u))); + bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, IR::U32(IR::Value(1u)))); + bits = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x55555555u))); + return ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, IR::U32(IR::Value(1u)))); + }; + const auto source = ReadU32(inst.src0); + const auto low = ir.BitwiseAnd(source, IR::U32(IR::Value(0xffffu))); + const auto high = ir.ShiftRightLogical(source, IR::U32(IR::Value(16u))); + WriteOperand(DestinationOperand(inst), ir.ConstructU64(replicate(low), replicate(high))); + return true; +} + +bool Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) { + const auto compact = [&](IR::U32 value) { + auto bits = ir.BitwiseOr(value, ir.ShiftRightLogical(value, IR::U32(IR::Value(1u)))); + bits = ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, IR::U32(IR::Value(2u)))); + bits = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x11111111u))); + bits = ir.BitwiseAnd(ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, IR::U32(IR::Value(3u)))), + IR::U32(IR::Value(0x03030303u))); + bits = ir.BitwiseAnd(ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, IR::U32(IR::Value(6u)))), + IR::U32(IR::Value(0x000f000fu))); + return ir.BitwiseAnd( + ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, IR::U32(IR::Value(12u)))), + IR::U32(IR::Value(0xffu))); + }; + const auto source = ReadU32Pair(inst.src0); + const auto quads = ir.BitwiseOr( + compact(source[0]), ir.ShiftLeftLogical(compact(source[1]), IR::U32(IR::Value(8u)))); + const auto result = ir.ConstructU64(quads, IR::U32(IR::Value(0u))); + WriteOperand(DestinationOperand(inst), result); + ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); + return true; +} + +bool Translator::BFM_B32(const Decoder::Instruction& inst) { + const auto count = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(31u))); + const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u))); + const auto result = ir.Emit(IR::ValueOpcode::BitFieldInsert, + {IR::Value(0u), IR::Value(0xffffffffu), offset, count}); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +IR::U32 Translator::RightMask32(IR::U32 count) { + return IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert, + {IR::Value(0u), IR::Value(0xffffffffu), IR::Value(0u), count})); +} + +IR::U64 Translator::RightMask64(IR::U32 count) { + const auto below32 = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {count, IR::Value(32u)})); + const auto above32 = IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThan32, {count, IR::Value(32u)})); + const auto low_count = ir.Select(below32, count, IR::U32(IR::Value(32u))); + const auto high_count = + ir.Select(above32, ir.ISub(count, IR::U32(IR::Value(32u))), IR::U32(IR::Value(0u))); + return ir.ConstructU64(RightMask32(low_count), RightMask32(high_count)); +} + +bool Translator::S_BFM_B64(const Decoder::Instruction& inst) { + const auto count = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(63u))); + const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(63u))); + const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {RightMask64(count), offset}); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::S_BFE_U32(const Decoder::Instruction& inst, bool sign) { + const auto source = ReadU32(inst.src0); + const auto field = ReadU32(inst.src1); + const auto offset = + IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {field, IR::Value(0u), IR::Value(5u)})); + const auto raw_count = + IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {field, IR::Value(16u), IR::Value(7u)})); + const auto count = IR::U32( + ir.Emit(IR::ValueOpcode::UMin32, {raw_count, ir.ISub(IR::U32(IR::Value(32u)), offset)})); + const auto opcode = + sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract; + const auto result = IR::U32(ir.Emit(opcode, {source, offset, count})); + WriteOperand(DestinationOperand(inst), result); + ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u)))); + return true; +} + +bool Translator::S_BFE_U64(const Decoder::Instruction& inst) { + const auto source = ReadU64(inst.src0); + const auto field = ReadU32(inst.src1); + const auto offset = + IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {field, IR::Value(0u), IR::Value(6u)})); + const auto raw_count = + IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {field, IR::Value(16u), IR::Value(7u)})); + const auto available = ir.ISub(IR::U32(IR::Value(64u)), offset); + const auto count = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {raw_count, available})); + const auto shifted = IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {source, offset})); + const auto result = ir.Emit(IR::ValueOpcode::BitwiseAnd64, {shifted, RightMask64(count)}); + WriteOperand(DestinationOperand(inst), result); + ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); + return true; +} + +bool Translator::V_BFE_U32(const Decoder::Instruction& inst, bool sign) { + const auto source = ReadU32(inst.src0); + const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u))); + const auto raw_count = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u))); + const auto count = IR::U32( + ir.Emit(IR::ValueOpcode::UMin32, {raw_count, ir.ISub(IR::U32(IR::Value(32u)), offset)})); + const auto opcode = + sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract; + WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {source, offset, count})); + return true; +} + +bool Translator::V_BFI_B32(const Decoder::Instruction& inst) { + const auto bits = ReadU32(inst.src0); + const auto insert = ReadU32(inst.src1); + const auto base = ReadU32(inst.src2); + const auto result = + ir.BitwiseOr(ir.BitwiseAnd(bits, insert), ir.BitwiseAnd(ir.BitwiseNot(bits), base)); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::S_BITCMP_B32(const Decoder::Instruction& inst, bool expected) { + const auto value = ReadU32(inst.src0); + const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u))); + const auto bit = + IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {value, offset, IR::Value(1u)})); + WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u)))); + return true; +} + +bool Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) { + const auto hi = ReadU32(inst.src0); + const auto lo = ReadU32(inst.src1); + const auto shift = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u))); + const auto lo_part = ir.ShiftRightLogical(lo, shift); + const auto inverse = + ir.BitwiseAnd(ir.ISub(IR::U32(IR::Value(32u)), shift), IR::U32(IR::Value(31u))); + const auto hi_part_raw = ir.ShiftLeftLogical(hi, inverse); + const auto hi_part = + ir.Select(ir.INotEqual(shift, IR::U32(IR::Value(0u))), hi_part_raw, IR::U32(IR::Value(0u))); + WriteOperand(DestinationOperand(inst), ir.BitwiseOr(lo_part, hi_part)); + return true; +} + +bool Translator::V_ALIGNBYTE_B32(const Decoder::Instruction& inst) { + const auto hi = ReadU32(inst.src0); + const auto lo = ReadU32(inst.src1); + const auto byte_offset = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u))); + const auto bit_offset = ir.ShiftLeftLogical(byte_offset, IR::U32(IR::Value(3u))); + const auto concatenated = ir.ConstructU64(lo, hi); + const auto shifted = + IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, + {concatenated, ir.BitwiseAnd(bit_offset, IR::U32(IR::Value(63u)))})); + const auto in_range = + IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {byte_offset, IR::Value(8u)})); + WriteOperand(DestinationOperand(inst), + ir.Select(in_range, ExtractU64(shifted)[0], IR::U32(IR::Value(0u)))); + return true; +} + +bool Translator::V_LSHL_ADD_U32(const Decoder::Instruction& inst) { + const auto shift = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u))); + const auto result = ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src0), shift), ReadU32(inst.src2)); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::V_ADD_LSHL_U32(const Decoder::Instruction& inst) { + const auto shift = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u))); + const auto result = ir.ShiftLeftLogical(ir.IAdd(ReadU32(inst.src0), ReadU32(inst.src1)), shift); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::V_XAD_U32(const Decoder::Instruction& inst) { + const auto result = + ir.IAdd(ir.BitwiseXor(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2)); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::V_LSHL_OR_B32(const Decoder::Instruction& inst) { + const auto shift = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u))); + const auto result = + ir.BitwiseOr(ir.ShiftLeftLogical(ReadU32(inst.src0), shift), ReadU32(inst.src2)); + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::V_CNDMASK_B32(const Decoder::Instruction& inst) { + Decoder::Operand mask_operand; + mask_operand.kind = Decoder::OperandKind::VccLo; + if (inst.src_count >= 3u) { + mask_operand = inst.src2; + } + const auto condition = ReadMask(mask_operand); + IR::Value result; + if (inst.src0.negate || inst.src0.absolute || inst.src1.negate || inst.src1.absolute) { + result = + ir.Emit(IR::ValueOpcode::SelectF32, {condition, ReadOperand(inst.src1, IR::Type::F32), + ReadOperand(inst.src0, IR::Type::F32)}); + } else { + result = ir.Select(condition, ReadU32(inst.src1), ReadU32(inst.src0)); + } + WriteOperand(DestinationOperand(inst), result); + return true; +} + +bool Translator::PackB16(const Decoder::Instruction& inst, bool high0, bool high1) { + const auto lo = high0 ? ir.ShiftRightLogical(ReadU32(inst.src0), IR::U32(IR::Value(16u))) + : ReadU32(inst.src0); + const auto hi = high1 ? ir.ShiftRightLogical(ReadU32(inst.src1), IR::U32(IR::Value(16u))) + : ReadU32(inst.src1); + const auto low_bits = ir.BitwiseAnd(lo, IR::U32(IR::Value(0xffffu))); + const auto high_bits = ir.ShiftLeftLogical(ir.BitwiseAnd(hi, IR::U32(IR::Value(0xffffu))), + IR::U32(IR::Value(16u))); + WriteOperand(DestinationOperand(inst), ir.BitwiseOr(low_bits, high_bits)); + return true; +} + } // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 651c948f5..487202aed 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -63,17 +63,6 @@ uint32_t RawScalarLoadBase(const Decoder::Operand& operand) { return operand.kind == Decoder::OperandKind::VccLo ? 106u : 0u; } -bool TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code) { - switch (operand.kind) { - case Decoder::OperandKind::Sgpr: code = operand.reg; return true; - case Decoder::OperandKind::VccLo: code = 106u; return true; - case Decoder::OperandKind::VccHi: code = 107u; return true; - case Decoder::OperandKind::ExecLo: code = 126u; return true; - case Decoder::OperandKind::ExecHi: code = 127u; return true; - default: return false; - } -} - ResourceKind FlatSegmentResourceKind(uint32_t segment) { switch (segment) { case 1u: return ResourceKind::Scratch; @@ -196,136 +185,6 @@ bool IsScalarBufferLoad(Decoder::Opcode opcode) { } } -bool IsDsReturnAtomic(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::DS_ADD_RTN_U32: - case Decoder::Opcode::DS_SUB_RTN_U32: - case Decoder::Opcode::DS_MIN_RTN_I32: - case Decoder::Opcode::DS_MAX_RTN_I32: - case Decoder::Opcode::DS_MIN_RTN_U32: - case Decoder::Opcode::DS_MAX_RTN_U32: - case Decoder::Opcode::DS_AND_RTN_B32: - case Decoder::Opcode::DS_OR_RTN_B32: - case Decoder::Opcode::DS_XOR_RTN_B32: - case Decoder::Opcode::DS_WRXCHG_RTN_B32: return true; - default: return false; - } -} - -bool MemoryOpcodeMatchesFamily(Decoder::Opcode opcode, Decoder::Family family) { - switch (family) { - case Decoder::Family::SMEM: - return IsScalarAddressLoad(opcode) || IsScalarBufferLoad(opcode); - case Decoder::Family::MUBUF: - return (opcode >= Decoder::Opcode::BUFFER_LOAD_FORMAT_X && - opcode <= Decoder::Opcode::BUFFER_STORE_DWORDX4) || - (opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP && - opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX); - case Decoder::Family::MTBUF: - return opcode >= Decoder::Opcode::TBUFFER_LOAD_FORMAT_X && - opcode <= Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW; - case Decoder::Family::FLAT: - return opcode >= Decoder::Opcode::FLAT_LOAD_UBYTE && - opcode <= Decoder::Opcode::FLAT_STORE_DWORDX4; - case Decoder::Family::DS: - return opcode >= Decoder::Opcode::DS_ADD_U32 && - opcode <= Decoder::Opcode::DS_READ_ADDTID_B32; - case Decoder::Family::MIMG: - return opcode >= Decoder::Opcode::IMAGE_GET_RESINFO && - opcode <= Decoder::Opcode::IMAGE_GATHER4H; - default: return false; - } -} - -bool UsesDestinationAsMemorySource(Decoder::Opcode opcode) { - return (opcode >= Decoder::Opcode::BUFFER_STORE_FORMAT_X && - opcode <= Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW) || - (opcode >= Decoder::Opcode::BUFFER_STORE_BYTE && - opcode <= Decoder::Opcode::BUFFER_STORE_DWORDX4) || - (opcode >= Decoder::Opcode::TBUFFER_STORE_FORMAT_X && - opcode <= Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW) || - (opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP && - opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX) || - (opcode >= Decoder::Opcode::FLAT_STORE_BYTE && - opcode <= Decoder::Opcode::FLAT_STORE_DWORDX4) || - opcode == Decoder::Opcode::IMAGE_STORE || - opcode == Decoder::Opcode::IMAGE_STORE_MIP || - (opcode >= Decoder::Opcode::IMAGE_ATOMIC_ADD && - opcode <= Decoder::Opcode::IMAGE_ATOMIC_XOR); -} - -bool IsRegisterOperand(const Decoder::Operand& operand) { - switch (operand.kind) { - case Decoder::OperandKind::Sgpr: - case Decoder::OperandKind::Vgpr: - case Decoder::OperandKind::VccLo: - case Decoder::OperandKind::VccHi: - case Decoder::OperandKind::VccZ: - case Decoder::OperandKind::ExecLo: - case Decoder::OperandKind::ExecHi: - case Decoder::OperandKind::ExecZ: - case Decoder::OperandKind::Scc: - case Decoder::OperandKind::M0: return true; - default: return false; - } -} - -bool RequiresRegisterDestination(const Decoder::Instruction& inst) { - if (IsScalarAddressLoad(inst.opcode) || IsScalarBufferLoad(inst.opcode)) { - return true; - } - if (inst.family == Decoder::Family::MUBUF || inst.family == Decoder::Family::MTBUF) { - const bool load = (inst.opcode >= Decoder::Opcode::BUFFER_LOAD_FORMAT_X && - inst.opcode <= Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW) || - (inst.opcode >= Decoder::Opcode::BUFFER_LOAD_UBYTE && - inst.opcode <= Decoder::Opcode::BUFFER_LOAD_DWORDX4) || - (inst.opcode >= Decoder::Opcode::TBUFFER_LOAD_FORMAT_X && - inst.opcode <= Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZW); - const bool atomic = inst.opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP && - inst.opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX; - return load || (atomic && inst.glc); - } - if (inst.family == Decoder::Family::FLAT) { - return inst.opcode >= Decoder::Opcode::FLAT_LOAD_UBYTE && - inst.opcode <= Decoder::Opcode::FLAT_LOAD_DWORDX4; - } - if (inst.family == Decoder::Family::MIMG) { - const bool atomic = inst.opcode >= Decoder::Opcode::IMAGE_ATOMIC_ADD && - inst.opcode <= Decoder::Opcode::IMAGE_ATOMIC_XOR; - const bool load = inst.opcode == Decoder::Opcode::IMAGE_GET_RESINFO || - inst.opcode == Decoder::Opcode::IMAGE_GET_LOD || - inst.opcode == Decoder::Opcode::IMAGE_LOAD || - inst.opcode == Decoder::Opcode::IMAGE_LOAD_MIP || - inst.opcode == Decoder::Opcode::IMAGE_SAMPLE || - (inst.opcode >= Decoder::Opcode::IMAGE_GATHER4_LZ && - inst.opcode <= Decoder::Opcode::IMAGE_GATHER4H); - return load || (atomic && inst.glc); - } - if (inst.family == Decoder::Family::DS) { - switch (inst.opcode) { - case Decoder::Opcode::DS_READ2_B32: - case Decoder::Opcode::DS_READ2ST64_B32: - case Decoder::Opcode::DS_READ2_B64: - case Decoder::Opcode::DS_READ2ST64_B64: - case Decoder::Opcode::DS_READ_I8: - case Decoder::Opcode::DS_READ_U8: - case Decoder::Opcode::DS_READ_I16: - case Decoder::Opcode::DS_READ_U16: - case Decoder::Opcode::DS_READ_U16_D16: - case Decoder::Opcode::DS_READ_B32: - case Decoder::Opcode::DS_READ_B64: - case Decoder::Opcode::DS_READ_B96: - case Decoder::Opcode::DS_READ_B128: - case Decoder::Opcode::DS_SWIZZLE_B32: - case Decoder::Opcode::DS_CONSUME: - case Decoder::Opcode::DS_APPEND: - case Decoder::Opcode::DS_READ_ADDTID_B32: return true; - default: return IsDsReturnAtomic(inst.opcode); - } - } - return false; -} - Decoder::Operand MakeM0Operand() { Decoder::Operand operand; operand.kind = Decoder::OperandKind::M0; @@ -353,7 +212,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in decoded.opcode <= Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW) || (decoded.opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP && decoded.opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX); - uint32_t cursor = 0; + uint32_t cursor = 0; if (store_or_atomic) { if (index == cursor++) return decoded.dst; } @@ -533,18 +392,6 @@ void Translator::WriteImageComponents(const Decoder::Operand& dst, IR::Value val } } -IR::ValueOpcode Translator::ImageAtomicOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::IMAGE_ATOMIC_ADD: return IR::ValueOpcode::ImageAtomicIAdd32; - case Decoder::Opcode::IMAGE_ATOMIC_UMIN: return IR::ValueOpcode::ImageAtomicUMin32; - case Decoder::Opcode::IMAGE_ATOMIC_UMAX: return IR::ValueOpcode::ImageAtomicUMax32; - case Decoder::Opcode::IMAGE_ATOMIC_AND: return IR::ValueOpcode::ImageAtomicAnd32; - case Decoder::Opcode::IMAGE_ATOMIC_OR: return IR::ValueOpcode::ImageAtomicOr32; - case Decoder::Opcode::IMAGE_ATOMIC_XOR: return IR::ValueOpcode::ImageAtomicXor32; - default: EXIT("invalid image atomic opcode"); - } -} - Translator::BufferAddress Translator::ReadBufferAddress(const Decoder::Instruction& inst, uint32_t first_source) { const auto memory = MemoryInfoFromDecoded(inst); @@ -571,74 +418,8 @@ IR::Value Translator::NarrowSubdword(IR::U32 value, uint32_t bits) { : ir.Emit(IR::ValueOpcode::ConvertU16U32, {value}); } -IR::ValueOpcode Translator::BufferAtomicOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::BUFFER_ATOMIC_SWAP: return IR::ValueOpcode::BufferAtomicSwap32; - case Decoder::Opcode::BUFFER_ATOMIC_ADD: return IR::ValueOpcode::BufferAtomicIAdd32; - case Decoder::Opcode::BUFFER_ATOMIC_SUB: return IR::ValueOpcode::BufferAtomicISub32; - case Decoder::Opcode::BUFFER_ATOMIC_SMIN: return IR::ValueOpcode::BufferAtomicSMin32; - case Decoder::Opcode::BUFFER_ATOMIC_UMIN: return IR::ValueOpcode::BufferAtomicUMin32; - case Decoder::Opcode::BUFFER_ATOMIC_SMAX: return IR::ValueOpcode::BufferAtomicSMax32; - case Decoder::Opcode::BUFFER_ATOMIC_UMAX: return IR::ValueOpcode::BufferAtomicUMax32; - case Decoder::Opcode::BUFFER_ATOMIC_AND: return IR::ValueOpcode::BufferAtomicAnd32; - case Decoder::Opcode::BUFFER_ATOMIC_OR: return IR::ValueOpcode::BufferAtomicOr32; - case Decoder::Opcode::BUFFER_ATOMIC_XOR: return IR::ValueOpcode::BufferAtomicXor32; - case Decoder::Opcode::BUFFER_ATOMIC_FMIN: return IR::ValueOpcode::BufferAtomicFMin32; - case Decoder::Opcode::BUFFER_ATOMIC_FMAX: return IR::ValueOpcode::BufferAtomicFMax32; - default: EXIT("invalid buffer atomic opcode"); - } -} - -IR::ValueOpcode Translator::SharedAtomicOpcode(Decoder::Opcode opcode) { - switch (opcode) { - case Decoder::Opcode::DS_WRXCHG_RTN_B32: return IR::ValueOpcode::SharedAtomicSwap32; - case Decoder::Opcode::DS_ADD_U32: - case Decoder::Opcode::DS_ADD_RTN_U32: return IR::ValueOpcode::SharedAtomicIAdd32; - case Decoder::Opcode::DS_SUB_U32: - case Decoder::Opcode::DS_SUB_RTN_U32: return IR::ValueOpcode::SharedAtomicISub32; - case Decoder::Opcode::DS_MIN_I32: - case Decoder::Opcode::DS_MIN_RTN_I32: return IR::ValueOpcode::SharedAtomicSMin32; - case Decoder::Opcode::DS_MIN_U32: - case Decoder::Opcode::DS_MIN_RTN_U32: return IR::ValueOpcode::SharedAtomicUMin32; - case Decoder::Opcode::DS_MAX_I32: - case Decoder::Opcode::DS_MAX_RTN_I32: return IR::ValueOpcode::SharedAtomicSMax32; - case Decoder::Opcode::DS_MAX_U32: - case Decoder::Opcode::DS_MAX_RTN_U32: return IR::ValueOpcode::SharedAtomicUMax32; - case Decoder::Opcode::DS_AND_B32: - case Decoder::Opcode::DS_AND_RTN_B32: return IR::ValueOpcode::SharedAtomicAnd32; - case Decoder::Opcode::DS_OR_B32: - case Decoder::Opcode::DS_OR_RTN_B32: return IR::ValueOpcode::SharedAtomicOr32; - case Decoder::Opcode::DS_XOR_B32: - case Decoder::Opcode::DS_XOR_RTN_B32: return IR::ValueOpcode::SharedAtomicXor32; - default: EXIT("invalid shared atomic opcode"); - } -} - -bool Translator::TranslateScalarMemory(const Decoder::Instruction& inst, std::string* error) { - if (!IsScalarAddressLoad(inst.opcode) && !IsScalarBufferLoad(inst.opcode)) { - return false; - } - const auto memory = MemoryInfoFromDecoded(inst); - uint32_t dst_code = 0; - const auto alignment = memory.data_dwords == 1u ? 1u : memory.data_dwords == 2u ? 2u : 4u; - if (!TryGetEncodedScalarCode(inst.dst, dst_code) || dst_code % alignment != 0u || - dst_code + memory.data_dwords > 108u) { - if (error != nullptr) { - *error = "scalar-memory destination has an invalid aligned SGPR span"; - } - return false; - } - uint32_t base_code = 0; - const bool raw = IsScalarAddressLoad(inst.opcode); - const auto base_alignment = raw ? 2u : 4u; - const auto base_width = raw ? 2u : 4u; - if (!TryGetEncodedScalarCode(inst.src0, base_code) || base_code % base_alignment != 0u || - base_code + base_width > 108u) { - if (error != nullptr) { - *error = "scalar-memory base has an invalid aligned SGPR span"; - } - return false; - } +bool Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) { + const auto memory = MemoryInfoFromDecoded(inst); const auto resource = raw ? GetScalarAddressResource(memory.resource) : GetBufferResource(memory); const auto offset = ReadU32(inst.src1); @@ -663,7 +444,7 @@ bool Translator::TranslateScalarMemory(const Decoder::Instruction& inst, std::st return true; } -bool Translator::TranslateBufferLoad(const Decoder::Instruction& inst, std::string*) { +bool Translator::BUFFER_LOAD(const Decoder::Instruction& inst) { const auto memory = MemoryInfoFromDecoded(inst); IR::ValueOpcode opcode; const auto bits = memory.data_bits; @@ -700,7 +481,7 @@ bool Translator::TranslateBufferLoad(const Decoder::Instruction& inst, std::stri return true; } -bool Translator::TranslateBufferStore(const Decoder::Instruction& inst, std::string*) { +bool Translator::BUFFER_STORE(const Decoder::Instruction& inst) { const auto memory = MemoryInfoFromDecoded(inst); const auto resource = GetBufferResource(memory); const auto address = ReadBufferAddress(inst, 1); @@ -751,48 +532,46 @@ bool Translator::TranslateBufferStore(const Decoder::Instruction& inst, std::str return true; } -bool Translator::TranslateAtomicMemory(const Decoder::Instruction& inst, std::string*) { - const auto memory = MemoryInfoFromDecoded(inst); - IR::Value result; - switch (memory.kind) { - case IR::ResourceKind::Buffer: { - const auto resource = GetBufferResource(memory); - const auto address = ReadBufferAddress(inst, 1); - result = ir.Emit(BufferAtomicOpcode(inst.opcode), - {resource, address.index, address.offset, address.soffset, - ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, - AddMemoryInfo(memory, inst.pc)); - break; - } - case IR::ResourceKind::Image: - case IR::ResourceKind::ImageUint: - case IR::ResourceKind::StorageImage: - case IR::ResourceKind::StorageImageUint: { - const auto resource = GetImageResource(memory); - const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 1)); - const auto flags = AddMemoryInfo(memory, inst.pc); - result = - ir.Emit(ImageAtomicOpcode(inst.opcode), - {resource, address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, flags); - break; - } - case IR::ResourceKind::Lds: - case IR::ResourceKind::Gds: { - const auto address = ReadU32(MemorySourceAt(inst, 1)); - result = ir.Emit(SharedAtomicOpcode(inst.opcode), - {address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, - AddMemoryInfo(memory, inst.pc)); - break; - } - default: return false; - } - if (inst.glc || IsDsReturnAtomic(inst.opcode)) { +bool Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto resource = GetBufferResource(memory); + const auto address = ReadBufferAddress(inst, 1); + const auto result = ir.Emit(opcode, + {resource, address.index, address.offset, address.soffset, + ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); + if (inst.glc) { WriteOperand(inst.dst, result); } return true; } -bool Translator::TranslateFlatLoad(const Decoder::Instruction& inst, std::string*) { +bool Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto resource = GetImageResource(memory); + const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 1)); + const auto result = + ir.Emit(opcode, {resource, address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); + if (inst.glc) { + WriteOperand(inst.dst, result); + } + return true; +} + +bool Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool returns_value) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto address = ReadU32(MemorySourceAt(inst, 1)); + const auto result = ir.Emit(opcode, {address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); + if (returns_value) { + WriteOperand(inst.dst, result); + } + return true; +} + +bool Translator::FLAT_LOAD(const Decoder::Instruction& inst) { const auto memory = MemoryInfoFromDecoded(inst); IR::ValueOpcode opcode; const auto bits = memory.data_bits; @@ -819,7 +598,7 @@ bool Translator::TranslateFlatLoad(const Decoder::Instruction& inst, std::string return true; } -bool Translator::TranslateFlatStore(const Decoder::Instruction& inst, std::string*) { +bool Translator::FLAT_STORE(const Decoder::Instruction& inst) { const auto memory = MemoryInfoFromDecoded(inst); const auto data_op = MemorySourceAt(inst, 0); const auto address = ReadAddressOperands(inst, 1); @@ -846,338 +625,272 @@ bool Translator::TranslateFlatStore(const Decoder::Instruction& inst, std::strin return true; } -bool Translator::TranslateImageMemory(const Decoder::Instruction& inst, std::string*) { - const auto memory = MemoryInfoFromDecoded(inst); - const bool image = memory.kind == IR::ResourceKind::Image || - memory.kind == IR::ResourceKind::ImageUint || - memory.kind == IR::ResourceKind::StorageImage || - memory.kind == IR::ResourceKind::StorageImageUint; - if (!image) { - return false; - } +bool Translator::IMAGE_GET_RESINFO(const Decoder::Instruction& inst) { + const auto memory = MemoryInfoFromDecoded(inst); const auto resource = GetImageResource(memory); - const bool store = inst.opcode == Decoder::Opcode::IMAGE_STORE || - inst.opcode == Decoder::Opcode::IMAGE_STORE_MIP; - const auto address = MakeImageAddress(inst, MemorySourceAt(inst, store ? 1u : 0u)); - const auto flags = AddMemoryInfo(memory, inst.pc); - switch (inst.opcode) { - case Decoder::Opcode::IMAGE_GET_RESINFO: { - const auto result = - ir.Emit(IR::ValueOpcode::ImageQueryDimensions, {resource, address}, flags); - WriteImageComponents(inst.dst, result, memory, 4u); - return true; + const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 0)); + const auto result = ir.Emit(IR::ValueOpcode::ImageQueryDimensions, {resource, address}, + AddMemoryInfo(memory, inst.pc)); + WriteImageComponents(inst.dst, result, memory, 4u); + return true; +} + +bool Translator::IMAGE_GET_LOD(const Decoder::Instruction& inst) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto resource = GetImageResource(memory); + const auto sampler = GetSamplerResource(memory); + const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 0)); + const auto result = ir.Emit(IR::ValueOpcode::ImageQueryLod, {resource, sampler, address}, + AddMemoryInfo(memory, inst.pc)); + WriteImageComponents(inst.dst, result, memory, 2u); + return true; +} + +bool Translator::IMAGE_LOAD(const Decoder::Instruction& inst) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto resource = GetImageResource(memory); + const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 0)); + const auto result = ir.Emit(IR::ValueOpcode::ImageRead, {resource, address, ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); + WriteImageComponents(inst.dst, result, memory, 4u); + return true; +} + +bool Translator::IMAGE_STORE(const Decoder::Instruction& inst) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto resource = GetImageResource(memory); + const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 1)); + const auto data = ConstructU32x4(MemorySourceAt(inst, 0), memory.data_dwords); + ir.Emit(IR::ValueOpcode::ImageWrite, {resource, address, data, ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); + return true; +} + +bool Translator::IMAGE_SAMPLE(const Decoder::Instruction& inst) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto resource = GetImageResource(memory); + const auto sampler = GetSamplerResource(memory); + const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 0)); + const auto result = ir.Emit(IR::ValueOpcode::ImageSampleRaw, {resource, sampler, address}, + AddMemoryInfo(memory, inst.pc)); + const bool dref = (memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u; + if (dref && memory.data_bits != 16u) { + const auto component = + ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(0u)}); + for (uint32_t index = 0; index < memory.data_dwords; index++) { + WriteOperand(OffsetOperand(inst.dst, index), component); } - case Decoder::Opcode::IMAGE_GET_LOD: { - const auto sampler = GetSamplerResource(memory); - const auto result = - ir.Emit(IR::ValueOpcode::ImageQueryLod, {resource, sampler, address}, flags); - WriteImageComponents(inst.dst, result, memory, 2u); - return true; - } - case Decoder::Opcode::IMAGE_LOAD: - case Decoder::Opcode::IMAGE_LOAD_MIP: { - const auto result = - ir.Emit(IR::ValueOpcode::ImageRead, {resource, address, ir.GetExec()}, flags); - WriteImageComponents(inst.dst, result, memory, 4u); - return true; - } - case Decoder::Opcode::IMAGE_STORE: - case Decoder::Opcode::IMAGE_STORE_MIP: { - const auto data = ConstructU32x4(MemorySourceAt(inst, 0), memory.data_dwords); - ir.Emit(IR::ValueOpcode::ImageWrite, {resource, address, data, ir.GetExec()}, flags); - return true; - } - case Decoder::Opcode::IMAGE_SAMPLE: - case Decoder::Opcode::IMAGE_GATHER4_LZ: - case Decoder::Opcode::IMAGE_GATHER4_C: - case Decoder::Opcode::IMAGE_GATHER4_C_LZ: - case Decoder::Opcode::IMAGE_GATHER4_LZ_O: - case Decoder::Opcode::IMAGE_GATHER4_C_O: - case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O: - case Decoder::Opcode::IMAGE_GATHER4H: { - const auto sampler = GetSamplerResource(memory); - const bool sample = inst.opcode == Decoder::Opcode::IMAGE_SAMPLE; - const auto opcode = - sample ? IR::ValueOpcode::ImageSampleRaw : IR::ValueOpcode::ImageGatherRaw; - const auto result = ir.Emit(opcode, {resource, sampler, address}, flags); - const bool dref = (memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u; - if (sample && dref && memory.data_bits != 16u) { - const auto component = - ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(0u)}); - for (uint32_t index = 0; index < memory.data_dwords; index++) { - WriteOperand(OffsetOperand(inst.dst, index), component); - } - } else if (!sample) { - for (uint32_t index = 0; index < memory.data_dwords; index++) { - WriteOperand(OffsetOperand(inst.dst, index), - ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, - {result, IR::Value(index)})); - } - } else { - WriteImageComponents(inst.dst, result, memory, 4u); - } - return true; - } - default: return false; + } else { + WriteImageComponents(inst.dst, result, memory, 4u); + } + return true; +} + +bool Translator::IMAGE_GATHER(const Decoder::Instruction& inst) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto resource = GetImageResource(memory); + const auto sampler = GetSamplerResource(memory); + const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 0)); + const auto result = ir.Emit(IR::ValueOpcode::ImageGatherRaw, {resource, sampler, address}, + AddMemoryInfo(memory, inst.pc)); + for (uint32_t index = 0; index < memory.data_dwords; index++) { + WriteOperand(OffsetOperand(inst.dst, index), + ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(index)})); + } + return true; +} + +IR::Value Translator::LoadSharedU32(uint32_t width, IR::U32 address, const IR::MemoryInfo& memory, + uint32_t pc) { + IR::ValueOpcode opcode; + switch (width) { + case 1u: opcode = IR::ValueOpcode::LoadSharedU32; break; + case 2u: opcode = IR::ValueOpcode::LoadSharedU32x2; break; + case 3u: opcode = IR::ValueOpcode::LoadSharedU32x3; break; + case 4u: opcode = IR::ValueOpcode::LoadSharedU32x4; break; + default: EXIT("invalid shared load width"); + } + return ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(memory, pc)); +} + +IR::Value Translator::ExtractSharedU32(IR::Value value, uint32_t width, uint32_t index) { + if (width == 1u) { + return value; + } + IR::ValueOpcode opcode; + switch (width) { + case 2u: opcode = IR::ValueOpcode::CompositeExtractU32x2; break; + case 3u: opcode = IR::ValueOpcode::CompositeExtractU32x3; break; + case 4u: opcode = IR::ValueOpcode::CompositeExtractU32x4; break; + default: EXIT("invalid shared extract width"); + } + return ir.Emit(opcode, {value, IR::Value(index)}); +} + +void Translator::WriteSharedU32(uint32_t width, IR::U32 address, + const std::array& values, + const IR::MemoryInfo& memory, uint32_t pc) { + switch (width) { + case 1u: + ir.Emit(IR::ValueOpcode::WriteSharedU32, {address, values[0], ir.GetExec()}, + AddMemoryInfo(memory, pc)); + break; + case 2u: + ir.Emit(IR::ValueOpcode::WriteSharedU32x2, + {address, values[0], values[1], ir.GetExec()}, AddMemoryInfo(memory, pc)); + break; + case 3u: + ir.Emit(IR::ValueOpcode::WriteSharedU32x3, + {address, values[0], values[1], values[2], ir.GetExec()}, + AddMemoryInfo(memory, pc)); + break; + case 4u: + ir.Emit(IR::ValueOpcode::WriteSharedU32x4, + {address, values[0], values[1], values[2], values[3], ir.GetExec()}, + AddMemoryInfo(memory, pc)); + break; + default: EXIT("invalid shared store width"); } } -bool Translator::TranslateSharedMemory(const Decoder::Instruction& inst, std::string*) { +bool Translator::DS_READ(const Decoder::Instruction& inst) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto address = ReadU32(MemorySourceAt(inst, 0)); + if (memory.data_bits == 32u) { + const auto width = memory.data_dwords; + const auto loaded = LoadSharedU32(width, address, memory, inst.pc); + for (uint32_t index = 0; index < width; index++) { + WriteOperand(OffsetOperand(inst.dst, index), ExtractSharedU32(loaded, width, index)); + } + return true; + } + const auto opcode = + memory.data_bits == 8u ? IR::ValueOpcode::LoadSharedU8 : IR::ValueOpcode::LoadSharedU16; + const auto loaded = ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(memory, inst.pc)); + WriteOperand(inst.dst, WidenSubdword(loaded, memory.data_bits, memory.data_signed)); + return true; +} + +bool Translator::DS_READ2(const Decoder::Instruction& inst) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto width = memory.data_dwords / 2u; + const auto address = ReadU32(MemorySourceAt(inst, 0)); + auto first = memory; + first.data_dwords = width; + first.component_count = width; + const auto first_value = LoadSharedU32(width, address, first, inst.pc); + IR::Value second_value = first_value; + if (memory.secondary_offset != memory.offset) { + auto second = first; + second.offset = memory.secondary_offset; + second_value = LoadSharedU32(width, address, second, inst.pc); + } + for (uint32_t index = 0; index < width; index++) { + WriteOperand(OffsetOperand(inst.dst, index), ExtractSharedU32(first_value, width, index)); + WriteOperand(OffsetOperand(inst.dst, width + index), + ExtractSharedU32(second_value, width, index)); + } + return true; +} + +bool Translator::DS_WRITE(const Decoder::Instruction& inst) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto width = memory.data_dwords; + const auto data_operand = MemorySourceAt(inst, 0); + std::array values {}; + for (uint32_t index = 0; index < width; index++) { + values[index] = ReadU32(OffsetOperand(data_operand, index)); + } + const auto address = ReadU32(MemorySourceAt(inst, 1)); + if (memory.data_bits == 32u) { + WriteSharedU32(width, address, values, memory, inst.pc); + return true; + } + const auto opcode = + memory.data_bits == 8u ? IR::ValueOpcode::WriteSharedU8 : IR::ValueOpcode::WriteSharedU16; + ir.Emit(opcode, {address, NarrowSubdword(IR::U32(values[0]), memory.data_bits), ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); + return true; +} + +bool Translator::DS_WRITE2(const Decoder::Instruction& inst) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto width = memory.data_dwords / 2u; + const auto address = ReadU32(MemorySourceAt(inst, 1)); + const auto first_data = MemorySourceAt(inst, 0); + const auto second_data = MemorySourceAt(inst, 2); + std::array first_values {}; + std::array second_values {}; + for (uint32_t index = 0; index < width; index++) { + first_values[index] = ReadU32(OffsetOperand(first_data, index)); + second_values[index] = ReadU32(OffsetOperand(second_data, index)); + } + auto first = memory; + first.data_dwords = width; + first.component_count = width; + WriteSharedU32(width, address, first_values, first, inst.pc); + if (memory.secondary_offset != memory.offset) { + auto second = first; + second.offset = memory.secondary_offset; + WriteSharedU32(width, address, second_values, second, inst.pc); + } + return true; +} + +bool Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { const auto memory = MemoryInfoFromDecoded(inst); - const bool shared = - memory.kind == IR::ResourceKind::Lds || memory.kind == IR::ResourceKind::Gds; - if (!shared && inst.opcode != Decoder::Opcode::DS_SWIZZLE_B32) { - return false; - } - const auto shared_address = [&](uint32_t source) { - return ReadU32(MemorySourceAt(inst, source)); - }; - const auto load_u32 = [&](uint32_t width, IR::U32 address, const IR::MemoryInfo& source) { - IR::ValueOpcode opcode; - switch (width) { - case 1u: opcode = IR::ValueOpcode::LoadSharedU32; break; - case 2u: opcode = IR::ValueOpcode::LoadSharedU32x2; break; - case 3u: opcode = IR::ValueOpcode::LoadSharedU32x3; break; - case 4u: opcode = IR::ValueOpcode::LoadSharedU32x4; break; - default: EXIT("invalid shared load width"); - } - return ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(source, inst.pc)); - }; - const auto extract_u32 = [&](IR::Value value, uint32_t width, uint32_t index) { - if (width == 1u) { - return value; - } - const auto opcode = width == 2u ? IR::ValueOpcode::CompositeExtractU32x2 - : width == 3u ? IR::ValueOpcode::CompositeExtractU32x3 - : IR::ValueOpcode::CompositeExtractU32x4; - return ir.Emit(opcode, {value, IR::Value(index)}); - }; - const auto write_u32 = [&](uint32_t width, IR::U32 address, - const std::array& values, - const IR::MemoryInfo& source) { - switch (width) { - case 1u: - ir.Emit(IR::ValueOpcode::WriteSharedU32, {address, values[0], ir.GetExec()}, - AddMemoryInfo(source, inst.pc)); - break; - case 2u: - ir.Emit(IR::ValueOpcode::WriteSharedU32x2, - {address, values[0], values[1], ir.GetExec()}, - AddMemoryInfo(source, inst.pc)); - break; - case 3u: - ir.Emit(IR::ValueOpcode::WriteSharedU32x3, - {address, values[0], values[1], values[2], ir.GetExec()}, - AddMemoryInfo(source, inst.pc)); - break; - case 4u: - ir.Emit(IR::ValueOpcode::WriteSharedU32x4, - {address, values[0], values[1], values[2], values[3], ir.GetExec()}, - AddMemoryInfo(source, inst.pc)); - break; - default: EXIT("invalid shared store width"); - } - }; - switch (inst.opcode) { - case Decoder::Opcode::DS_READ_I8: - case Decoder::Opcode::DS_READ_U8: - case Decoder::Opcode::DS_READ_I16: - case Decoder::Opcode::DS_READ_U16: - case Decoder::Opcode::DS_READ_U16_D16: - case Decoder::Opcode::DS_READ_B32: - case Decoder::Opcode::DS_READ_B64: - case Decoder::Opcode::DS_READ_B96: - case Decoder::Opcode::DS_READ_B128: { - IR::ValueOpcode opcode; - uint32_t bits; - bool sign; - if (memory.data_bits == 8u) { - opcode = IR::ValueOpcode::LoadSharedU8; - bits = 8u; - sign = memory.data_signed; - } else if (memory.data_bits == 16u) { - opcode = IR::ValueOpcode::LoadSharedU16; - bits = 16u; - sign = memory.data_signed; - } else { - const auto width = memory.data_dwords; - const auto loaded = load_u32(width, shared_address(0), memory); - for (uint32_t index = 0; index < width; index++) { - WriteOperand(OffsetOperand(inst.dst, index), extract_u32(loaded, width, index)); - } - return true; - } - const auto loaded = - ir.Emit(opcode, {shared_address(0), ir.GetExec()}, AddMemoryInfo(memory, inst.pc)); - WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign)); - return true; - } - case Decoder::Opcode::DS_READ2_B32: - case Decoder::Opcode::DS_READ2ST64_B32: - case Decoder::Opcode::DS_READ2_B64: - case Decoder::Opcode::DS_READ2ST64_B64: { - const auto width = memory.data_dwords / 2u; - const auto address = shared_address(0); - auto first = memory; - first.data_dwords = width; - first.component_count = width; - const auto first_value = load_u32(width, address, first); - IR::Value second_value = first_value; - if (memory.secondary_offset != memory.offset) { - auto second = first; - second.offset = memory.secondary_offset; - second_value = load_u32(width, address, second); - } - for (uint32_t index = 0; index < width; index++) { - WriteOperand(OffsetOperand(inst.dst, index), - extract_u32(first_value, width, index)); - WriteOperand(OffsetOperand(inst.dst, width + index), - extract_u32(second_value, width, index)); - } - return true; - } - case Decoder::Opcode::DS_WRITE_B8: - case Decoder::Opcode::DS_WRITE_B16: - case Decoder::Opcode::DS_WRITE_B32: - case Decoder::Opcode::DS_WRITE_B64: - case Decoder::Opcode::DS_WRITE_B96: - case Decoder::Opcode::DS_WRITE_B128: { - const auto width = memory.data_dwords; - const auto data_operand = MemorySourceAt(inst, 0); - std::array values {}; - for (uint32_t index = 0; index < width; index++) { - values[index] = ReadU32(OffsetOperand(data_operand, index)); - } - const auto data = IR::U32(values[0]); - const auto address = shared_address(1); - IR::ValueOpcode opcode; - IR::Value value; - if (memory.data_bits == 8u) { - opcode = IR::ValueOpcode::WriteSharedU8; - value = NarrowSubdword(data, 8u); - } else if (memory.data_bits == 16u) { - opcode = IR::ValueOpcode::WriteSharedU16; - value = NarrowSubdword(data, 16u); - } else { - write_u32(width, address, values, memory); - return true; - } - ir.Emit(opcode, {address, value, ir.GetExec()}, AddMemoryInfo(memory, inst.pc)); - return true; - } - case Decoder::Opcode::DS_WRITE2_B32: - case Decoder::Opcode::DS_WRITE2ST64_B32: - case Decoder::Opcode::DS_WRITE2_B64: - case Decoder::Opcode::DS_WRITE2ST64_B64: { - const auto width = memory.data_dwords / 2u; - const auto address = shared_address(1); - const auto first_data = MemorySourceAt(inst, 0); - const auto second_data = MemorySourceAt(inst, 2); - std::array first_values {}; - std::array second_values {}; - for (uint32_t index = 0; index < width; index++) { - first_values[index] = ReadU32(OffsetOperand(first_data, index)); - second_values[index] = ReadU32(OffsetOperand(second_data, index)); - } - auto first = memory; - first.data_dwords = width; - first.component_count = width; - write_u32(width, address, first_values, first); - if (memory.secondary_offset != memory.offset) { - auto second = first; - second.offset = memory.secondary_offset; - write_u32(width, address, second_values, second); - } - return true; - } - case Decoder::Opcode::DS_MIN_F32: - case Decoder::Opcode::DS_MAX_F32: { - const auto opcode = inst.opcode == Decoder::Opcode::DS_MIN_F32 - ? IR::ValueOpcode::SharedAtomicFMin32 - : IR::ValueOpcode::SharedAtomicFMax32; - ir.Emit(opcode, - {shared_address(1), ReadU32(MemorySourceAt(inst, 0)), - ReadU32(MemorySourceAt(inst, 2)), ir.GetExec()}, - AddMemoryInfo(memory, inst.pc)); - return true; - } - case Decoder::Opcode::DS_APPEND: - case Decoder::Opcode::DS_CONSUME: { - const bool append = inst.opcode == Decoder::Opcode::DS_APPEND; - const auto opcode = append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume; - WriteOperand(inst.dst, ir.Emit(opcode, - {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec(), - ir.GetExecLo(), ir.GetExecHi()}, - AddMemoryInfo(memory, inst.pc))); - return true; - } - case Decoder::Opcode::DS_WRITE_ADDTID_B32: - case Decoder::Opcode::DS_READ_ADDTID_B32: { - const auto m0_index = inst.opcode == Decoder::Opcode::DS_WRITE_ADDTID_B32 ? 1u : 0u; - const auto base = - ir.BitwiseAnd(ReadU32(MemorySourceAt(inst, m0_index)), IR::U32(IR::Value(0xffffu))); - const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); - const auto address = ir.IAdd(base, ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u)))); - if (inst.opcode == Decoder::Opcode::DS_WRITE_ADDTID_B32) { - ir.Emit(IR::ValueOpcode::WriteSharedU32, - {address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, - AddMemoryInfo(memory, inst.pc)); - } else { - WriteOperand(inst.dst, - ir.Emit(IR::ValueOpcode::LoadSharedU32, {address, ir.GetExec()}, - AddMemoryInfo(memory, inst.pc))); - } - return true; - } - case Decoder::Opcode::DS_SWIZZLE_B32: - WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::SwizzleU32, - {ReadU32(MemorySourceAt(inst, 0)), - ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()})); - return true; - default: return false; - } + ir.Emit(opcode, + {ReadU32(MemorySourceAt(inst, 1)), ReadU32(MemorySourceAt(inst, 0)), + ReadU32(MemorySourceAt(inst, 2)), ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); + return true; } -bool Translator::TranslateMemoryOperation(const Decoder::Instruction& inst, std::string* error) { - if (!MemoryOpcodeMatchesFamily(inst.opcode, inst.family)) { - if (error != nullptr) { - *error = fmt::format("memory opcode {} does not belong to decoded family {}", - magic_enum::enum_name(inst.opcode), magic_enum::enum_name(inst.family)); - } - return false; - } - if (RequiresRegisterDestination(inst) && !IsRegisterOperand(inst.dst)) { - if (error != nullptr) { - *error = "decoded operand cannot be used as an IR register"; - } - return false; - } - if (UsesDestinationAsMemorySource(inst.opcode) && - inst.dst.kind == Decoder::OperandKind::Unknown) { - if (error != nullptr) { - *error = "decoded operand cannot be used as an IR register"; - } - return false; - } - for (uint32_t index = 0; index < inst.src_count; index++) { - if (DecodedSourceAt(inst, index).kind == Decoder::OperandKind::Unknown) { - if (error != nullptr) { - *error = "decoded operand cannot be used as an IR register"; - } - return false; - } +bool Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { + const auto memory = MemoryInfoFromDecoded(inst); + WriteOperand(inst.dst, ir.Emit(opcode, + {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec(), ir.GetExecLo(), + ir.GetExecHi()}, + AddMemoryInfo(memory, inst.pc))); + return true; +} + +bool Translator::DS_ADDTID(const Decoder::Instruction& inst, bool write) { + const auto memory = MemoryInfoFromDecoded(inst); + const auto base = + ir.BitwiseAnd(ReadU32(MemorySourceAt(inst, write ? 1u : 0u)), IR::U32(IR::Value(0xffffu))); + const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); + const auto address = ir.IAdd(base, ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u)))); + if (write) { + ir.Emit(IR::ValueOpcode::WriteSharedU32, + {address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, + AddMemoryInfo(memory, inst.pc)); + } else { + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::LoadSharedU32, {address, ir.GetExec()}, + AddMemoryInfo(memory, inst.pc))); } + return true; +} + +bool Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) { + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::SwizzleU32, + {ReadU32(MemorySourceAt(inst, 0)), + ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()})); + return true; +} + +bool Translator::EmitMemory(const Decoder::Instruction& inst, std::string* error) { switch (inst.opcode) { case Decoder::Opcode::S_LOAD_DWORD: case Decoder::Opcode::S_LOAD_DWORDX2: case Decoder::Opcode::S_LOAD_DWORDX4: case Decoder::Opcode::S_LOAD_DWORDX8: - case Decoder::Opcode::S_LOAD_DWORDX16: + case Decoder::Opcode::S_LOAD_DWORDX16: return S_LOAD(inst, true); case Decoder::Opcode::S_BUFFER_LOAD_DWORD: case Decoder::Opcode::S_BUFFER_LOAD_DWORDX2: case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4: case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8: - case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16: return TranslateScalarMemory(inst, error); + case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16: return S_LOAD(inst, false); case Decoder::Opcode::BUFFER_LOAD_UBYTE: case Decoder::Opcode::BUFFER_LOAD_SBYTE: @@ -1194,7 +907,7 @@ bool Translator::TranslateMemoryOperation(const Decoder::Instruction& inst, std: case Decoder::Opcode::TBUFFER_LOAD_FORMAT_X: case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XY: case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZ: - case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZW: return TranslateBufferLoad(inst, error); + case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZW: return BUFFER_LOAD(inst); case Decoder::Opcode::BUFFER_STORE_DWORD: case Decoder::Opcode::BUFFER_STORE_DWORDX2: @@ -1209,45 +922,84 @@ bool Translator::TranslateMemoryOperation(const Decoder::Instruction& inst, std: case Decoder::Opcode::TBUFFER_STORE_FORMAT_X: case Decoder::Opcode::TBUFFER_STORE_FORMAT_XY: case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZ: - case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW: return TranslateBufferStore(inst, error); + case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW: return BUFFER_STORE(inst); case Decoder::Opcode::BUFFER_ATOMIC_SWAP: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicSwap32); case Decoder::Opcode::BUFFER_ATOMIC_ADD: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicIAdd32); case Decoder::Opcode::BUFFER_ATOMIC_SUB: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicISub32); case Decoder::Opcode::BUFFER_ATOMIC_SMIN: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicSMin32); case Decoder::Opcode::BUFFER_ATOMIC_UMIN: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicUMin32); case Decoder::Opcode::BUFFER_ATOMIC_SMAX: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicSMax32); case Decoder::Opcode::BUFFER_ATOMIC_UMAX: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicUMax32); case Decoder::Opcode::BUFFER_ATOMIC_AND: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicAnd32); case Decoder::Opcode::BUFFER_ATOMIC_OR: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicOr32); case Decoder::Opcode::BUFFER_ATOMIC_XOR: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicXor32); case Decoder::Opcode::BUFFER_ATOMIC_FMIN: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicFMin32); case Decoder::Opcode::BUFFER_ATOMIC_FMAX: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicFMax32); + case Decoder::Opcode::DS_ADD_U32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, false); case Decoder::Opcode::DS_ADD_RTN_U32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, true); case Decoder::Opcode::DS_SUB_U32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicISub32, false); case Decoder::Opcode::DS_SUB_RTN_U32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicISub32, true); case Decoder::Opcode::DS_MIN_I32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSMin32, false); case Decoder::Opcode::DS_MIN_RTN_I32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSMin32, true); case Decoder::Opcode::DS_MAX_I32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSMax32, false); case Decoder::Opcode::DS_MAX_RTN_I32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSMax32, true); case Decoder::Opcode::DS_MIN_U32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicUMin32, false); case Decoder::Opcode::DS_MIN_RTN_U32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicUMin32, true); case Decoder::Opcode::DS_MAX_U32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicUMax32, false); case Decoder::Opcode::DS_MAX_RTN_U32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicUMax32, true); case Decoder::Opcode::DS_AND_B32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicAnd32, false); case Decoder::Opcode::DS_AND_RTN_B32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicAnd32, true); case Decoder::Opcode::DS_OR_B32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicOr32, false); case Decoder::Opcode::DS_OR_RTN_B32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicOr32, true); case Decoder::Opcode::DS_XOR_B32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, false); case Decoder::Opcode::DS_XOR_RTN_B32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, true); case Decoder::Opcode::DS_WRXCHG_RTN_B32: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSwap32, true); + case Decoder::Opcode::IMAGE_ATOMIC_ADD: + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32); case Decoder::Opcode::IMAGE_ATOMIC_UMIN: + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32); case Decoder::Opcode::IMAGE_ATOMIC_UMAX: + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMax32); case Decoder::Opcode::IMAGE_ATOMIC_AND: + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32); case Decoder::Opcode::IMAGE_ATOMIC_OR: - case Decoder::Opcode::IMAGE_ATOMIC_XOR: return TranslateAtomicMemory(inst, error); + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicOr32); + case Decoder::Opcode::IMAGE_ATOMIC_XOR: + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicXor32); case Decoder::Opcode::FLAT_LOAD_UBYTE: case Decoder::Opcode::FLAT_LOAD_SBYTE: @@ -1256,41 +1008,45 @@ bool Translator::TranslateMemoryOperation(const Decoder::Instruction& inst, std: case Decoder::Opcode::FLAT_LOAD_DWORD: case Decoder::Opcode::FLAT_LOAD_DWORDX2: case Decoder::Opcode::FLAT_LOAD_DWORDX3: - case Decoder::Opcode::FLAT_LOAD_DWORDX4: return TranslateFlatLoad(inst, error); + case Decoder::Opcode::FLAT_LOAD_DWORDX4: return FLAT_LOAD(inst); case Decoder::Opcode::FLAT_STORE_BYTE: case Decoder::Opcode::FLAT_STORE_SHORT: case Decoder::Opcode::FLAT_STORE_DWORD: case Decoder::Opcode::FLAT_STORE_DWORDX2: case Decoder::Opcode::FLAT_STORE_DWORDX3: - case Decoder::Opcode::FLAT_STORE_DWORDX4: return TranslateFlatStore(inst, error); + case Decoder::Opcode::FLAT_STORE_DWORDX4: return FLAT_STORE(inst); - case Decoder::Opcode::IMAGE_GET_RESINFO: - case Decoder::Opcode::IMAGE_GET_LOD: + case Decoder::Opcode::IMAGE_GET_RESINFO: return IMAGE_GET_RESINFO(inst); + case Decoder::Opcode::IMAGE_GET_LOD: return IMAGE_GET_LOD(inst); case Decoder::Opcode::IMAGE_LOAD: - case Decoder::Opcode::IMAGE_LOAD_MIP: + case Decoder::Opcode::IMAGE_LOAD_MIP: return IMAGE_LOAD(inst); case Decoder::Opcode::IMAGE_STORE: - case Decoder::Opcode::IMAGE_STORE_MIP: + case Decoder::Opcode::IMAGE_STORE_MIP: return IMAGE_STORE(inst); + case Decoder::Opcode::IMAGE_SAMPLE: return IMAGE_SAMPLE(inst); case Decoder::Opcode::IMAGE_GATHER4_LZ: case Decoder::Opcode::IMAGE_GATHER4_C: case Decoder::Opcode::IMAGE_GATHER4_C_LZ: case Decoder::Opcode::IMAGE_GATHER4_LZ_O: case Decoder::Opcode::IMAGE_GATHER4_C_O: case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O: - case Decoder::Opcode::IMAGE_GATHER4H: - case Decoder::Opcode::IMAGE_SAMPLE: return TranslateImageMemory(inst, error); + case Decoder::Opcode::IMAGE_GATHER4H: return IMAGE_GATHER(inst); case Decoder::Opcode::DS_MIN_F32: + return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMin32); case Decoder::Opcode::DS_MAX_F32: - case Decoder::Opcode::DS_SWIZZLE_B32: + return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMax32); + case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst); case Decoder::Opcode::DS_CONSUME: + return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataConsume); case Decoder::Opcode::DS_APPEND: - case Decoder::Opcode::DS_WRITE_ADDTID_B32: - case Decoder::Opcode::DS_READ_ADDTID_B32: + return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataAppend); + case Decoder::Opcode::DS_WRITE_ADDTID_B32: return DS_ADDTID(inst, true); + case Decoder::Opcode::DS_READ_ADDTID_B32: return DS_ADDTID(inst, false); case Decoder::Opcode::DS_READ2_B32: case Decoder::Opcode::DS_READ2ST64_B32: case Decoder::Opcode::DS_READ2_B64: - case Decoder::Opcode::DS_READ2ST64_B64: + case Decoder::Opcode::DS_READ2ST64_B64: return DS_READ2(inst); case Decoder::Opcode::DS_READ_I8: case Decoder::Opcode::DS_READ_U8: case Decoder::Opcode::DS_READ_I16: @@ -1299,17 +1055,17 @@ bool Translator::TranslateMemoryOperation(const Decoder::Instruction& inst, std: case Decoder::Opcode::DS_READ_B32: case Decoder::Opcode::DS_READ_B64: case Decoder::Opcode::DS_READ_B96: - case Decoder::Opcode::DS_READ_B128: + case Decoder::Opcode::DS_READ_B128: return DS_READ(inst); case Decoder::Opcode::DS_WRITE2_B32: case Decoder::Opcode::DS_WRITE2ST64_B32: case Decoder::Opcode::DS_WRITE2_B64: - case Decoder::Opcode::DS_WRITE2ST64_B64: + case Decoder::Opcode::DS_WRITE2ST64_B64: return DS_WRITE2(inst); case Decoder::Opcode::DS_WRITE_B8: case Decoder::Opcode::DS_WRITE_B16: case Decoder::Opcode::DS_WRITE_B32: case Decoder::Opcode::DS_WRITE_B64: case Decoder::Opcode::DS_WRITE_B96: - case Decoder::Opcode::DS_WRITE_B128: return TranslateSharedMemory(inst, error); + case Decoder::Opcode::DS_WRITE_B128: return DS_WRITE(inst); default: if (error != nullptr) { *error = fmt::format("memory-family opcode has no specialized IR translation: {}", diff --git a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp new file mode 100644 index 000000000..9541e8e18 --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp @@ -0,0 +1,221 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +namespace Libs::Graphics::ShaderRecompiler::Frontend { + +bool Translator::EmitScalar(const Decoder::Instruction& inst, std::string* error) { + using O = Decoder::Opcode; + switch (inst.opcode) { + case O::S_MOV_B32: + case O::S_MOVK_I32: MOV_B32(inst, false); return true; + case O::S_MOV_B64: S_MOV_B64(inst); return true; + case O::S_WQM_B64: S_WQM_B64(inst); return true; + case O::S_GETPC_B64: return S_GETPC_B64(inst, error); + case O::S_SETPC_B64: return true; + case O::S_CSELECT_B32: S_CSELECT_B32(inst); return true; + case O::S_CSELECT_B64: S_CSELECT_B64(inst); return true; + case O::S_SETREG_B32: EmitControlNop(); return true; + case O::S_WAITCNT: EmitWaitcnt(); return true; + + case O::S_AND_SAVEEXEC_B32: + S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, false); + return true; + case O::S_ANDN1_SAVEEXEC_B32: + S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, true, false); + return true; + case O::S_AND_SAVEEXEC_B64: + S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, true); + return true; + case O::S_ANDN1_SAVEEXEC_B64: + S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, true, true); + return true; + case O::S_ORN2_SAVEEXEC_B64: + S_SAVEEXEC(inst, IR::ValueOpcode::LogicalOr, true, false, true); + return true; + case O::S_ADD_U32: ADD_U32(inst, false, false); return true; + case O::S_ADDC_U32: ADD_U32(inst, false, true); return true; + case O::S_SUB_U32: SUB_U32(inst, false, false); return true; + case O::S_SUBB_U32: SUBB_U32(inst, false); return true; + case O::S_ADD_I32: S_ADD_SUB_I32(inst, false); return true; + case O::S_SUB_I32: S_ADD_SUB_I32(inst, true); return true; + case O::S_LSHL1_ADD_U32: S_LSHL_ADD_U32(inst, 1u); return true; + case O::S_LSHL2_ADD_U32: S_LSHL_ADD_U32(inst, 2u); return true; + case O::S_LSHL3_ADD_U32: S_LSHL_ADD_U32(inst, 3u); return true; + case O::S_LSHL4_ADD_U32: S_LSHL_ADD_U32(inst, 4u); return true; + case O::S_MIN_I32: + ScalarMinMax32(inst, IR::ValueOpcode::SMin32, IR::ValueOpcode::SLessThan32); + return true; + case O::S_MAX_I32: + ScalarMinMax32(inst, IR::ValueOpcode::SMax32, IR::ValueOpcode::SGreaterThan32); + return true; + case O::S_MIN_U32: + ScalarMinMax32(inst, IR::ValueOpcode::UMin32, IR::ValueOpcode::ULessThan32); + return true; + case O::S_MAX_U32: + ScalarMinMax32(inst, IR::ValueOpcode::UMax32, IR::ValueOpcode::UGreaterThan32); + return true; + + case O::S_CMP_EQ_U32: + case O::S_CMP_EQ_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, true, false); + return true; + case O::S_CMP_LG_U32: + case O::S_CMP_LG_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, true, false); + return true; + case O::S_CMP_GT_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, true, false); + return true; + case O::S_CMP_GE_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, true, + false); + return true; + case O::S_CMP_LT_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, true, false); + return true; + case O::S_CMP_LE_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, true, false); + return true; + case O::S_CMP_GT_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, true, false); + return true; + case O::S_CMP_GE_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, true, + false); + return true; + case O::S_CMP_LT_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, true, false); + return true; + case O::S_CMP_LE_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, true, false); + return true; + case O::S_CMP_EQ_U64: + EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, true, false); + return true; + case O::S_CMP_LG_U64: + EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, true, false); + return true; + + case O::S_AND_B64: + return S_U64_MASK(inst, IR::ValueOpcode::LogicalAnd, IR::ValueOpcode::BitwiseAnd32, + false, false, false); + case O::S_ANDN2_B64: + return S_U64_MASK(inst, IR::ValueOpcode::LogicalAnd, IR::ValueOpcode::BitwiseAnd32, + true, false, false); + case O::S_OR_B64: + return S_U64_MASK(inst, IR::ValueOpcode::LogicalOr, IR::ValueOpcode::BitwiseOr32, false, + false, false); + case O::S_ORN2_B64: + return S_U64_MASK(inst, IR::ValueOpcode::LogicalOr, IR::ValueOpcode::BitwiseOr32, true, + false, false); + case O::S_XOR_B64: + return S_U64_MASK(inst, IR::ValueOpcode::LogicalXor, IR::ValueOpcode::BitwiseXor32, + false, false, false); + case O::S_NAND_B64: + return S_U64_MASK(inst, IR::ValueOpcode::LogicalAnd, IR::ValueOpcode::BitwiseAnd32, + false, true, false); + case O::S_NOR_B64: + return S_U64_MASK(inst, IR::ValueOpcode::LogicalOr, IR::ValueOpcode::BitwiseOr32, false, + true, false); + case O::S_XNOR_B64: + return S_U64_MASK(inst, IR::ValueOpcode::LogicalXor, IR::ValueOpcode::BitwiseXor32, + false, true, false); + case O::S_NOT_B64: + return S_U64_MASK(inst, IR::ValueOpcode::LogicalAnd, IR::ValueOpcode::BitwiseAnd32, + false, false, true); + + case O::S_ABS_I32: + return SimpleInteger(inst, IR::ValueOpcode::IAbs32, IR::Type::U32, false, false, true); + case O::S_MUL_I32: + case O::S_MULK_I32: + return SimpleInteger(inst, IR::ValueOpcode::IMul32, IR::Type::U32, false, false, false); + case O::S_MUL_HI_U32: + return SimpleInteger(inst, IR::ValueOpcode::UMulHi, IR::Type::U32, false, false, false); + case O::S_AND_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitwiseAnd32, IR::Type::U32, false, false, + true); + case O::S_OR_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitwiseOr32, IR::Type::U32, false, false, + true); + case O::S_XOR_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitwiseXor32, IR::Type::U32, false, false, + true); + case O::S_NOT_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitwiseNot32, IR::Type::U32, false, false, + true); + case O::S_BREV_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitReverse32, IR::Type::U32, false, false, + false); + case O::S_BCNT1_I32_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false, + true); + case O::S_BCNT1_I32_B64: + return SimpleInteger(inst, IR::ValueOpcode::BitCount64, IR::Type::U64, false, false, + true); + case O::S_FF1_I32_B32: + return SimpleInteger(inst, IR::ValueOpcode::FindILsb32, IR::Type::U32, false, false, + false); + case O::S_LSHL_B32: + return SimpleInteger(inst, IR::ValueOpcode::ShiftLeftLogical32, IR::Type::U32, false, + true, true); + case O::S_LSHR_B32: + return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical32, IR::Type::U32, false, + true, true); + case O::S_ASHR_I32: + return SimpleInteger(inst, IR::ValueOpcode::ShiftRightArithmetic32, IR::Type::U32, + false, true, true); + case O::S_LSHL_B64: + return SimpleInteger(inst, IR::ValueOpcode::ShiftLeftLogical64, IR::Type::U64, false, + false, true); + case O::S_LSHR_B64: + return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical64, IR::Type::U64, false, + false, true); + + case O::S_ANDN2_B32: + return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseAnd32, true, false, true); + case O::S_ORN2_B32: + return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseOr32, true, false, true); + case O::S_NAND_B32: + return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseAnd32, false, true, true); + case O::S_NOR_B32: + return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseOr32, false, true, true); + case O::S_XNOR_B32: + return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseXor32, false, true, true); + case O::S_FF1_I32_B64: return S_FF1_I32_B64(inst); + case O::S_FLBIT_I32_B64: return S_FLBIT_I32_B64(inst); + + case O::S_BITSET0_B32: return S_BITSET_B32(inst, false); + case O::S_BITSET1_B32: return S_BITSET_B32(inst, true); + case O::S_BITREPLICATE_B64_B32: return S_BITREPLICATE_B64_B32(inst); + case O::S_QUADMASK_B64: return S_QUADMASK_B64(inst); + case O::S_BFM_B32: return BFM_B32(inst); + case O::S_BFM_B64: return S_BFM_B64(inst); + case O::S_BFE_U32: return S_BFE_U32(inst, false); + case O::S_BFE_I32: return S_BFE_U32(inst, true); + case O::S_BFE_U64: return S_BFE_U64(inst); + case O::S_BITCMP0_B32: return S_BITCMP_B32(inst, false); + case O::S_BITCMP1_B32: return S_BITCMP_B32(inst, true); + case O::S_PACK_LL_B32_B16: return PackB16(inst, false, false); + case O::S_PACK_LH_B32_B16: return PackB16(inst, false, true); + case O::S_PACK_HH_B32_B16: return PackB16(inst, true, true); + + case O::S_NOP: + case O::S_SLEEP: + case O::S_TRAP: EmitControlNop(); return true; + case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return true; + case O::S_BARRIER: S_BARRIER(); return true; + case O::S_SENDMSG: S_SENDMSG(); return true; + case O::S_TTRACEDATA: S_TTRACEDATA(); return true; + case O::S_INST_PREFETCH: S_INST_PREFETCH(); return true; + case O::S_BRANCH: + case O::S_CBRANCH_SCC0: + case O::S_CBRANCH_SCC1: + case O::S_CBRANCH_VCCZ: + case O::S_CBRANCH_VCCNZ: + case O::S_CBRANCH_EXECZ: + case O::S_CBRANCH_EXECNZ: + case O::S_ENDPGM: return true; + default: return false; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 0fdc2756b..89c74e832 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -75,24 +75,40 @@ private: IR::Value GetScalarAddressResource(uint32_t base); IR::Value GetImageResource(const IR::MemoryInfo& memory); IR::Value GetSamplerResource(const IR::MemoryInfo& memory); - IR::Value MakeImageAddress(const Decoder::Instruction& inst, const Decoder::Operand& base); - IR::Value ConstructU32x4(const Decoder::Operand& base, uint32_t count); - void WriteImageComponents(const Decoder::Operand& dst, IR::Value value, - const IR::MemoryInfo& memory, uint32_t component_limit); - IR::ValueOpcode ImageAtomicOpcode(Decoder::Opcode opcode); - BufferAddress ReadBufferAddress(const Decoder::Instruction& inst, uint32_t source_offset); - IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign); - IR::Value NarrowSubdword(IR::U32 value, uint32_t bits); - IR::ValueOpcode BufferAtomicOpcode(Decoder::Opcode opcode); - IR::ValueOpcode SharedAtomicOpcode(Decoder::Opcode opcode); - bool TranslateScalarMemory(const Decoder::Instruction& inst, std::string* error); - bool TranslateBufferLoad(const Decoder::Instruction& inst, std::string* error); - bool TranslateBufferStore(const Decoder::Instruction& inst, std::string* error); - bool TranslateAtomicMemory(const Decoder::Instruction& inst, std::string* error); - bool TranslateFlatLoad(const Decoder::Instruction& inst, std::string* error); - bool TranslateFlatStore(const Decoder::Instruction& inst, std::string* error); - bool TranslateImageMemory(const Decoder::Instruction& inst, std::string* error); - bool TranslateSharedMemory(const Decoder::Instruction& inst, std::string* error); + IR::Value MakeImageAddress(const Decoder::Instruction& inst, const Decoder::Operand& base); + IR::Value ConstructU32x4(const Decoder::Operand& base, uint32_t count); + void WriteImageComponents(const Decoder::Operand& dst, IR::Value value, + const IR::MemoryInfo& memory, uint32_t component_limit); + BufferAddress ReadBufferAddress(const Decoder::Instruction& inst, uint32_t source_offset); + IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign); + IR::Value NarrowSubdword(IR::U32 value, uint32_t bits); + bool S_LOAD(const Decoder::Instruction& inst, bool raw); + bool BUFFER_LOAD(const Decoder::Instruction& inst); + bool BUFFER_STORE(const Decoder::Instruction& inst); + bool BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode); + bool IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode); + bool DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value); + bool FLAT_LOAD(const Decoder::Instruction& inst); + bool FLAT_STORE(const Decoder::Instruction& inst); + bool IMAGE_GET_RESINFO(const Decoder::Instruction& inst); + bool IMAGE_GET_LOD(const Decoder::Instruction& inst); + bool IMAGE_LOAD(const Decoder::Instruction& inst); + bool IMAGE_STORE(const Decoder::Instruction& inst); + bool IMAGE_SAMPLE(const Decoder::Instruction& inst); + bool IMAGE_GATHER(const Decoder::Instruction& inst); + IR::Value LoadSharedU32(uint32_t width, IR::U32 address, const IR::MemoryInfo& memory, + uint32_t pc); + IR::Value ExtractSharedU32(IR::Value value, uint32_t width, uint32_t index); + void WriteSharedU32(uint32_t width, IR::U32 address, const std::array& values, + const IR::MemoryInfo& memory, uint32_t pc); + bool DS_READ(const Decoder::Instruction& inst); + bool DS_READ2(const Decoder::Instruction& inst); + bool DS_WRITE(const Decoder::Instruction& inst); + bool DS_WRITE2(const Decoder::Instruction& inst); + bool DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode); + bool DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode); + bool DS_ADDTID(const Decoder::Instruction& inst, bool write); + bool DS_SWIZZLE_B32(const Decoder::Instruction& inst); IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value); IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper, @@ -101,27 +117,136 @@ private: float safe_upper, uint32_t lower_result, uint32_t upper_result); IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high); + void EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar, bool cmpx); + void EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar, bool cmpx); + void EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type, + bool scalar, bool cmpx); + void EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool signed_value, bool cmpx); + void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half, + bool cmpx); + void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered); + void V_CMP_CLASS_F32(const Decoder::Instruction& inst); + void V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index); + void V_CVT_F32_U32(const Decoder::Instruction& inst); + void V_CVT_F32_I32(const Decoder::Instruction& inst); + void V_CVT_U32_F32(const Decoder::Instruction& inst); + void V_CVT_I32_F32(const Decoder::Instruction& inst); + void V_CVT_F16_F32(const Decoder::Instruction& inst); + void V_CVT_F32_F16(const Decoder::Instruction& inst); + void V_CVT_F16_16(const Decoder::Instruction& inst, bool signed_value); + void V_CVT_16_F16(const Decoder::Instruction& inst, bool signed_value); + void V_CVT_RPI_I32_F32(const Decoder::Instruction& inst); + void V_CVT_FLR_I32_F32(const Decoder::Instruction& inst); + void V_FREXP_EXP_I32_F32(const Decoder::Instruction& inst); + void V_CVT_OFF_F32_I4(const Decoder::Instruction& inst); + void V_CVT_PKRTZ_F16_F32(const Decoder::Instruction& inst); + void V_CVT_PKNORM_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode); + void V_CVT_PK_U8_F32(const Decoder::Instruction& inst); + void V_PACK_B32_F16(const Decoder::Instruction& inst); + bool PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator, + bool quiet_snan); + bool Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool invalid_negative); + bool Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse); + bool Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator, + bool mix); + bool FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode); + bool FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse); + bool FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator, + bool mix); + bool V_FREXP_MANT_F32(const Decoder::Instruction& inst); + bool V_DOT2C_F32_F16(const Decoder::Instruction& inst); + bool V_CUBEID_F32(const Decoder::Instruction& inst); + bool V_CUBESC_F32(const Decoder::Instruction& inst); + bool V_CUBETC_F32(const Decoder::Instruction& inst); + bool V_CUBEMA_F32(const Decoder::Instruction& inst); + bool FloatCube(const Decoder::Instruction& inst, uint32_t result_kind); + bool Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool arithmetic); + bool Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign); + bool V_MED3_I16(const Decoder::Instruction& inst); + bool PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool arithmetic); + bool PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode); + bool PackedInteger16Mad(const Decoder::Instruction& inst, bool sign); + bool PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign); + bool S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode, + IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, bool unary); + IR::U1 U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool negate_rhs, + bool negate_result); + bool SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type, + bool reverse, bool mask_shift_count, bool update_scc); + bool ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, + bool negate_rhs, bool negate_result, bool update_scc); + bool V_AND_OR_B32(const Decoder::Instruction& inst); + bool V_OR3_B32(const Decoder::Instruction& inst); + bool V_XOR3_B32(const Decoder::Instruction& inst); + bool S_FF1_I32_B64(const Decoder::Instruction& inst); + bool V_FFBH_U32(const Decoder::Instruction& inst); + bool S_FLBIT_I32_B64(const Decoder::Instruction& inst); + bool Integer24(const Decoder::Instruction& inst, bool sign, bool addend); + bool V_MAD_U64_U32(const Decoder::Instruction& inst); + bool V_SAD_U32(const Decoder::Instruction& inst); + bool V_ADD3_U32(const Decoder::Instruction& inst); + bool S_BITSET_B32(const Decoder::Instruction& inst, bool set); + bool V_BCNT_U32_B32(const Decoder::Instruction& inst); + bool V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low); + bool S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst); + bool S_QUADMASK_B64(const Decoder::Instruction& inst); + bool BFM_B32(const Decoder::Instruction& inst); + IR::U32 RightMask32(IR::U32 count); + IR::U64 RightMask64(IR::U32 count); + bool S_BFM_B64(const Decoder::Instruction& inst); + bool S_BFE_U32(const Decoder::Instruction& inst, bool sign); + bool S_BFE_U64(const Decoder::Instruction& inst); + bool V_BFE_U32(const Decoder::Instruction& inst, bool sign); + bool V_BFI_B32(const Decoder::Instruction& inst); + bool S_BITCMP_B32(const Decoder::Instruction& inst, bool expected); + bool V_ALIGNBIT_B32(const Decoder::Instruction& inst); + bool V_ALIGNBYTE_B32(const Decoder::Instruction& inst); + bool V_LSHL_ADD_U32(const Decoder::Instruction& inst); + bool V_ADD_LSHL_U32(const Decoder::Instruction& inst); + bool V_XAD_U32(const Decoder::Instruction& inst); + bool V_LSHL_OR_B32(const Decoder::Instruction& inst); + bool V_CNDMASK_B32(const Decoder::Instruction& inst); + bool PackB16(const Decoder::Instruction& inst, bool high0, bool high1); - bool TranslateStateOperation(const Decoder::Instruction& inst); - bool TranslateControlOperation(const Decoder::Instruction& inst); - bool TranslateMove(const Decoder::Instruction& inst, std::string* error); - bool TranslateLaneOperation(const Decoder::Instruction& inst, std::string* error); - bool TranslateAttributeOperation(const Decoder::Instruction& inst, std::string* error); - bool TranslateMemoryOperation(const Decoder::Instruction& inst, std::string* error); - bool TranslateIntegerCompare(const Decoder::Instruction& inst); - bool TranslateInteger16Compare(const Decoder::Instruction& inst); - bool TranslateFloatCompare(const Decoder::Instruction& inst); - bool TranslateConversion(const Decoder::Instruction& inst); - bool TranslateInteger16Operation(const Decoder::Instruction& inst); - bool TranslatePackedInteger16(const Decoder::Instruction& inst); - bool TranslatePackedFloat16(const Decoder::Instruction& inst); - bool TranslateFloat16Operation(const Decoder::Instruction& inst); - bool TranslateFloatOperation(const Decoder::Instruction& inst); - IR::U1 EvaluateU64Mask(const Decoder::Instruction& inst); - bool TranslateU64MaskOperation(const Decoder::Instruction& inst); - bool TranslateSimpleInteger(const Decoder::Instruction& inst); - bool TranslateComposedInteger(const Decoder::Instruction& inst); - bool TranslateExtendedInteger(const Decoder::Instruction& inst); + void S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation, bool negate_exec, + bool negate_source, bool write_64); + void ADD_U32(const Decoder::Instruction& inst, bool vector, bool use_carry_in); + void SUB_U32(const Decoder::Instruction& inst, bool vector, bool reverse); + void SUBB_U32(const Decoder::Instruction& inst, bool vector); + void S_ADD_SUB_I32(const Decoder::Instruction& inst, bool subtract); + void S_LSHL_ADD_U32(const Decoder::Instruction& inst, uint32_t shift_amount); + void ScalarMinMax32(const Decoder::Instruction& inst, IR::ValueOpcode value_opcode, + IR::ValueOpcode compare_opcode); + void EmitControlNop(); + void EmitWaitcnt(); + void S_BARRIER(); + void S_SENDMSG(); + void S_TTRACEDATA(); + void S_INST_PREFETCH(); + bool S_GETPC_B64(const Decoder::Instruction& inst, std::string* error); + void S_CSELECT_B32(const Decoder::Instruction& inst); + void S_CSELECT_B64(const Decoder::Instruction& inst); + void MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers); + void S_MOV_B64(const Decoder::Instruction& inst); + void S_WQM_B64(const Decoder::Instruction& inst); + bool V_MOVRELS_B32(const Decoder::Instruction& inst, std::string* error); + bool V_MOVRELD_B32(const Decoder::Instruction& inst, std::string* error); + void V_READFIRSTLANE_B32(const Decoder::Instruction& inst); + void V_READLANE_B32(const Decoder::Instruction& inst); + void V_WRITELANE_B32(const Decoder::Instruction& inst); + void V_PERMLANE16_B32(const Decoder::Instruction& inst, bool x16); + void V_INTERP_P1_F32(); + void V_INTERP_P2_F32(const Decoder::Instruction& inst); + bool V_INTERP_MOV_F32(const Decoder::Instruction& inst, std::string* error); + bool EXP(const Decoder::Instruction& inst, std::string* error); + + bool EmitScalar(const Decoder::Instruction& inst, std::string* error); + bool EmitVector(const Decoder::Instruction& inst, std::string* error); + bool EmitInterpolation(const Decoder::Instruction& inst, std::string* error); + bool EmitMemory(const Decoder::Instruction& inst, std::string* error); IR::Program& program; IR::IREmitter ir; diff --git a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp new file mode 100644 index 000000000..c0100cf3f --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp @@ -0,0 +1,514 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +namespace Libs::Graphics::ShaderRecompiler::Frontend { + +bool Translator::EmitVector(const Decoder::Instruction& inst, std::string* error) { + using O = Decoder::Opcode; + switch (inst.opcode) { + case O::V_NOP: return true; + case O::V_ADD_I32: ADD_U32(inst, true, false); return true; + case O::V_ADDC_U32: ADD_U32(inst, true, true); return true; + case O::V_SUB_I32: SUB_U32(inst, true, false); return true; + case O::V_SUBREV_I32: SUB_U32(inst, true, true); return true; + case O::V_SUBREV_CO_CI_U32: SUBB_U32(inst, true); return true; + + case O::V_MOV_B32: MOV_B32(inst, true); return true; + case O::V_MOVRELS_B32: return V_MOVRELS_B32(inst, error); + case O::V_MOVRELD_B32: return V_MOVRELD_B32(inst, error); + case O::V_READFIRSTLANE_B32: V_READFIRSTLANE_B32(inst); return true; + case O::V_READLANE_B32: V_READLANE_B32(inst); return true; + case O::V_WRITELANE_B32: V_WRITELANE_B32(inst); return true; + case O::V_PERMLANE16_B32: V_PERMLANE16_B32(inst, false); return true; + case O::V_PERMLANEX16_B32: V_PERMLANE16_B32(inst, true); return true; + + case O::V_CMP_F_I32: + case O::V_CMP_F_U32: EmitCompareConstant(inst, false, false, false); return true; + case O::V_CMP_T_I32: + case O::V_CMP_T_U32: EmitCompareConstant(inst, true, false, false); return true; + case O::V_CMP_EQ_U32: + case O::V_CMP_EQ_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, false); + return true; + case O::V_CMPX_EQ_U32: + case O::V_CMPX_EQ_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, true); + return true; + case O::V_CMP_NE_U32: + case O::V_CMP_NE_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, false); + return true; + case O::V_CMPX_NE_U32: + case O::V_CMPX_NE_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, true); + return true; + case O::V_CMP_GT_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, false); + return true; + case O::V_CMPX_GT_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, true); + return true; + case O::V_CMP_GE_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false, + false); + return true; + case O::V_CMPX_GE_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false, + true); + return true; + case O::V_CMP_LT_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, false); + return true; + case O::V_CMPX_LT_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, true); + return true; + case O::V_CMP_LE_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false, + false); + return true; + case O::V_CMPX_LE_U32: + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false, true); + return true; + case O::V_CMP_GT_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, false); + return true; + case O::V_CMPX_GT_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, true); + return true; + case O::V_CMP_GE_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false, + false); + return true; + case O::V_CMPX_GE_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false, + true); + return true; + case O::V_CMP_LT_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, false); + return true; + case O::V_CMPX_LT_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, true); + return true; + case O::V_CMP_LE_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false, + false); + return true; + case O::V_CMPX_LE_I32: + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false, true); + return true; + case O::V_CMP_EQ_I64: + EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false, false); + return true; + case O::V_CMP_GT_U64: + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, false); + return true; + case O::V_CMP_NE_U64: + EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, false); + return true; + case O::V_CMPX_NE_I64: + case O::V_CMPX_NE_U64: + EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, true); + return true; + + case O::V_CMP_EQ_U16: + EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false); + return true; + case O::V_CMP_EQ_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, false); + return true; + case O::V_CMP_NE_U16: + EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false); + return true; + case O::V_CMP_NE_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false); + return true; + case O::V_CMP_GT_U16: + EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, false); + return true; + case O::V_CMP_GE_U16: + EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThanEqual32, false, false); + return true; + case O::V_CMP_LT_U16: + EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, false); + return true; + case O::V_CMP_LE_U16: + EmitInteger16Compare(inst, IR::ValueOpcode::ULessThanEqual32, false, false); + return true; + case O::V_CMP_GT_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, false); + return true; + case O::V_CMP_GE_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, false); + return true; + case O::V_CMP_LT_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, false); + return true; + case O::V_CMP_LE_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, false); + return true; + + case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return true; + case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return true; + case O::V_CMP_EQ_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, false); + return true; + case O::V_CMPX_EQ_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true); + return true; + case O::V_CMP_LG_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, false); + return true; + case O::V_CMPX_LG_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, true); + return true; + case O::V_CMP_GT_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, false); + return true; + case O::V_CMPX_GT_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, true); + return true; + case O::V_CMP_GE_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, false); + return true; + case O::V_CMPX_GE_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, true); + return true; + case O::V_CMP_LT_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, false); + return true; + case O::V_CMPX_LT_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, true); + return true; + case O::V_CMP_LE_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, false); + return true; + case O::V_CMPX_LE_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, true); + return true; + case O::V_CMP_NLG_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, false); + return true; + case O::V_CMPX_NLG_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, true); + return true; + case O::V_CMP_NEQ_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, false); + return true; + case O::V_CMPX_NEQ_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, true); + return true; + case O::V_CMP_NLE_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, false); + return true; + case O::V_CMPX_NLE_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, true); + return true; + case O::V_CMP_NLT_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, false); + return true; + case O::V_CMPX_NLT_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, true); + return true; + case O::V_CMP_NGE_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, false); + return true; + case O::V_CMPX_NGE_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, true); + return true; + case O::V_CMP_NGT_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, false); + return true; + case O::V_CMPX_NGT_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, true); + return true; + case O::V_CMP_EQ_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, false); + return true; + case O::V_CMPX_EQ_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, true); + return true; + case O::V_CMP_LG_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, true, false); + return true; + case O::V_CMP_GT_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, false); + return true; + case O::V_CMPX_GT_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, true); + return true; + case O::V_CMP_GE_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, false); + return true; + case O::V_CMPX_GE_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, true); + return true; + case O::V_CMP_LT_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, false); + return true; + case O::V_CMPX_LT_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, true); + return true; + case O::V_CMP_LE_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, false); + return true; + case O::V_CMPX_LE_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, true); + return true; + case O::V_CMP_NEQ_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, false); + return true; + case O::V_CMPX_NEQ_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, true); + return true; + case O::V_CMPX_NLT_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true, true); + return true; + case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true); return true; + case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false); return true; + case O::V_CMP_CLASS_F32: V_CMP_CLASS_F32(inst); return true; + + case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return true; + case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return true; + case O::V_CVT_F32_UBYTE2: V_CVT_F32_UBYTE(inst, 2); return true; + case O::V_CVT_F32_UBYTE3: V_CVT_F32_UBYTE(inst, 3); return true; + case O::V_CVT_F32_U32: V_CVT_F32_U32(inst); return true; + case O::V_CVT_F32_I32: V_CVT_F32_I32(inst); return true; + case O::V_CVT_U32_F32: V_CVT_U32_F32(inst); return true; + case O::V_CVT_I32_F32: V_CVT_I32_F32(inst); return true; + case O::V_CVT_F16_F32: V_CVT_F16_F32(inst); return true; + case O::V_CVT_F32_F16: V_CVT_F32_F16(inst); return true; + case O::V_CVT_F16_U16: V_CVT_F16_16(inst, false); return true; + case O::V_CVT_F16_I16: V_CVT_F16_16(inst, true); return true; + case O::V_CVT_U16_F16: V_CVT_16_F16(inst, false); return true; + case O::V_CVT_I16_F16: V_CVT_16_F16(inst, true); return true; + case O::V_CVT_RPI_I32_F32: V_CVT_RPI_I32_F32(inst); return true; + case O::V_CVT_FLR_I32_F32: V_CVT_FLR_I32_F32(inst); return true; + case O::V_FREXP_EXP_I32_F32: V_FREXP_EXP_I32_F32(inst); return true; + case O::V_CVT_OFF_F32_I4: V_CVT_OFF_F32_I4(inst); return true; + case O::V_CVT_PKRTZ_F16_F32: V_CVT_PKRTZ_F16_F32(inst); return true; + case O::V_CVT_PKNORM_I16_F32: + V_CVT_PKNORM_F32(inst, IR::ValueOpcode::PackSnorm2x16); + return true; + case O::V_CVT_PKNORM_U16_F32: + V_CVT_PKNORM_F32(inst, IR::ValueOpcode::PackUnorm2x16); + return true; + case O::V_CVT_PK_U8_F32: V_CVT_PK_U8_F32(inst); return true; + case O::V_PACK_B32_F16: V_PACK_B32_F16(inst); return true; + case O::V_CVT_PK_U16_U32: + case O::V_CVT_PK_I16_I32: return PackB16(inst, false, false); + + case O::V_LSHLREV_B16: + return Integer16Shift(inst, IR::ValueOpcode::ShiftLeftLogical32, false); + case O::V_LSHRREV_B16: + return Integer16Shift(inst, IR::ValueOpcode::ShiftRightLogical32, false); + case O::V_ASHRREV_I16: + return Integer16Shift(inst, IR::ValueOpcode::ShiftRightArithmetic32, true); + case O::V_ADD_NC_U16: + case O::V_ADD_NC_I16: return Integer16Binary(inst, IR::ValueOpcode::IAdd32, false); + case O::V_SUB_NC_U16: + case O::V_SUB_NC_I16: return Integer16Binary(inst, IR::ValueOpcode::ISub32, false); + case O::V_MED3_I16: return V_MED3_I16(inst); + case O::V_MIN_I16: return Integer16Binary(inst, IR::ValueOpcode::SMin32, true); + case O::V_MAX_I16: return Integer16Binary(inst, IR::ValueOpcode::SMax32, true); + case O::V_MIN_U16: return Integer16Binary(inst, IR::ValueOpcode::UMin32, false); + case O::V_MAX_U16: return Integer16Binary(inst, IR::ValueOpcode::UMax32, false); + + case O::V_PK_LSHLREV_B16: + return PackedInteger16Shift(inst, IR::ValueOpcode::ShiftLeftLogical32, false); + case O::V_PK_LSHRREV_B16: + return PackedInteger16Shift(inst, IR::ValueOpcode::ShiftRightLogical32, false); + case O::V_PK_ASHRREV_I16: + return PackedInteger16Shift(inst, IR::ValueOpcode::ShiftRightArithmetic32, true); + case O::V_PK_MAD_I16: return PackedInteger16Mad(inst, true); + case O::V_PK_MAD_U16: return PackedInteger16Mad(inst, false); + case O::V_PK_MUL_LO_U16: return PackedInteger16Binary(inst, IR::ValueOpcode::IMul32); + case O::V_PK_ADD_I16: + case O::V_PK_ADD_U16: return PackedInteger16Binary(inst, IR::ValueOpcode::IAdd32); + case O::V_PK_SUB_I16: + case O::V_PK_SUB_U16: return PackedInteger16Binary(inst, IR::ValueOpcode::ISub32); + case O::V_PK_MAX_I16: return PackedInteger16MinMax(inst, IR::ValueOpcode::SMax32, true); + case O::V_PK_MIN_I16: return PackedInteger16MinMax(inst, IR::ValueOpcode::SMin32, true); + case O::V_PK_MAX_U16: return PackedInteger16MinMax(inst, IR::ValueOpcode::UMax32, false); + case O::V_PK_MIN_U16: return PackedInteger16MinMax(inst, IR::ValueOpcode::UMin32, false); + + case O::V_PK_ADD_F16: return PackedFloat16(inst, IR::ValueOpcode::FPAdd32, false, false); + case O::V_PK_MUL_F16: return PackedFloat16(inst, IR::ValueOpcode::FPMul32, false, false); + case O::V_PK_MIN_F16: return PackedFloat16(inst, IR::ValueOpcode::FPMin32, false, true); + case O::V_PK_MAX_F16: return PackedFloat16(inst, IR::ValueOpcode::FPMax32, false, true); + case O::V_PK_FMA_F16: return PackedFloat16(inst, IR::ValueOpcode::FPFma32, false, false); + case O::V_PK_FMAC_F16: return PackedFloat16(inst, IR::ValueOpcode::FPFma32, true, false); + + case O::V_ADD_F16: return Float16Binary(inst, IR::ValueOpcode::FPAdd32, false); + case O::V_SUB_F16: return Float16Binary(inst, IR::ValueOpcode::FPSub32, false); + case O::V_SUBREV_F16: return Float16Binary(inst, IR::ValueOpcode::FPSub32, true); + case O::V_MUL_F16: return Float16Binary(inst, IR::ValueOpcode::FPMul32, false); + case O::V_MIN_F16: return Float16Binary(inst, IR::ValueOpcode::FPMin32, false); + case O::V_MAX_F16: return Float16Binary(inst, IR::ValueOpcode::FPMax32, false); + case O::V_FMAC_F16: return Float16Ternary(inst, IR::ValueOpcode::FPFma32, true, false); + case O::V_FMAMK_F16: + case O::V_FMAAK_F16: + case O::V_FMA_F16: return Float16Ternary(inst, IR::ValueOpcode::FPFma32, false, false); + case O::V_MAD_MIXLO_F16: + case O::V_MAD_MIXHI_F16: return Float16Ternary(inst, IR::ValueOpcode::FPFma32, false, true); + case O::V_RCP_F16: return Float16Unary(inst, IR::ValueOpcode::FPRecip32, false); + case O::V_SQRT_F16: return Float16Unary(inst, IR::ValueOpcode::FPSqrt, true); + case O::V_RSQ_F16: return Float16Unary(inst, IR::ValueOpcode::FPRecipSqrt32, true); + case O::V_LOG_F16: return Float16Unary(inst, IR::ValueOpcode::FPLog2, true); + case O::V_EXP_F16: return Float16Unary(inst, IR::ValueOpcode::FPExp2, false); + case O::V_FLOOR_F16: return Float16Unary(inst, IR::ValueOpcode::FPFloor32, false); + case O::V_CEIL_F16: return Float16Unary(inst, IR::ValueOpcode::FPCeil32, false); + case O::V_TRUNC_F16: return Float16Unary(inst, IR::ValueOpcode::FPTrunc32, false); + case O::V_RNDNE_F16: return Float16Unary(inst, IR::ValueOpcode::FPRoundEven32, false); + case O::V_MIN3_F16: return Float16Ternary(inst, IR::ValueOpcode::FPMinTri32, false, false); + case O::V_MAX3_F16: return Float16Ternary(inst, IR::ValueOpcode::FPMaxTri32, false, false); + case O::V_MED3_F16: return Float16Ternary(inst, IR::ValueOpcode::FPMedTri32, false, false); + + case O::V_FREXP_MANT_F32: return V_FREXP_MANT_F32(inst); + case O::V_RCP_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecip32); + case O::V_RCP_IFLAG_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipIFlag32); + case O::V_FRACT_F32: return FloatUnary(inst, IR::ValueOpcode::FPFract32); + case O::V_TRUNC_F32: return FloatUnary(inst, IR::ValueOpcode::FPTrunc32); + case O::V_CEIL_F32: return FloatUnary(inst, IR::ValueOpcode::FPCeil32); + case O::V_RNDNE_F32: return FloatUnary(inst, IR::ValueOpcode::FPRoundEven32); + case O::V_FLOOR_F32: return FloatUnary(inst, IR::ValueOpcode::FPFloor32); + case O::V_EXP_F32: return FloatUnary(inst, IR::ValueOpcode::FPExp2); + case O::V_LOG_F32: return FloatUnary(inst, IR::ValueOpcode::FPLog2); + case O::V_RSQ_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipSqrt32); + case O::V_SQRT_F32: return FloatUnary(inst, IR::ValueOpcode::FPSqrt); + case O::V_SIN_F32: return FloatUnary(inst, IR::ValueOpcode::FPSin); + case O::V_COS_F32: return FloatUnary(inst, IR::ValueOpcode::FPCos); + case O::V_ADD_F32: return FloatBinary(inst, IR::ValueOpcode::FPAdd32, false); + case O::V_SUB_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, false); + case O::V_SUBREV_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, true); + case O::V_MUL_F32: return FloatBinary(inst, IR::ValueOpcode::FPMul32, false); + case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false); + case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false); + case O::V_LDEXP_F32: return FloatBinary(inst, IR::ValueOpcode::FPLdexp, false); + case O::V_MAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, true, true); + case O::V_MADMK_F32: + case O::V_MADAK_F32: + case O::V_MAD_F32: + case O::V_FMA_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, false, true); + case O::V_MIN3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMinTri32, false, false); + case O::V_MAX3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMaxTri32, false, false); + case O::V_MED3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMedTri32, false, false); + case O::V_DOT2C_F32_F16: return V_DOT2C_F32_F16(inst); + case O::V_CUBEID_F32: return V_CUBEID_F32(inst); + case O::V_CUBESC_F32: return V_CUBESC_F32(inst); + case O::V_CUBETC_F32: return V_CUBETC_F32(inst); + case O::V_CUBEMA_F32: return V_CUBEMA_F32(inst); + + case O::V_MUL_LO_U32: + case O::V_MUL_LO_I32: + return SimpleInteger(inst, IR::ValueOpcode::IMul32, IR::Type::U32, false, false, false); + case O::V_MUL_HI_U32: + return SimpleInteger(inst, IR::ValueOpcode::UMulHi, IR::Type::U32, false, false, false); + case O::V_MUL_HI_I32: + return SimpleInteger(inst, IR::ValueOpcode::SMulHi, IR::Type::U32, false, false, false); + case O::V_ADD_NC_U32: + return SimpleInteger(inst, IR::ValueOpcode::IAdd32, IR::Type::U32, false, false, false); + case O::V_SUB_NC_U32: + return SimpleInteger(inst, IR::ValueOpcode::ISub32, IR::Type::U32, false, false, false); + case O::V_SUBREV_NC_U32: + return SimpleInteger(inst, IR::ValueOpcode::ISub32, IR::Type::U32, true, false, false); + case O::V_MIN_I32: + return SimpleInteger(inst, IR::ValueOpcode::SMin32, IR::Type::U32, false, false, false); + case O::V_MAX_I32: + return SimpleInteger(inst, IR::ValueOpcode::SMax32, IR::Type::U32, false, false, false); + case O::V_MIN_U32: + return SimpleInteger(inst, IR::ValueOpcode::UMin32, IR::Type::U32, false, false, false); + case O::V_MAX_U32: + return SimpleInteger(inst, IR::ValueOpcode::UMax32, IR::Type::U32, false, false, false); + case O::V_MIN3_I32: + return SimpleInteger(inst, IR::ValueOpcode::SMinTri32, IR::Type::U32, false, false, + false); + case O::V_MAX3_I32: + return SimpleInteger(inst, IR::ValueOpcode::SMaxTri32, IR::Type::U32, false, false, + false); + case O::V_MED3_I32: + return SimpleInteger(inst, IR::ValueOpcode::SMedTri32, IR::Type::U32, false, false, + false); + case O::V_MIN3_U32: + return SimpleInteger(inst, IR::ValueOpcode::UMinTri32, IR::Type::U32, false, false, + false); + case O::V_MAX3_U32: + return SimpleInteger(inst, IR::ValueOpcode::UMaxTri32, IR::Type::U32, false, false, + false); + case O::V_MED3_U32: + return SimpleInteger(inst, IR::ValueOpcode::UMedTri32, IR::Type::U32, false, false, + false); + case O::V_AND_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitwiseAnd32, IR::Type::U32, false, false, + false); + case O::V_OR_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitwiseOr32, IR::Type::U32, false, false, + false); + case O::V_XOR_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitwiseXor32, IR::Type::U32, false, false, + false); + case O::V_NOT_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitwiseNot32, IR::Type::U32, false, false, + false); + case O::V_BFREV_B32: + return SimpleInteger(inst, IR::ValueOpcode::BitReverse32, IR::Type::U32, false, false, + false); + case O::V_FFBL_B32: + return SimpleInteger(inst, IR::ValueOpcode::FindILsb32, IR::Type::U32, false, false, + false); + case O::V_LSHL_B32: + return SimpleInteger(inst, IR::ValueOpcode::ShiftLeftLogical32, IR::Type::U32, false, + true, false); + case O::V_LSHLREV_B32: + return SimpleInteger(inst, IR::ValueOpcode::ShiftLeftLogical32, IR::Type::U32, true, + true, false); + case O::V_LSHR_B32: + return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical32, IR::Type::U32, false, + true, false); + case O::V_LSHRREV_B32: + return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical32, IR::Type::U32, true, + true, false); + case O::V_ASHR_I32: + return SimpleInteger(inst, IR::ValueOpcode::ShiftRightArithmetic32, IR::Type::U32, + false, true, false); + case O::V_ASHRREV_I32: + return SimpleInteger(inst, IR::ValueOpcode::ShiftRightArithmetic32, IR::Type::U32, true, + true, false); + case O::V_LSHLREV_B64: + return SimpleInteger(inst, IR::ValueOpcode::ShiftLeftLogical64, IR::Type::U64, true, + false, false); + case O::V_LSHRREV_B64: + return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical64, IR::Type::U64, true, + false, false); + + case O::V_XNOR_B32: + return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseXor32, false, true, false); + case O::V_AND_OR_B32: return V_AND_OR_B32(inst); + case O::V_OR3_B32: return V_OR3_B32(inst); + case O::V_XOR3_B32: return V_XOR3_B32(inst); + case O::V_FFBH_U32: return V_FFBH_U32(inst); + + case O::V_MAD_I32_I24: return Integer24(inst, true, true); + case O::V_MAD_U32_U24: return Integer24(inst, false, true); + case O::V_MUL_I32_I24: return Integer24(inst, true, false); + case O::V_MUL_U32_U24: return Integer24(inst, false, false); + case O::V_MAD_U64_U32: return V_MAD_U64_U32(inst); + case O::V_SAD_U32: return V_SAD_U32(inst); + case O::V_ADD3_U32: return V_ADD3_U32(inst); + case O::V_BCNT_U32_B32: return V_BCNT_U32_B32(inst); + case O::V_MBCNT_LO_U32_B32: return V_MBCNT_U32_B32(inst, true); + case O::V_MBCNT_HI_U32_B32: return V_MBCNT_U32_B32(inst, false); + case O::V_BFM_B32: return BFM_B32(inst); + case O::V_BFE_U32: return V_BFE_U32(inst, false); + case O::V_BFE_I32: return V_BFE_U32(inst, true); + case O::V_BFI_B32: return V_BFI_B32(inst); + case O::V_ALIGNBIT_B32: return V_ALIGNBIT_B32(inst); + case O::V_ALIGNBYTE_B32: return V_ALIGNBYTE_B32(inst); + case O::V_LSHL_ADD_U32: return V_LSHL_ADD_U32(inst); + case O::V_ADD_LSHL_U32: return V_ADD_LSHL_U32(inst); + case O::V_XAD_U32: return V_XAD_U32(inst); + case O::V_LSHL_OR_B32: return V_LSHL_OR_B32(inst); + case O::V_CNDMASK_B32: return V_CNDMASK_B32(inst); + default: return false; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index ba2d7a75c..17a09af52 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -4107,36 +4107,7 @@ void TestNewShaderRecompilerIrLookupMissFailsExplicitly() { "early translation failure retained a stale output program"); options.wave_size = 64u; - decoded.instructions.front().family = - ShaderRecompiler::Decoder::Family::MUBUF; - decoded.instructions.front().opcode = - ShaderRecompiler::Decoder::Opcode::V_ADD_NC_U32; - error.clear(); - Check(!ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, - &error), - "memory-family opcode bypassed specialized memory translation"); - Check(Common::ContainsStr(error, "does not belong to decoded family"), - "memory-family mismatch did not report an explicit error"); - auto &store = decoded.instructions.front(); - store.opcode = ShaderRecompiler::Decoder::Opcode::BUFFER_STORE_DWORD; - store.data_dwords = 1u; - store.data_bits = 32u; - store.offen = true; - store.src_count = 3u; - store.src0.kind = ShaderRecompiler::Decoder::OperandKind::Vgpr; - store.src1.kind = ShaderRecompiler::Decoder::OperandKind::Sgpr; - store.src2.kind = - ShaderRecompiler::Decoder::OperandKind::IntegerInlineConstant; - store.dst.kind = ShaderRecompiler::Decoder::OperandKind::Unknown; - error.clear(); - Check(!ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options, ir, - &error), - "buffer store accepted an unreadable data operand"); - Check(Common::ContainsStr(error, - "decoded operand cannot be used as an IR register"), - "buffer store data rejection did not report an explicit error"); - store = {}; store.family = ShaderRecompiler::Decoder::Family::MUBUF; store.opcode = ShaderRecompiler::Decoder::Opcode::UNSUPPORTED; @@ -5930,29 +5901,6 @@ void TestNewShaderRecompilerNativeWideScalarMemoryIr() { Check(buffer_x2_targets_vcc[0] && buffer_x2_targets_vcc[1], "s_buffer_load_dwordx2 did not preserve its VCC destination span"); - const auto reject = [&](uint32_t opcode, uint32_t dst, uint32_t sbase, - const char *reason) { - const uint32_t invalid[] = {EncodeSmem0(opcode, dst, sbase), 125u << 25u, - EncodeSopp(0x01)}; - ShaderRecompiler::Decoder::Program invalid_decoded; - ShaderRecompiler::CFG::Graph invalid_graph; - ShaderRecompiler::IR::Program invalid_ir; - std::string invalid_error; - Check(ShaderRecompiler::Decoder::DecodeProgram(invalid, invalid_decoded, - &invalid_error) && - ShaderRecompiler::CFG::BuildGraph(invalid_decoded, invalid_graph, - &invalid_error) && - !ShaderRecompiler::Frontend::TranslateProgram( - invalid_decoded, invalid_graph, translate_options, invalid_ir, - &invalid_error) && - Common::ContainsStr(invalid_error, reason), - "invalid SMEM register span was not rejected"); - }; - reject(0x01, 1, 4, "destination"); - reject(0x02, 2, 4, "destination"); - reject(0x02, 124, 4, "destination"); - reject(0x00, 0, 63, "base"); - reject(0x08, 0, 1, "base"); } void TestNewShaderRecompilerNativeWideBufferIr() {