From 6758d679422ae0022b4444791754b595abddc3a1 Mon Sep 17 00:00:00 2001 From: nomolao2-cell Date: Sun, 30 Aug 2026 17:04:16 -0600 Subject: [PATCH] shader: preserve raw VCC bits in S_MOV_B64 (#418) * shader: preserve raw VCC bits in S_MOV_B64 * fix(shader): centralize scalar U64 mask provenance --------- Co-authored-by: nmzik --- .../recompiler/frontend/translate/Compare.cpp | 1 + .../recompiler/frontend/translate/Control.cpp | 100 ++++------ .../recompiler/frontend/translate/Integer.cpp | 65 +----- .../recompiler/frontend/translate/Memory.cpp | 16 +- .../frontend/translate/Translate.cpp | 123 +++++++++--- .../frontend/translate/Translator.h | 39 ++-- .../shader/recompiler/ir/IREmitter.cpp | 16 ++ src/graphics/shader/recompiler/ir/IREmitter.h | 4 + src/graphics/shader/recompiler/ir/Program.cpp | 4 + .../recompiler/ir/opcodes/ValueOpcodes.inc | 4 + .../recompiler/ir/passes/SsaRewrite.cpp | 42 +++- tests/ShaderRecompilerComputeTests.cpp | 187 +++++++++++++++++- tests/shaderCfgTests.cpp | 28 +-- 13 files changed, 444 insertions(+), 185 deletions(-) diff --git a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp index 4006374d8..2c1efef2f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp @@ -12,6 +12,7 @@ void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 valu if (cmpx) { const auto mask = BallotMask(masked); ir.SetExec(masked); + ir.SetExecMaskTag(IR::U1(IR::Value(true))); ir.SetExecLo(mask[0]); ir.SetExecHi(mask[1]); return; diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index 9b2d7eaa9..c718e3812 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -23,18 +23,28 @@ Decoder::Operand ConditionOperand(Decoder::OperandKind kind) { void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation, bool negate_exec, bool negate_source, bool write_64) { + if (write_64) { + const auto exec_operand = ConditionOperand(Decoder::OperandKind::ExecLo); + const auto old = ReadScalarU64(exec_operand); + const auto bit_operation = operation == IR::ValueOpcode::LogicalAnd + ? IR::ValueOpcode::BitwiseAnd32 + : IR::ValueOpcode::BitwiseOr32; + const auto result = BinaryScalarU64(old, ReadScalarU64(inst.src0), operation, bit_operation, + negate_exec, negate_source, false); + WriteScalarU64(inst.dst, old); + WriteScalarU64(exec_operand, result); + ir.SetScc(ScalarU64NonZero(result)); + return; + } const auto old = ir.GetExec(); const auto src = ReadMask(inst.src0); const auto lhs = negate_exec ? ir.LogicalNot(old) : old; const auto rhs = negate_source ? ir.LogicalNot(src) : src; const auto result = IR::U1(ir.Emit(operation, {lhs, rhs})); - if (write_64) { - WriteMask64(inst.dst, old); - } else { - WriteMask(inst.dst, old); - } + WriteMask(inst.dst, old); const auto mask = BallotMask(result); ir.SetExec(result); + ir.SetExecMaskTag(IR::U1(IR::Value(true))); ir.SetExecLo(mask[0]); ir.SetExecHi(mask[1]); ir.SetScc(result); @@ -196,30 +206,21 @@ void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) { } void Translator::S_CSELECT_B64(const Decoder::Instruction& inst) { - const auto condition = ir.GetScc(); - const auto lhs = ReadU32Pair(inst.src0); - const auto rhs = ReadU32Pair(inst.src1); - const auto selected_mask = IR::U1( - ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMask(inst.src0), ReadMask(inst.src1)})); - const auto selected_mask_valid = - IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, - {condition, ReadMaskValid(inst.src0), ReadMaskValid(inst.src1)})); - if (IsExecOrVcc(inst.dst)) { - WriteMask64(inst.dst, selected_mask); - return; - } - WriteU32Pair(inst.dst, - {ir.Select(condition, lhs[0], rhs[0]), ir.Select(condition, lhs[1], rhs[1])}); - if (inst.dst.kind == Decoder::OperandKind::Sgpr) { - const auto dst = static_cast(inst.dst.reg); - ir.SetThreadBitScalarReg(dst, selected_mask); - ir.SetScalarMaskTag(dst, selected_mask_valid); - } + const auto condition = ir.GetScc(); + const auto lhs = ReadScalarU64(inst.src0); + const auto rhs = ReadScalarU64(inst.src1); + WriteScalarU64(inst.dst, SelectScalarU64(condition, lhs, rhs)); } void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers) { + if (inst.src0.kind == Decoder::OperandKind::Sgpr && inst.dst.kind == Decoder::OperandKind::Sgpr && + inst.src0.reg == inst.dst.reg) { + return; + } if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) { - WriteMask(inst.dst, ReadMask(inst.src0)); + if (inst.src0.kind != inst.dst.kind) { + WriteRawU32(inst.dst, ReadRawU32(inst.src0)); + } } else if (apply_float_modifiers && (inst.src0.negate || inst.src0.absolute)) { WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::F32)); } else { @@ -228,50 +229,17 @@ void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modi } void Translator::S_MOV_B64(const Decoder::Instruction& inst) { - if (IsExecOrVcc(inst.dst) || IsExecOrVcc(inst.src0)) { - WriteMask64(inst.dst, ReadMask(inst.src0)); - return; - } - const bool scalar_copy = - inst.dst.kind == Decoder::OperandKind::Sgpr && inst.src0.kind == Decoder::OperandKind::Sgpr; - IR::U1 source_mask; - IR::U1 source_mask_valid; - if (scalar_copy) { - source_mask = ir.GetThreadBitScalarReg(static_cast(inst.src0.reg)); - source_mask_valid = ir.GetScalarMaskTag(static_cast(inst.src0.reg)); - } - WriteU32Pair(inst.dst, ReadU32Pair(inst.src0)); - if (scalar_copy) { - ir.SetThreadBitScalarReg(static_cast(inst.dst.reg), source_mask); - ir.SetScalarMaskTag(static_cast(inst.dst.reg), source_mask_valid); - } + WriteScalarU64(inst.dst, ReadScalarU64(inst.src0)); } void Translator::S_WQM_B64(const Decoder::Instruction& inst) { - if (!IsExecOrVcc(inst.dst) && !IsExecOrVcc(inst.src0)) { - const auto mask_valid = ReadMaskValid(inst.src0); - const auto invocation_result = - IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)})); - const auto result = - IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)})); - WriteOperand(DestinationOperand(inst), result); - if (inst.dst.kind == Decoder::OperandKind::Sgpr) { - const auto dst = static_cast(inst.dst.reg); - ir.SetThreadBitScalarReg(dst, invocation_result); - ir.SetScalarMaskTag(dst, mask_valid); - const auto raw_nonzero = - IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})); - ir.SetScc(IR::U1( - ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero}))); - } else { - ir.SetScc( - IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); - } - return; - } - const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)})); - WriteMask64(inst.dst, result); - ir.SetScc(result); + const auto source = ReadScalarU64(inst.src0); + const auto raw = ExtractU64( + IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ir.ConstructU64(source.raw[0], source.raw[1])}))); + const auto invocation = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {source.invocation})); + const auto result = MakeScalarU64Result(raw, invocation, source.mask_valid); + WriteScalarU64(inst.dst, result); + ir.SetScc(ScalarU64NonZero(result)); } bool Translator::V_MOVRELS_B32(const Decoder::Instruction& inst, std::string* error) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp index 39dc0b6b3..cf8ddfdcf 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -77,68 +77,15 @@ bool Translator::PackedInteger16MinMax(const Decoder::Instruction& inst, IR::Val return true; } -IR::U1 Translator::U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, - bool negate_rhs, bool negate_result) { - const auto lhs = ReadMask(inst.src0); - auto rhs = ReadMask(inst.src1); - if (negate_rhs) { - rhs = ir.LogicalNot(rhs); - } - auto result = IR::U1(ir.Emit(opcode, {lhs, rhs})); - return negate_result ? ir.LogicalNot(result) : result; -} - bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode, IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, bool unary) { - const auto invocation_result = - unary ? ir.LogicalNot(ReadMask(inst.src0)) - : U64MaskBinary(inst, logical_opcode, negate_rhs, negate_result); - const auto is_exec_or_vcc = [](const Decoder::Operand& operand) { - switch (operand.kind) { - case Decoder::OperandKind::ExecLo: - case Decoder::OperandKind::ExecHi: - case Decoder::OperandKind::VccLo: - case Decoder::OperandKind::VccHi: return true; - default: return false; - } - }; - if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) || - (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) { - WriteMask64(inst.dst, invocation_result); - ir.SetScc(invocation_result); - return true; - } - - const auto lhs = ReadU32Pair(inst.src0); - auto mask_valid = ReadMaskValid(inst.src0); - if (inst.src_count > 1u) { - mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src1)); - } - std::array result; - if (unary) { - result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])}; - } else { - const auto rhs = ReadU32Pair(inst.src1); - for (uint32_t component = 0; component < 2u; component++) { - auto value = ir.Emit( - bit_opcode, {lhs[component], negate_rhs ? IR::Value(ir.BitwiseNot(rhs[component])) - : IR::Value(rhs[component])}); - result[component] = negate_result ? ir.BitwiseNot(IR::U32(value)) : IR::U32(value); - } - } - WriteU32Pair(inst.dst, result); - if (inst.dst.kind == Decoder::OperandKind::Sgpr) { - const auto dst = static_cast(inst.dst.reg); - ir.SetThreadBitScalarReg(dst, invocation_result); - ir.SetScalarMaskTag(dst, mask_valid); - const auto raw_nonzero = - ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))); - ir.SetScc(IR::U1( - ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero}))); - } else { - ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)))); - } + const auto lhs = ReadScalarU64(inst.src0); + const auto result = unary ? NotScalarU64(lhs) + : BinaryScalarU64(lhs, ReadScalarU64(inst.src1), logical_opcode, + bit_opcode, false, negate_rhs, negate_result); + WriteScalarU64(inst.dst, result); + ir.SetScc(ScalarU64NonZero(result)); return true; } diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 123010712..cfc2a3c69 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -57,11 +57,14 @@ uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) { } } -uint32_t RawScalarLoadBase(const Decoder::Operand& operand) { - if (operand.kind == Decoder::OperandKind::Sgpr) { - return operand.reg; +Decoder::Operand RawScalarLoadBase(const Decoder::Operand& operand) { + if (operand.kind != Decoder::OperandKind::M0) { + return operand; } - return operand.kind == Decoder::OperandKind::VccLo ? 106u : 0u; + auto result = operand; + // The compressed SBASE value 62 decodes as code 124, but means NULL for S_LOAD. + result.kind = Decoder::OperandKind::Null; + return result; } ResourceKind FlatSegmentResourceKind(uint32_t segment) { @@ -321,8 +324,9 @@ Translator::AddressOperands Translator::ReadAddressOperands(const Decoder::Instr return {GetAddressResource(low, high), low, high}; } -IR::Value Translator::GetScalarAddressResource(uint32_t base) { - return GetAddressResource(ReadScalarCode(base), ReadScalarCode(base + 1u)); +IR::Value Translator::GetScalarAddressResource(const Decoder::Operand& base) { + const auto raw = ReadU32Pair(base); + return GetAddressResource(raw[0], raw[1]); } IR::Value Translator::GetImageResource(const IR::MemoryInfo& memory) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index e0f09499f..f0ae7b1ea 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -153,11 +153,8 @@ IR::U32 Translator::ReadScalarCode(uint32_t code) { case 106u: return ir.GetVccLo(); case 107u: return ir.GetVccHi(); case 124u: return ir.GetM0(); - case 126u: - case 127u: { - const auto mask = BallotMask(ir.GetExec()); - return mask[code - 126u]; - } + case 126u: return ir.GetExecLo(); + case 127u: return ir.GetExecHi(); default: return IR::U32(IR::Value(0u)); } } @@ -317,19 +314,23 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) { case Decoder::OperandKind::VccLo: ir.SetVccLo(IR::U32(value)); ir.SetVcc(ThreadBit(IR::U32(value))); + ir.SetVccMaskValidTag(IR::U1(IR::Value(false))); break; case Decoder::OperandKind::VccHi: ir.SetVccHi(IR::U32(value)); ir.SetVcc(ThreadBit(ir.GetVccLo())); + ir.SetVccMaskValidTag(IR::U1(IR::Value(false))); break; case Decoder::OperandKind::M0: ir.SetM0(IR::U32(value)); break; case Decoder::OperandKind::ExecLo: ir.SetExecLo(IR::U32(value)); ir.SetExec(ThreadBit(IR::U32(value))); + ir.SetExecMaskTag(IR::U1(IR::Value(false))); break; case Decoder::OperandKind::ExecHi: ir.SetExecHi(IR::U32(value)); ir.SetExec(ThreadBit(ir.GetExecLo())); + ir.SetExecMaskTag(IR::U1(IR::Value(false))); break; case Decoder::OperandKind::Scc: ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); @@ -374,6 +375,7 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value) case Decoder::OperandKind::ExecHi: { const auto mask = BallotMask(IR::U1(value)); ir.SetExec(IR::U1(value)); + ir.SetExecMaskTag(IR::U1(IR::Value(true))); ir.SetExecLo(mask[0]); ir.SetExecHi(mask[1]); return; @@ -382,6 +384,7 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value) case Decoder::OperandKind::VccHi: { const auto mask = BallotMask(IR::U1(value)); ir.SetVcc(IR::U1(value)); + ir.SetVccMaskValidTag(IR::U1(IR::Value(true))); ir.SetVccLo(mask[0]); ir.SetVccHi(mask[1]); return; @@ -583,11 +586,13 @@ void Translator::WriteU32Pair(const Decoder::Operand& operand, switch (operand.kind) { case Decoder::OperandKind::ExecLo: ir.SetExec(thread_bit); + ir.SetExecMaskTag(IR::U1(IR::Value(false))); ir.SetExecLo(value[0]); ir.SetExecHi(value[1]); return; case Decoder::OperandKind::VccLo: ir.SetVcc(thread_bit); + ir.SetVccMaskValidTag(IR::U1(IR::Value(false))); ir.SetVccLo(value[0]); ir.SetVccHi(value[1]); return; @@ -598,6 +603,87 @@ void Translator::WriteU32Pair(const Decoder::Operand& operand, WriteRawU32(OffsetOperand(operand, 1), value[1]); } +Translator::ScalarU64 Translator::ReadScalarU64(const Decoder::Operand& operand) { + return {ReadU32Pair(operand), ReadMask(operand), ReadMaskValid(operand)}; +} + +Translator::ScalarU64 Translator::MakeScalarU64Mask(IR::U1 value) { + return {BallotMask(value), value, IR::U1(IR::Value(true))}; +} + +Translator::ScalarU64 Translator::MakeScalarU64Result(const std::array& raw, + IR::U1 invocation, IR::U1 mask_valid) { + return {raw, + IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation, ThreadBit(raw[0])})), + mask_valid}; +} + +Translator::ScalarU64 Translator::SelectScalarU64(IR::U1 condition, const ScalarU64& lhs, + const ScalarU64& rhs) { + return { + {ir.Select(condition, lhs.raw[0], rhs.raw[0]), + ir.Select(condition, lhs.raw[1], rhs.raw[1])}, + IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, lhs.invocation, rhs.invocation})), + IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, lhs.mask_valid, rhs.mask_valid}))}; +} + +Translator::ScalarU64 Translator::NotScalarU64(const ScalarU64& value) { + return MakeScalarU64Result({ir.BitwiseNot(value.raw[0]), ir.BitwiseNot(value.raw[1])}, + ir.LogicalNot(value.invocation), value.mask_valid); +} + +Translator::ScalarU64 Translator::BinaryScalarU64(const ScalarU64& lhs, const ScalarU64& rhs, + IR::ValueOpcode logical_opcode, + IR::ValueOpcode bit_opcode, bool negate_lhs, + bool negate_rhs, bool negate_result) { + auto lhs_mask = negate_lhs ? ir.LogicalNot(lhs.invocation) : lhs.invocation; + auto rhs_mask = negate_rhs ? ir.LogicalNot(rhs.invocation) : rhs.invocation; + auto mask = IR::U1(ir.Emit(logical_opcode, {lhs_mask, rhs_mask})); + if (negate_result) { + mask = ir.LogicalNot(mask); + } + std::array raw; + for (uint32_t component = 0; component < raw.size(); component++) { + auto lhs_raw = negate_lhs ? ir.BitwiseNot(lhs.raw[component]) : lhs.raw[component]; + auto rhs_raw = negate_rhs ? ir.BitwiseNot(rhs.raw[component]) : rhs.raw[component]; + raw[component] = IR::U32(ir.Emit(bit_opcode, {lhs_raw, rhs_raw})); + if (negate_result) { + raw[component] = ir.BitwiseNot(raw[component]); + } + } + return MakeScalarU64Result(raw, mask, ir.LogicalAnd(lhs.mask_valid, rhs.mask_valid)); +} + +void Translator::WriteScalarU64(const Decoder::Operand& operand, const ScalarU64& value) { + WriteU32Pair(operand, value.raw); + switch (operand.kind) { + case Decoder::OperandKind::ExecLo: + case Decoder::OperandKind::ExecHi: + ir.SetExec(value.invocation); + ir.SetExecMaskTag(value.mask_valid); + break; + case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccHi: + ir.SetVcc(value.invocation); + ir.SetVccMaskValidTag(value.mask_valid); + break; + case Decoder::OperandKind::Sgpr: { + const auto reg = static_cast(operand.reg); + ir.SetThreadBitScalarReg(reg, value.invocation); + ir.SetScalarMaskTag(reg, value.mask_valid); + break; + } + default: break; + } +} + +IR::U1 Translator::ScalarU64NonZero(const ScalarU64& value) { + const auto raw_nonzero = + ir.INotEqual(ir.BitwiseOr(value.raw[0], value.raw[1]), IR::U32(IR::Value(0u))); + return IR::U1( + ir.Emit(IR::ValueOpcode::SelectU1, {value.mask_valid, value.invocation, raw_nonzero})); +} + IR::U1 Translator::ThreadBit(IR::U32 low) { return ir.INotEqual(low, IR::U32(IR::Value(0u))); } @@ -653,9 +739,9 @@ IR::U1 Translator::ReadMaskValid(const Decoder::Operand& operand) { case Decoder::OperandKind::Sgpr: return ir.GetScalarMaskTag(static_cast(operand.reg)); case Decoder::OperandKind::ExecLo: - case Decoder::OperandKind::ExecHi: + case Decoder::OperandKind::ExecHi: return ir.GetExecMaskTag(); case Decoder::OperandKind::VccLo: - case Decoder::OperandKind::VccHi: + case Decoder::OperandKind::VccHi: return ir.GetVccMaskValidTag(); case Decoder::OperandKind::VccZ: case Decoder::OperandKind::ExecZ: case Decoder::OperandKind::Scc: return IR::U1(IR::Value(true)); @@ -688,6 +774,7 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) { case Decoder::OperandKind::ExecHi: { const auto mask = BallotMask(value); ir.SetExec(value); + ir.SetExecMaskTag(IR::U1(IR::Value(true))); ir.SetExecLo(mask[0]); ir.SetExecHi(mask[1]); return; @@ -696,6 +783,7 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) { case Decoder::OperandKind::VccHi: { const auto mask = BallotMask(value); ir.SetVcc(value); + ir.SetVccMaskValidTag(IR::U1(IR::Value(true))); ir.SetVccLo(mask[0]); ir.SetVccHi(mask[1]); return; @@ -709,25 +797,7 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) { } void Translator::WriteMask64(const Decoder::Operand& operand, IR::U1 value) { - if (operand.kind != Decoder::OperandKind::Sgpr) { - WriteMask(operand, value); - return; - } - const auto reg = static_cast(operand.reg); - const auto mask = BallotMask(value); - ir.SetThreadBitScalarReg(reg, value); - ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true))); - if (IR::RegIndex(reg) > 0u) { - ir.SetScalarMaskTag(static_cast(IR::RegIndex(reg) - 1u), - IR::U1(IR::Value(false))); - } - ir.SetScalarReg(reg, mask[0]); - if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) { - const auto high = static_cast(IR::RegIndex(reg) + 1u); - ir.SetScalarReg(high, mask[1]); - ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false))); - ir.SetScalarMaskTag(high, IR::U1(IR::Value(false))); - } + WriteScalarU64(operand, MakeScalarU64Mask(value)); } void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 value) { @@ -1037,6 +1107,7 @@ bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, entry_ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false))); } entry_ir.SetExec(IR::U1(IR::Value(true))); + entry_ir.SetExecMaskTag(IR::U1(IR::Value(true))); entry_ir.SetExecLo(IR::U32(IR::Value(1u))); entry_ir.SetExecHi(IR::U32(IR::Value(0u))); if (options.stage == ShaderType::Compute) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index fd3918361..5d452325f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -19,6 +19,14 @@ public: bool AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info, std::string* error); private: + struct ScalarU64 { + std::array raw; + // Raw words and Kyty's per-invocation mask model remain independent. + // mask_valid selects invocation for logical consumers and SCC. + IR::U1 invocation; + IR::U1 mask_valid; + }; + Decoder::Operand SourceAt(const Decoder::Instruction& inst, uint32_t index); Decoder::Operand DestinationOperand(const Decoder::Instruction& inst); Decoder::Operand OffsetOperand(const Decoder::Operand& operand, uint32_t offset); @@ -48,14 +56,25 @@ private: IR::U32 ReadU16AsU32(const Decoder::Operand& operand, bool sign_extend); IR::U32 ReadF16LaneBits(const Decoder::Operand& operand, bool high_lane); std::array ExtractU64(IR::U64 value); - void WriteU32Pair(const Decoder::Operand& operand, const std::array& value); - IR::U1 ReadCondition(const Decoder::Operand& operand); - IR::U32 ConditionBit(const Decoder::Operand& operand); - IR::U1 ReadMask(const Decoder::Operand& operand); - IR::U1 ReadMaskValid(const Decoder::Operand& operand); - void WriteMask(const Decoder::Operand& operand, IR::U1 value); - void WriteMask64(const Decoder::Operand& operand, IR::U1 value); - void WriteCompareResult(const Decoder::Operand& operand, IR::U1 value); + void WriteU32Pair(const Decoder::Operand& operand, const std::array& value); + ScalarU64 ReadScalarU64(const Decoder::Operand& operand); + ScalarU64 MakeScalarU64Mask(IR::U1 value); + ScalarU64 MakeScalarU64Result(const std::array& raw, IR::U1 invocation, + IR::U1 mask_valid); + ScalarU64 SelectScalarU64(IR::U1 condition, const ScalarU64& lhs, const ScalarU64& rhs); + ScalarU64 NotScalarU64(const ScalarU64& value); + ScalarU64 BinaryScalarU64(const ScalarU64& lhs, const ScalarU64& rhs, + IR::ValueOpcode logical_opcode, IR::ValueOpcode bit_opcode, + bool negate_lhs, bool negate_rhs, bool negate_result); + void WriteScalarU64(const Decoder::Operand& operand, const ScalarU64& value); + IR::U1 ScalarU64NonZero(const ScalarU64& value); + IR::U1 ReadCondition(const Decoder::Operand& operand); + IR::U32 ConditionBit(const Decoder::Operand& operand); + IR::U1 ReadMask(const Decoder::Operand& operand); + IR::U1 ReadMaskValid(const Decoder::Operand& operand); + void WriteMask(const Decoder::Operand& operand, IR::U1 value); + void WriteMask64(const Decoder::Operand& operand, IR::U1 value); + void WriteCompareResult(const Decoder::Operand& operand, IR::U1 value); IR::MemoryFlags AddMemoryInfo(const IR::MemoryInfo& memory, uint32_t pc); IR::ExportFlags AddExportInfo(const Decoder::Instruction& inst); @@ -73,7 +92,7 @@ private: IR::U32 GetResourceDword(uint32_t index, uint32_t dword); IR::Value GetBufferResource(const IR::MemoryInfo& memory); IR::Value GetAddressResource(IR::Value low, IR::Value high); - IR::Value GetScalarAddressResource(uint32_t base); + IR::Value GetScalarAddressResource(const Decoder::Operand& base); IR::Value GetImageResource(const IR::MemoryInfo& memory); IR::Value GetSamplerResource(const IR::MemoryInfo& memory); IR::Value MakeImageAddress(const Decoder::Instruction& inst, const Decoder::Operand& base); @@ -175,8 +194,6 @@ private: bool PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign); bool S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode, IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, bool unary); - IR::U1 U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool negate_rhs, - bool negate_result); bool SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type, bool reverse, bool mask_shift_count, bool update_scc); bool ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, diff --git a/src/graphics/shader/recompiler/ir/IREmitter.cpp b/src/graphics/shader/recompiler/ir/IREmitter.cpp index 50db3c27a..3d72b8802 100644 --- a/src/graphics/shader/recompiler/ir/IREmitter.cpp +++ b/src/graphics/shader/recompiler/ir/IREmitter.cpp @@ -76,6 +76,14 @@ void IREmitter::SetExec(U1 value) { Emit(ValueOpcode::SetExec, {value}); } +U1 IREmitter::GetExecMaskTag() { + return U1(Emit(ValueOpcode::GetExecMaskTag)); +} + +void IREmitter::SetExecMaskTag(U1 value) { + Emit(ValueOpcode::SetExecMaskTag, {value}); +} + U32 IREmitter::GetExecLo() { return U32(Emit(ValueOpcode::GetExecLo)); } @@ -100,6 +108,14 @@ void IREmitter::SetVcc(U1 value) { Emit(ValueOpcode::SetVcc, {value}); } +U1 IREmitter::GetVccMaskValidTag() { + return U1(Emit(ValueOpcode::GetVccMaskValidTag)); +} + +void IREmitter::SetVccMaskValidTag(U1 value) { + Emit(ValueOpcode::SetVccMaskValidTag, {value}); +} + U32 IREmitter::GetVccLo() { return U32(Emit(ValueOpcode::GetVccLo)); } diff --git a/src/graphics/shader/recompiler/ir/IREmitter.h b/src/graphics/shader/recompiler/ir/IREmitter.h index e429ac04d..fdcecbaa7 100644 --- a/src/graphics/shader/recompiler/ir/IREmitter.h +++ b/src/graphics/shader/recompiler/ir/IREmitter.h @@ -39,12 +39,16 @@ public: void SetScc(U1 value); U1 GetExec(); void SetExec(U1 value); + U1 GetExecMaskTag(); + void SetExecMaskTag(U1 value); U32 GetExecLo(); void SetExecLo(U32 value); U32 GetExecHi(); void SetExecHi(U32 value); U1 GetVcc(); void SetVcc(U1 value); + U1 GetVccMaskValidTag(); + void SetVccMaskValidTag(U1 value); U32 GetVccLo(); void SetVccLo(U32 value); U32 GetVccHi(); diff --git a/src/graphics/shader/recompiler/ir/Program.cpp b/src/graphics/shader/recompiler/ir/Program.cpp index c7a4758f0..1944de5ea 100644 --- a/src/graphics/shader/recompiler/ir/Program.cpp +++ b/src/graphics/shader/recompiler/ir/Program.cpp @@ -31,12 +31,16 @@ bool IsRegisterStatePseudo(ValueOpcode opcode) { case ValueOpcode::SetScc: case ValueOpcode::GetExec: case ValueOpcode::SetExec: + case ValueOpcode::GetExecMaskTag: + case ValueOpcode::SetExecMaskTag: case ValueOpcode::GetExecLo: case ValueOpcode::SetExecLo: case ValueOpcode::GetExecHi: case ValueOpcode::SetExecHi: case ValueOpcode::GetVcc: case ValueOpcode::SetVcc: + case ValueOpcode::GetVccMaskValidTag: + case ValueOpcode::SetVccMaskValidTag: case ValueOpcode::GetVccLo: case ValueOpcode::SetVccLo: case ValueOpcode::GetVccHi: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 4e540cabc..c3a7e81d3 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -23,12 +23,16 @@ VALUE_OPCODE(GetScc, U1) VALUE_OPCODE(SetScc, Void, U1) VALUE_OPCODE(GetExec, U1) VALUE_OPCODE(SetExec, Void, U1) +VALUE_OPCODE(GetExecMaskTag, U1) +VALUE_OPCODE(SetExecMaskTag, Void, U1) VALUE_OPCODE(GetExecLo, U32) VALUE_OPCODE(SetExecLo, Void, U32) VALUE_OPCODE(GetExecHi, U32) VALUE_OPCODE(SetExecHi, Void, U32) VALUE_OPCODE(GetVcc, U1) VALUE_OPCODE(SetVcc, Void, U1) +VALUE_OPCODE(GetVccMaskValidTag, U1) +VALUE_OPCODE(SetVccMaskValidTag, Void, U1) VALUE_OPCODE(GetVccLo, U32) VALUE_OPCODE(SetVccLo, Void, U32) VALUE_OPCODE(GetVccHi, U32) diff --git a/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp b/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp index ce89b9008..c29703611 100644 --- a/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp @@ -22,6 +22,9 @@ struct ScalarMaskTag { struct ExecTag { auto operator<=>(const ExecTag&) const = default; }; +struct ExecMaskTag { + auto operator<=>(const ExecMaskTag&) const = default; +}; struct ExecLoTag { auto operator<=>(const ExecLoTag&) const = default; }; @@ -31,6 +34,9 @@ struct ExecHiTag { struct VccTag { auto operator<=>(const VccTag&) const = default; }; +struct VccMaskValidTag { + auto operator<=>(const VccMaskValidTag&) const = default; +}; struct VccLoTag { auto operator<=>(const VccLoTag&) const = default; }; @@ -45,9 +51,9 @@ struct GotoVariable { auto operator<=>(const GotoVariable&) const = default; }; -using Variable = - std::variant; +using Variable = std::variant; using ValueMap = std::unordered_map; struct DefTable { @@ -99,12 +105,16 @@ struct DefTable { void Set(Block* block, SccTag, Value value) { scc[block] = value; } const Value& Get(Block* block, ExecTag) { return exec[block]; } void Set(Block* block, ExecTag, Value value) { exec[block] = value; } + const Value& Get(Block* block, ExecMaskTag) { return exec_mask_tag[block]; } + void Set(Block* block, ExecMaskTag, Value value) { exec_mask_tag[block] = value; } const Value& Get(Block* block, ExecLoTag) { return exec_lo[block]; } void Set(Block* block, ExecLoTag, Value value) { exec_lo[block] = value; } const Value& Get(Block* block, ExecHiTag) { return exec_hi[block]; } void Set(Block* block, ExecHiTag, Value value) { exec_hi[block] = value; } const Value& Get(Block* block, VccTag) { return vcc[block]; } void Set(Block* block, VccTag, Value value) { vcc[block] = value; } + const Value& Get(Block* block, VccMaskValidTag) { return vcc_mask_valid_tag[block]; } + void Set(Block* block, VccMaskValidTag, Value value) { vcc_mask_valid_tag[block] = value; } const Value& Get(Block* block, VccLoTag) { return vcc_lo[block]; } void Set(Block* block, VccLoTag, Value value) { vcc_lo[block] = value; } const Value& Get(Block* block, VccHiTag) { return vcc_hi[block]; } @@ -114,9 +124,11 @@ struct DefTable { ValueMap scc; ValueMap exec; + ValueMap exec_mask_tag; ValueMap exec_lo; ValueMap exec_hi; ValueMap vcc; + ValueMap vcc_mask_valid_tag; ValueMap vcc_lo; ValueMap vcc_hi; ValueMap m0; @@ -144,6 +156,9 @@ ValueOpcode UndefOpcode(SccTag) { ValueOpcode UndefOpcode(ExecTag) { return ValueOpcode::UndefU1; } +ValueOpcode UndefOpcode(ExecMaskTag) { + return ValueOpcode::UndefU1; +} ValueOpcode UndefOpcode(ExecLoTag) { return ValueOpcode::UndefU32; } @@ -153,6 +168,9 @@ ValueOpcode UndefOpcode(ExecHiTag) { ValueOpcode UndefOpcode(VccTag) { return ValueOpcode::UndefU1; } +ValueOpcode UndefOpcode(VccMaskValidTag) { + return ValueOpcode::UndefU1; +} ValueOpcode UndefOpcode(VccLoTag) { return ValueOpcode::UndefU32; } @@ -184,6 +202,9 @@ Value InitialValue(SccTag) { Value InitialValue(ExecTag) { return Value(false); } +Value InitialValue(ExecMaskTag) { + return Value(false); +} Value InitialValue(ExecLoTag) { return Value(0u); } @@ -193,6 +214,9 @@ Value InitialValue(ExecHiTag) { Value InitialValue(VccTag) { return Value(false); } +Value InitialValue(VccMaskValidTag) { + return Value(false); +} Value InitialValue(VccLoTag) { return Value(0u); } @@ -351,9 +375,15 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) { break; case ValueOpcode::SetScc: pass.Write(SccTag {}, block, inst.Arg(0)); break; case ValueOpcode::SetExec: pass.Write(ExecTag {}, block, inst.Arg(0)); break; + case ValueOpcode::SetExecMaskTag: + pass.Write(ExecMaskTag {}, block, inst.Arg(0)); + break; case ValueOpcode::SetExecLo: pass.Write(ExecLoTag {}, block, inst.Arg(0)); break; case ValueOpcode::SetExecHi: pass.Write(ExecHiTag {}, block, inst.Arg(0)); break; case ValueOpcode::SetVcc: pass.Write(VccTag {}, block, inst.Arg(0)); break; + case ValueOpcode::SetVccMaskValidTag: + pass.Write(VccMaskValidTag {}, block, inst.Arg(0)); + break; case ValueOpcode::SetVccLo: pass.Write(VccLoTag {}, block, inst.Arg(0)); break; case ValueOpcode::SetVccHi: pass.Write(VccHiTag {}, block, inst.Arg(0)); break; case ValueOpcode::SetM0: pass.Write(M0Tag {}, block, inst.Arg(0)); break; @@ -375,9 +405,15 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) { break; case ValueOpcode::GetScc: inst.ReplaceUsesWith(pass.Read(SccTag {}, block)); break; case ValueOpcode::GetExec: inst.ReplaceUsesWith(pass.Read(ExecTag {}, block)); break; + case ValueOpcode::GetExecMaskTag: + inst.ReplaceUsesWith(pass.Read(ExecMaskTag {}, block)); + break; case ValueOpcode::GetExecLo: inst.ReplaceUsesWith(pass.Read(ExecLoTag {}, block)); break; case ValueOpcode::GetExecHi: inst.ReplaceUsesWith(pass.Read(ExecHiTag {}, block)); break; case ValueOpcode::GetVcc: inst.ReplaceUsesWith(pass.Read(VccTag {}, block)); break; + case ValueOpcode::GetVccMaskValidTag: + inst.ReplaceUsesWith(pass.Read(VccMaskValidTag {}, block)); + break; case ValueOpcode::GetVccLo: inst.ReplaceUsesWith(pass.Read(VccLoTag {}, block)); break; case ValueOpcode::GetVccHi: inst.ReplaceUsesWith(pass.Read(VccHiTag {}, block)); break; case ValueOpcode::GetM0: inst.ReplaceUsesWith(pass.Read(M0Tag {}, block)); break; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 3714cf48b..795c12e5c 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -12740,6 +12740,160 @@ TestCase ScalarNotB64UpdatesScc() { O::S_ENDPGM}}; } +TestCase ScalarMovB64PreservesRawSpecialData() { + using O = ShaderOpcode; + + std::vector code; + AppendSMovLiteral(&code, 0, 0x09500000u); + AppendSMovLiteral(&code, 1, 0x81234567u); + code.push_back(EncodeSop1(0x04, 106, 0)); // s_mov_b64 vcc, s[0:1] + code.push_back(EncodeSop1(0x04, 2, 106)); // s_mov_b64 s[2:3], vcc + code.push_back(EncodeSop1(0x08, 4, 2)); // s_not_b64 s[4:5], s[2:3] + code.push_back(EncodeSop2(0x0a, 6, InlineU32(1), InlineU32(0))); + + AppendStoreSgprPair(&code, 2, 0); + AppendStoreSgprPair(&code, 4, 2); + AppendStoreSgpr(&code, 6, 4); + + AppendSMovLiteral(&code, 8, 0x09500001u); + AppendSMovLiteral(&code, 9, 0x81234567u); + code.push_back(EncodeSop1(0x04, 126, 8)); // s_mov_b64 exec, s[8:9] + code.push_back(EncodeSop1(0x04, 10, 126)); // s_mov_b64 s[10:11], exec + code.push_back(EncodeSop1(0x08, 12, 10)); // s_not_b64 s[12:13], s[10:11] + code.push_back(EncodeSop2(0x0a, 14, InlineU32(1), InlineU32(0))); + + AppendStoreSgprPair(&code, 10, 5); + AppendStoreSgprPair(&code, 12, 7); + AppendStoreSgpr(&code, 14, 9); + AppendEnd(&code); + + return {"ScalarMovB64PreservesRawSpecialData", + code, + {}, + {0x09500000u, 0x81234567u, 0xf6afffffu, 0x7edcba98u, 1u, + 0x09500001u, 0x81234567u, 0xf6affffeu, 0x7edcba98u, 1u}, + {O::S_MOV_B32, O::S_MOV_B64, O::S_NOT_B64, O::S_CSELECT_B32, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; +} + +TestCase Scalar64RawSpecialOps() { + using O = ShaderOpcode; + + std::vector code; + const auto capture_scc = [&](u32 dst) { + code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0))); + }; + + AppendSMovLiteral(&code, 0, 0x09500001u); + AppendSMovLiteral(&code, 1, 0x81234567u); + AppendSMovLiteral(&code, 2, 0xff00ff01u); + AppendSMovLiteral(&code, 3, 0x0f0f0f0fu); + code.push_back(EncodeSop1(0x04, 126, 0)); + code.push_back(EncodeSop2(0x0f, 106, 126, 2)); + code.push_back(EncodeSop1(0x04, 4, 106)); + capture_scc(6); + code.push_back(EncodeSop1(0x08, 106, 106)); + code.push_back(EncodeSop1(0x04, 8, 106)); + capture_scc(10); + + AppendSMovLiteral(&code, 12, 0x80000011u); + AppendSMovLiteral(&code, 13, 0x80000001u); + code.push_back(EncodeSop1(0x04, 106, 12)); + code.push_back(EncodeSop1(0x0a, 106, 106)); + code.push_back(EncodeSop1(0x04, 14, 106)); + capture_scc(16); + + code.push_back(EncodeSop1(0x0a, 26, 193u)); + capture_scc(28); + + code.push_back(EncodeSMovB32(18, InlineU32(1))); + code.push_back(EncodeSMovB32(19, InlineU32(0))); + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1))); + code.push_back(EncodeSop2(0x0b, 126, 0, 18)); + code.push_back(EncodeSop1(0x04, 20, 126)); + code.push_back(EncodeSop1(0x08, 106, 126)); + code.push_back(EncodeSop1(0x04, 22, 106)); + capture_scc(24); + code.push_back(EncodeSop1(0x04, 126, 18)); + + AppendStoreSgprPair(&code, 4, 0); + AppendStoreSgpr(&code, 6, 2); + AppendStoreSgprPair(&code, 8, 3); + AppendStoreSgpr(&code, 10, 5); + AppendStoreSgprPair(&code, 14, 6); + AppendStoreSgpr(&code, 16, 8); + AppendStoreSgprPair(&code, 26, 9); + AppendStoreSgpr(&code, 28, 11); + AppendStoreSgprPair(&code, 20, 12); + AppendStoreSgprPair(&code, 22, 14); + AppendStoreSgpr(&code, 24, 16); + AppendEnd(&code); + + return {"Scalar64RawSpecialOps", + code, + {}, + {0x09000001u, 0x01030507u, 1, 0xf6fffffeu, 0xfefcfaf8u, 1, + 0xf00000ffu, 0xf000000fu, 1, 0xffffffffu, 0xffffffffu, 1, + 0x09500001u, 0x81234567u, 0xf6affffeu, 0x7edcba98u, 1}, + {O::S_MOV_B32, O::S_MOV_B64, O::S_AND_B64, O::S_NOT_B64, + O::S_CSELECT_B32, O::S_WQM_B64, O::S_CMP_EQ_U32, O::S_CSELECT_B64, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; +} + +TestCase ScalarMovB32SpecialAliases() { + using O = ShaderOpcode; + + std::vector code; + const auto capture_scc = [&](u32 dst) { + code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0))); + }; + + AppendSMovLiteral(&code, 0, 0x09500001u); + AppendSMovLiteral(&code, 1, 0x81234567u); + AppendSMovLiteral(&code, 2, 0xaaaaaaaau); + AppendSMovLiteral(&code, 3, 0xbbbbbbbbu); + code.push_back(EncodeSMovB32(4, InlineU32(1))); + code.push_back(EncodeSMovB32(5, InlineU32(0))); + code.push_back(EncodeSop1(0x04, 126, 0)); + code.push_back(EncodeSop1(0x04, 106, 2)); + code.push_back(EncodeSMovB32(107, 126)); + code.push_back(EncodeSop1(0x04, 6, 106)); + code.push_back(EncodeSop1(0x08, 8, 106)); + capture_scc(10); + + AppendSMovLiteral(&code, 12, 0x13579bdfu); + AppendSMovLiteral(&code, 13, 0x2468ace0u); + code.push_back(EncodeSop1(0x04, 106, 12)); + code.push_back(EncodeSMovB32(126, 106)); + code.push_back(EncodeSop1(0x04, 14, 126)); + code.push_back(EncodeSop1(0x04, 126, 4)); + + code.push_back(EncodeVopc(0xc2, InlineU32(0), 0)); + code.push_back(EncodeSMovB32(106, 106)); + code.push_back(EncodeSop1(0x08, 16, 106)); + capture_scc(18); + code.push_back(EncodeSop1(0x04, 20, 106)); + code.push_back(EncodeSMovB32(20, 20)); + code.push_back(EncodeSop1(0x08, 22, 20)); + capture_scc(24); + + AppendStoreSgprPair(&code, 6, 0); + AppendStoreSgprPair(&code, 8, 2); + AppendStoreSgpr(&code, 10, 4); + AppendStoreSgprPair(&code, 14, 5); + AppendStoreSgpr(&code, 18, 7); + AppendStoreSgpr(&code, 24, 8); + AppendEnd(&code); + + return {"ScalarMovB32SpecialAliases", + code, + {}, + {0xaaaaaaaau, 0x09500001u, 0x55555555u, 0xf6affffeu, 1, 0x13579bdfu, + 0x81234567u, 0, 0}, + {O::S_MOV_B32, O::S_MOV_B64, O::S_NOT_B64, O::S_CSELECT_B32, + O::V_CMP_EQ_U32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; +} + TestCase ScalarQuadmaskB64() { using O = ShaderOpcode; @@ -12838,8 +12992,7 @@ TestCase ScalarSaveExecOps() { return {"ScalarSaveExecOps", code, {}, - // EXEC and saved EXEC values are invocation-local Booleans. - {1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0}, + {1, 0, 1, 0, 0xffffffffu, 0xffffffffu, 1, 1, 0, 1, 0, 0, 0}, {O::S_MOV_B32, O::S_AND_SAVEEXEC_B64, O::S_ORN2_SAVEEXEC_B64, O::S_ANDN1_SAVEEXEC_B64, O::S_AND_SAVEEXEC_B32, O::S_ANDN1_SAVEEXEC_B32, O::S_MOV_B64, O::V_MOV_B32, @@ -12866,7 +13019,7 @@ TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() { return {"ScalarOrn2SaveexecUsesSourceOrNotExec", code, {}, - {1, 0, 1, 0, 1}, + {0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1}, {O::S_MOV_B32, O::S_ORN2_SAVEEXEC_B64, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } @@ -16496,6 +16649,30 @@ TestCase ScalarMemoryLoadVariants() { O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase ScalarLoadUsesRawExecAddress() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(0))); + code.push_back(EncodeSMovB32(1, InlineU32(0))); + code.push_back(EncodeSMovB32(2, InlineU32(4))); + code.push_back(EncodeSMovB32(3, InlineU32(0))); + code.push_back(EncodeSop1(0x04, 126, 2)); + code.push_back(EncodeSmem0(0x00, 20, 63)); + code.push_back(EncodeSmem1(0)); + code.push_back(EncodeSMovB32(2, InlineU32(1))); + code.push_back(EncodeSop1(0x04, 126, 2)); + AppendStoreSgpr(&code, 20, 0); + AppendEnd(&code); + + return {"ScalarLoadUsesRawExecAddress", + code, + {0x11111111u, 0x22222222u}, + {0x22222222u, 0x22222222u}, + {O::S_MOV_B32, O::S_MOV_B64, O::S_LOAD_DWORD, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; +} + TestCase ScalarLoadSignedImmediateOffsetAddsSoffset() { using O = ShaderOpcode; @@ -20948,6 +21125,9 @@ std::vector MakeCases() { AddCase(ScalarCompareOps); AddCase(ScalarShiftAddAndMaskOps); AddCase(ScalarNotB64UpdatesScc); + AddCase(ScalarMovB64PreservesRawSpecialData); + AddCase(Scalar64RawSpecialOps); + AddCase(ScalarMovB32SpecialAliases); AddCase(ScalarQuadmaskB64); AddCase(ScalarFlbitI32B64Gpu); AddCase(ScalarSaveExecOps); @@ -21061,6 +21241,7 @@ std::vector MakeCases() { AddCase(BranchVccnzUsesInvocationMask); AddCase(BranchVccnzUsesCarryProducedInvocationMask); AddCase(ScalarMemoryLoadVariants); + AddCase(ScalarLoadUsesRawExecAddress); AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset); AddCase(ScalarLoadAlignsComponentsAndMasksAddress); AddCase(BufferLoadStore); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index e720f0c59..fb1a06934 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -9470,10 +9470,12 @@ void TestTypedEntryStateIsMinimal() { error.c_str()); uint32_t set_exec = 0; + uint32_t set_exec_mask_tag = 0; uint32_t set_exec_lo = 0; uint32_t set_exec_hi = 0; uint32_t ballots = 0; IR::Value exec; + IR::Value exec_mask_tag; IR::Value exec_lo; IR::Value exec_hi; for (const auto *block : values.blocks) { @@ -9486,6 +9488,10 @@ void TestTypedEntryStateIsMinimal() { set_exec++; exec = inst.Arg(0).Resolve(); break; + case IR::ValueOpcode::SetExecMaskTag: + set_exec_mask_tag++; + exec_mask_tag = inst.Arg(0).Resolve(); + break; case IR::ValueOpcode::SetExecLo: set_exec_lo++; exec_lo = inst.Arg(0).Resolve(); @@ -9500,6 +9506,7 @@ void TestTypedEntryStateIsMinimal() { case IR::ValueOpcode::SetVectorRegister: case IR::ValueOpcode::SetScc: case IR::ValueOpcode::SetVcc: + case IR::ValueOpcode::SetVccMaskValidTag: case IR::ValueOpcode::SetVccLo: case IR::ValueOpcode::SetVccHi: case IR::ValueOpcode::SetM0: @@ -9510,8 +9517,9 @@ void TestTypedEntryStateIsMinimal() { } } } - Check(set_exec == 1u && set_exec_lo == 1u && set_exec_hi == 1u && - ballots == 0u && exec.IsImmediate() && exec.U1() && + Check(set_exec == 1u && set_exec_mask_tag == 1u && set_exec_lo == 1u && + set_exec_hi == 1u && ballots == 0u && exec.IsImmediate() && + exec.U1() && exec_mask_tag.IsImmediate() && exec_mask_tag.U1() && exec_lo.IsImmediate() && exec_lo.U32() == 1u && exec_hi.IsImmediate() && exec_hi.U32() == 0u, "typed entry is not the local {1,0} invocation mask"); @@ -10372,11 +10380,8 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { error.c_str()); CheckSpirvBinaryValidates(result.spirv); const auto wqm_source = DisassembleSpirvBinary(result.spirv); - Check(Common::ContainsStr(wqm_source, "OpCapability GroupNonUniformBallot") && - Common::ContainsStr(wqm_source, "OpGroupNonUniformBallot"), - "per-invocation scalar WQM omitted its subgroup ballot capability"); - Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 2u, - "wave64 WQM did not compact exactly two ballot words"); + Check(!Common::ContainsStr(wqm_source, "OpGroupNonUniformBallot"), + "raw scalar WQM reconstructed its source from invocation provenance"); auto wave32_options = options; wave32_options.wave_size = 32u; @@ -10384,8 +10389,9 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { &error), error.c_str()); CheckSpirvBinaryValidates(result.spirv); - Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 1u, - "wave32 WQM retained the unused high ballot-word expansion"); + Check(!Common::ContainsStr(DisassembleSpirvBinary(result.spirv), + "OpGroupNonUniformBallot"), + "wave32 raw scalar WQM reconstructed invocation provenance"); const uint32_t cross_lane_shader[] = { EncodeSop2(0x25, 126, 132, 128), // s_bfm_b64 exec, 4, 0 @@ -12488,8 +12494,8 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { .conditional_branches = 1, .ballots = 1}, ShaderType::Vertex); - Check(Common::ContainsStr(wqm_result.ir_dump, "WqmMask"), - "WQM size fixture no longer reaches per-invocation WqmMask IR"); + Check(Common::ContainsStr(wqm_result.ir_dump, "WqmU64"), + "WQM size fixture no longer reaches raw WqmU64 IR"); const uint32_t dispatcher[] = { EncodeSopp(0x05, 2), // entry -> B, fallthrough A