diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 7b2260b1f..9dc78a8e8 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -819,6 +819,13 @@ uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) { return ctx.Ballot(predicate); } +uint32_t EmitLiveMask(EmitterState& state) { + const auto helper = + EmitNative(state, state.helper_invocation_variable); + return EmitNative( + state, ConstantU32(state, spv::ScopeSubgroup), EmitLogicalNotBool(state, helper)); +} + uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst) { const auto ballot = ctx.Ballot(inst.Arg(1)); const auto lane = ctx.FirstLane(ballot); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index bd23c6bd0..ec77d3268 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -230,6 +230,7 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& in uint32_t EmitWqmU64(EmitterState& state, uint32_t value); uint32_t EmitLaneId(EmitterState& state); uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate); +uint32_t EmitLiveMask(EmitterState& state); uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index f7a3f8e93..309e6748b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -146,6 +146,7 @@ struct EmitterState { const IR::Block* current_block = nullptr; uint32_t pixel_valid_mask_variable = 0; uint32_t subgroup_local_invocation_id_variable = 0; + uint32_t helper_invocation_variable = 0; uint32_t per_vertex_variable = 0; uint32_t point_size_variable = 0; uint32_t clip_distance_variable = 0; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 71bd27c81..a781a0885 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -1541,16 +1541,16 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) { const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); const auto access = PrepareMemoryResourceAccess(state, mem); - const auto exec = ctx.Arg(inst, 1); - const auto ballot = ctx.Ballot(inst.Arg(1)); - const auto low = state.builder.AllocateId(); - const auto high = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, ballot, 0); - state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), high, ballot, 1); + // Elect and count from the guest EXEC words, which V_MBCNT also ranks lanes by. + const auto low = ctx.Arg(inst, 1); + const auto high = ctx.Arg(inst, 2); const auto count = Binary(state, spv::OpIAdd, TypeU32(state), Unary(state, spv::OpBitCount, TypeU32(state), low), Unary(state, spv::OpBitCount, TypeU32(state), high)); - const auto first = ctx.FirstLane(ballot); + const auto mask = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), mask, low, high, + ConstantU32(state, 0), ConstantU32(state, 0)); + const auto first = ctx.FirstLane(mask); const auto source_lane = state.lane_count == 2 ? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31)) @@ -1564,9 +1564,7 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) { const auto condition = AndCondition( state, is_first, AndCondition(state, - state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count, - ConstantU32(state, 0)) - : exec, + Binary(state, spv::OpINotEqual, TypeBool(state), count, ConstantU32(state, 0)), bounds)); const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() { const auto value = state.builder.AllocateId(); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index cb0b14006..709db2f2d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -595,6 +595,13 @@ void DefineInputs(EmitterState& state) { state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationFlat); } } + if (state.program.info.helper_invocation) { + const auto variable = DefineInterfaceVariable(state, TypeBool(state), spv::StorageClassInput, + "gl_HelperInvocation"); + state.helper_invocation_variable = variable; + state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBuiltIn, + spv::BuiltInHelperInvocation); + } } void DefineOutputs(EmitterState& state) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp index 2bae195c9..98d49137b 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp @@ -5,30 +5,20 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend { -void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar, - bool cmpx) { +void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar) { if (scalar) { ir.SetScc(value); return; } - const auto masked = ir.LogicalAnd(ir.GetExec(), value); - if (cmpx) { - const auto mask = BallotMask(masked); - ir.SetExec(masked); - ir.SetExecLo(mask[0]); - ir.SetExecHi(mask[1]); - return; - } - WriteMask(inst.dst, masked); + WriteVectorMask(inst.dst, value); } -void Translator::EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar, - bool cmpx) { - EmitCompareResult(inst, IR::U1(IR::Value(value)), scalar, cmpx); +void Translator::EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar) { + EmitCompareResult(inst, IR::U1(IR::Value(value)), scalar); } void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, - IR::Type type, bool scalar, bool cmpx, bool swap_operands) { + IR::Type type, bool scalar, bool swap_operands) { // The equality compares share their IR opcode with the unsigned ones. const bool signed_64 = opcode == IR::ValueOpcode::SLessThan64 || opcode == IR::ValueOpcode::SLessThanEqual64 || @@ -49,18 +39,18 @@ void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueO if (swap_operands) { std::swap(lhs, rhs); } - EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), scalar, cmpx); + EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), scalar); } void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, - bool signed_value, bool cmpx) { + bool signed_value) { const auto lhs = ReadU16AsU32(inst.src0, signed_value); const auto rhs = ReadU16AsU32(inst.src1, signed_value); - EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx); + EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false); } void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, - bool half, bool cmpx) { + bool half) { const auto type = IR::ArgTypeOf(opcode, 0); const auto lhs = half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, type); @@ -68,10 +58,10 @@ void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpc half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, type); const IR::FPCompareFlags flags{ .flush_input_denorms = !half && type == IR::Type::F32 && flush_f32_inputs}; - EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx); + EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false); } -void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx, +void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool is64) { const auto type = is64 ? IR::Type::F64 : IR::Type::F32; const auto is_nan_op = is64 ? IR::ValueOpcode::FPIsNan64 : IR::ValueOpcode::FPIsNan32; @@ -79,16 +69,16 @@ void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool const auto rhs = ReadOperand(inst.src1, type); const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(is_nan_op, {lhs})), IR::U1(ir.Emit(is_nan_op, {rhs}))); - EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, cmpx); + EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false); } -void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx, bool half) { +void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool half) { const auto value = half ? IR::Value(Read16LaneBits(inst.src0, false)) : ReadOperand(inst.src0, IR::Type::F32); const auto mask = half ? IR::Value(Read16LaneBits(inst.src1, false)) : ReadOperand(inst.src1, IR::Type::U32); const auto opcode = half ? IR::ValueOpcode::FPCmpClass16 : IR::ValueOpcode::FPCmpClass32; - EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {value, mask})), false, cmpx); + EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {value, mask})), false); } } // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index 993630d25..2d7ac8546 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -51,33 +51,33 @@ void Translator::S_SUBVECTOR_LOOP(const Decoder::Instruction& inst, bool begin) void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation, bool negate_exec, bool negate_source, bool write_64) { - if (!write_64) { - // Read the encoded scalar word and preserve EXEC_HI, including in wave32. - const auto old = ir.GetExecLo(); - const auto src = ReadU32(inst.src0); + const auto combine = [&](IR::U32 old, IR::U32 src) { const auto lhs = negate_exec ? ir.BitwiseNot(old) : old; const auto rhs = negate_source ? ir.BitwiseNot(src) : src; - IR::U32 result; - switch (operation) { - case IR::ValueOpcode::LogicalAnd: result = ir.BitwiseAnd(lhs, rhs); break; - case IR::ValueOpcode::LogicalOr: result = ir.BitwiseOr(lhs, rhs); break; - default: EXIT("unsupported SAVEEXEC operation"); - } + return operation == IR::ValueOpcode::LogicalAnd ? ir.BitwiseAnd(lhs, rhs) + : ir.BitwiseOr(lhs, rhs); + }; + if (!write_64) { + // Read the encoded scalar word and preserve EXEC_HI, including in wave32. + const auto old = ir.GetExecLo(); + const auto result = combine(old, ReadU32(inst.src0)); WriteRawU32(inst.dst, old); WriteRawU32(ConditionOperand(Decoder::OperandKind::ExecLo), result); ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u)))); return; } - const auto old = ir.GetExec(); - const auto src = ReadMask(inst.src0); - const auto lhs = negate_exec ? ir.LogicalNot(old) : old; - const auto rhs = negate_source ? ir.LogicalNot(src) : src; - const auto result = IR::U1(ir.Emit(operation, {lhs, rhs})); - WriteMask(inst.dst, old, true); - const auto mask = BallotMask(result); - ir.SetExec(result); - ir.SetExecLo(mask[0]); - ir.SetExecHi(mask[1]); + // EXEC words combine bitwise; the per-lane EXEC combines the per-lane masks, so helper lanes + // stay enabled. + const auto old = ir.GetExec(); + const auto src = ReadMask(inst.src0); + const auto lhs = negate_exec ? ir.LogicalNot(old) : old; + const auto rhs = negate_source ? ir.LogicalNot(src) : src; + const auto src_mask = ReadU32Pair(inst.src0); + const std::array old_mask {ir.GetExecLo(), ir.GetExecHi()}; + const std::array mask {combine(old_mask[0], src_mask[0]), combine(old_mask[1], src_mask[1])}; + WriteMask(inst.dst, old, old_mask, true); + WriteMask(ConditionOperand(Decoder::OperandKind::ExecLo), IR::U1(ir.Emit(operation, {lhs, rhs})), + mask); ir.SetScc(ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u)))); } @@ -95,7 +95,7 @@ void Translator::ADD_U32(const Decoder::Instruction& inst, bool vector, bool use } WriteOperand(DestinationOperand(inst), result); if (vector) { - WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), carry)); + WriteVectorMask(inst.dst2, carry); } else { ir.SetScc(carry); } @@ -108,7 +108,7 @@ void Translator::SUB_U32(const Decoder::Instruction& inst, bool vector, bool rev const auto borrow = ir.UGreaterThan(rhs, lhs); WriteOperand(DestinationOperand(inst), result); if (vector) { - WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow)); + WriteVectorMask(inst.dst2, borrow); } else { ir.SetScc(borrow); } @@ -126,7 +126,7 @@ void Translator::SUBB_U32(const Decoder::Instruction& inst, bool vector, bool re const auto borrow = ir.LogicalOr(borrow0, borrow1); WriteOperand(DestinationOperand(inst), result); if (vector) { - WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow)); + WriteVectorMask(inst.dst2, borrow); } else { ir.SetScc(borrow); } diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp index 57762a903..6ac64bf27 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -107,19 +107,7 @@ void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo default: return false; } }; - if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) || - (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) { - const auto mask = WriteMask(inst.dst, invocation_result, true); - ir.SetScc(ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u)))); - return; - } - - // Descriptor tracking reasons about each scalar word independently. - const auto lhs = ReadU32Pair(inst.src0); - auto mask_valid = ReadMaskValid(inst.src0); - if (inst.src_count > 1u) { - mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src1)); - } + const auto lhs = ReadU32Pair(inst.src0); std::array result; if (unary) { result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])}; @@ -132,11 +120,21 @@ void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo result[component] = negate_result ? ir.BitwiseNot(IR::U32(value)) : IR::U32(value); } } - WriteU32Pair(inst.dst, result); - if (inst.dst.kind == Decoder::OperandKind::Sgpr) { - const auto dst = static_cast(inst.dst.reg); - ir.SetThreadBitScalarReg(dst, invocation_result); - ir.SetScalarMaskTag(dst, mask_valid); + if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) || + (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) { + WriteMask(inst.dst, invocation_result, result, true); + } else { + // Descriptor tracking reasons about each scalar word independently. + auto mask_valid = ReadMaskValid(inst.src0); + if (inst.src_count > 1u) { + mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src1)); + } + WriteU32Pair(inst.dst, result); + if (inst.dst.kind == Decoder::OperandKind::Sgpr) { + const auto dst = static_cast(inst.dst.reg); + ir.SetThreadBitScalarReg(dst, invocation_result); + ir.SetScalarMaskTag(dst, mask_valid); + } } ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)))); } @@ -292,7 +290,8 @@ void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) { if (inst.dst2.kind != Decoder::OperandKind::Null && inst.dst2.kind != Decoder::OperandKind::Unknown) { // The 32x32 product fits in 64 bits, so only the addition can carry. - WriteMask(inst.dst2, IR::U1(ir.Emit(IR::ValueOpcode::ULessThan64, {result, product}))); + const auto carry = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan64, {result, product})); + WriteMask(inst.dst2, carry, VectorMask(carry)); } } @@ -472,7 +471,7 @@ void Translator::S_BITCMP_B32(const Decoder::Instruction& inst, bool expected) { const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u))); const auto bit = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {value, offset, IR::Value(1u)})); - WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u)))); + ir.SetScc(ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u)))); } void Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) { @@ -483,7 +482,7 @@ void Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) { ir.Select(ir.ULessThan(offset, IR::U32(IR::Value(32u))), value[0], value[1]); const auto bit = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {word, word_bit, IR::Value(1u)})); - WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u)))); + ir.SetScc(ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u)))); } void Translator::V_ALIGN_B32(const Decoder::Instruction& inst, bool byte_offset) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 78928aa51..73993799c 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -863,9 +863,10 @@ void Translator::DS_WRITE2(const Decoder::Instruction& inst) { void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { const auto memory = MemoryInfoFromDecoded(inst); - WriteOperand(inst.dst, ir.Emit(opcode, - {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, - AddMemoryInfo(memory, inst.pc))); + const auto exec = ExecMask(); + WriteOperand(inst.dst, + ir.Emit(opcode, {ReadU32(MemorySourceAt(inst, 0)), exec[0], exec[1], ir.GetExec()}, + AddMemoryInfo(memory, inst.pc))); } void Translator::DS_ADDTID(const Decoder::Instruction& inst, bool write) { @@ -901,10 +902,7 @@ void Translator::DS_ORDERED_COUNT(const Decoder::Instruction& inst) { const auto packer = ir.BitwiseAnd(m0, IR::U32(IR::Value(0xfu))); address = ir.IAdd(address, ir.ShiftLeftLogical(packer, IR::U32(IR::Value(2u)))); } - const auto exec = ir.GetExec(); - const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {exec}); - const auto low = ir.CompositeExtract(ballot, 0u); - const auto high = ir.CompositeExtract(ballot, 1u); + const auto [low, high] = ExecMask(); const auto active = ir.INotEqual(ir.BitwiseOr(low, high), IR::U32(IR::Value(0u))); const auto first = ir.Select( ir.INotEqual(low, IR::U32(IR::Value(0u))), diff --git a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp index ba513f604..53bb07bb7 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp @@ -73,43 +73,41 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) { case O::S_CMP_EQ_U32: case O::S_CMP_EQ_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, true, false); + EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, true); return; case O::S_CMP_LG_U32: case O::S_CMP_LG_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, true, false); + EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, true); return; case O::S_CMP_GT_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, true, false); + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, true); return; case O::S_CMP_GE_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, true, - false); + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, true); return; case O::S_CMP_LT_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, true, false); + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, true); return; case O::S_CMP_LE_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, true, false); + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, true); return; case O::S_CMP_GT_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, true, false); + EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, true); return; case O::S_CMP_GE_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, true, - false); + EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, true); return; case O::S_CMP_LT_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, true, false); + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, true); return; case O::S_CMP_LE_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, true, false); + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, true); return; case O::S_CMP_EQ_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, true, false); + EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, true); return; case O::S_CMP_LG_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, true, false); + EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, true); return; case O::S_AND_B64: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 6a387a82a..4266ba167 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -129,6 +129,20 @@ std::array Translator::BallotMask(IR::U1 value) { program.wave_size == 64u ? ir.CompositeExtract(mask, 1) : IR::U32(IR::Value(0u))}; } +std::array Translator::ExecMask() { + return {ir.GetExecLo(), program.wave_size == 64u ? ir.GetExecHi() : IR::U32(IR::Value(0u))}; +} + +// Pixel EXEC words exclude the helper invocations that the per-lane EXEC keeps enabled. +std::array Translator::VectorMask(IR::U1 value) { + auto mask = BallotMask(value); + if (program.stage == ShaderType::Pixel) { + const auto exec = ExecMask(); + mask = {ir.BitwiseAnd(mask[0], exec[0]), ir.BitwiseAnd(mask[1], exec[1])}; + } + return mask; +} + IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) { switch (operand.kind) { case Decoder::OperandKind::LiteralConstant: @@ -390,21 +404,9 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value) switch (operand.kind) { case Decoder::OperandKind::Scc: ir.SetScc(IR::U1(value)); return; case Decoder::OperandKind::ExecLo: - case Decoder::OperandKind::ExecHi: { - const auto mask = BallotMask(IR::U1(value)); - ir.SetExec(IR::U1(value)); - ir.SetExecLo(mask[0]); - ir.SetExecHi(mask[1]); - return; - } + case Decoder::OperandKind::ExecHi: case Decoder::OperandKind::VccLo: - case Decoder::OperandKind::VccHi: { - const auto mask = BallotMask(IR::U1(value)); - ir.SetVcc(IR::U1(value)); - ir.SetVccLo(mask[0]); - ir.SetVccHi(mask[1]); - return; - } + case Decoder::OperandKind::VccHi: EXIT("lane masks are written through WriteMask"); default: WriteRawU32(operand, ir.Select(IR::U1(value), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)))); @@ -641,7 +643,7 @@ IR::U1 Translator::ReadMask(const Decoder::Operand& operand) { } case Decoder::OperandKind::ExecLo: case Decoder::OperandKind::ExecHi: return ir.GetExec(); - case Decoder::OperandKind::VccLo: + case Decoder::OperandKind::VccLo: return ir.GetVcc(); case Decoder::OperandKind::VccHi: return program.wave_size == 32u ? ThreadBit({ReadRawU32(operand), IR::U32(IR::Value(0u))}) @@ -679,9 +681,8 @@ IR::U1 Translator::ReadMaskValid(const Decoder::Operand& operand) { } } -std::array Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value, - bool write_64) { - const auto mask = BallotMask(value); +void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value, + const std::array& mask, bool write_64) { switch (operand.kind) { case Decoder::OperandKind::Sgpr: { const auto reg = static_cast(operand.reg); @@ -702,43 +703,43 @@ std::array Translator::WriteMask(const Decoder::Operand& operand, IR ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false))); ir.SetScalarMaskTag(high, IR::U1(IR::Value(false))); } - return mask; + return; } case Decoder::OperandKind::ExecLo: case Decoder::OperandKind::ExecHi: { ir.SetExec(value); ir.SetExecLo(mask[0]); ir.SetExecHi(mask[1]); - return mask; + return; } case Decoder::OperandKind::VccLo: case Decoder::OperandKind::VccHi: { - if (!write_64 && program.wave_size == 32u) { + if (operand.kind == Decoder::OperandKind::VccHi && !write_64 && + program.wave_size == 32u) { WriteRawU32(operand, mask[0]); - return mask; + return; } ir.SetVcc(value); ir.SetVccLo(mask[0]); - ir.SetVccHi(mask[1]); - return mask; + if (write_64 || program.wave_size == 64u) { + ir.SetVccHi(mask[1]); + } + return; } - case Decoder::OperandKind::Scc: ir.SetScc(value); return mask; + case Decoder::OperandKind::Scc: ir.SetScc(value); return; default: WriteRawU32(operand, mask[0]); WriteRawU32(OffsetOperand(operand, 1), mask[1]); - return mask; + return; } } -void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 value) { - if (operand.kind == Decoder::OperandKind::Scc) { - WriteOperand(operand, value); - return; - } +void Translator::WriteVectorMask(const Decoder::Operand& operand, IR::U1 value) { // VALU instructions only write lanes enabled by the incoming EXEC mask. This is // especially important for V_CMPX: replacing EXEC with the ungated comparison would // reactivate lanes disabled by an enclosing divergent region. - WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value)); + const auto active = ir.LogicalAnd(ir.GetExec(), value); + WriteMask(operand, active, VectorMask(active)); } void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source, @@ -1155,7 +1156,10 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& } } entry_ir.SetExec(initial_exec); - const auto initial_mask = entry_ir.Emit(IR::ValueOpcode::Ballot, {initial_exec}); + // Pixel EXEC words start with the live (non-helper) lanes; helpers keep the per-lane EXEC. + const auto initial_mask = options.stage == ShaderType::Pixel + ? entry_ir.Emit(IR::ValueOpcode::LiveMask) + : entry_ir.Emit(IR::ValueOpcode::Ballot, {initial_exec}); const auto initial_lo = entry_ir.CompositeExtract(initial_mask, 0); const auto initial_hi = options.wave_size == 64u ? entry_ir.CompositeExtract(initial_mask, 1) : IR::U32(IR::Value(0u)); diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 270d7d90f..349df316b 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -27,6 +27,8 @@ private: Decoder::Operand ScalarDestinationOperand(const Decoder::Operand& operand, uint32_t offset); Decoder::Operand PlainOperand(const Decoder::Operand& operand); std::array BallotMask(IR::U1 value); + std::array ExecMask(); + std::array VectorMask(IR::U1 value); IR::U32 ReadRawU32(const Decoder::Operand& operand); IR::U32 ReadScalarCode(uint32_t code); IR::U32 ApplyBitSourceModifiers(const Decoder::Operand& operand, IR::U32 value); @@ -52,9 +54,9 @@ private: IR::U32 ConditionBit(const Decoder::Operand& operand); IR::U1 ReadMask(const Decoder::Operand& operand); IR::U1 ReadMaskValid(const Decoder::Operand& operand); - std::array WriteMask(const Decoder::Operand& operand, IR::U1 value, - bool write_64 = false); - void WriteCompareResult(const Decoder::Operand& operand, IR::U1 value); + void WriteMask(const Decoder::Operand& operand, IR::U1 value, + const std::array& mask, bool write_64 = false); + void WriteVectorMask(const Decoder::Operand& operand, IR::U1 value); IR::MemoryFlags AddMemoryInfo(const IR::MemoryInfo& memory, uint32_t pc); IR::ExportFlags AddExportInfo(const Decoder::Instruction& inst); @@ -115,17 +117,15 @@ private: IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value); IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high); IR::U32 Saturate16(IR::U32 value, bool signed_value); - void EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar, bool cmpx); - void EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar, bool cmpx); + void EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar); + void EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar); void EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type, - bool scalar, bool cmpx, bool swap_operands = false); + bool scalar, bool swap_operands = false); void EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, - bool signed_value, bool cmpx); - void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half, - bool cmpx); - void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx, - bool is64 = false); - void EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx, bool half = false); + bool signed_value); + void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half); + void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool is64 = false); + void EmitFloatClassCompare(const Decoder::Instruction& inst, bool half = false); void V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index); void V_CVT_F32_U32(const Decoder::Instruction& inst); void V_CVT_F32_I32(const Decoder::Instruction& inst); diff --git a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp index 492a470d2..bfb234c69 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp @@ -25,444 +25,310 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_PERMLANEX16_B32: V_PERMLANE16_B32(inst, true); return; case O::V_CMP_F_I32: - case O::V_CMP_F_U32: EmitCompareConstant(inst, false, false, false); return; + case O::V_CMP_F_U32: EmitCompareConstant(inst, false, false); return; case O::V_CMP_T_I32: - case O::V_CMP_T_U32: EmitCompareConstant(inst, true, false, false); return; + case O::V_CMP_T_U32: EmitCompareConstant(inst, true, false); return; case O::V_CMP_EQ_U32: case O::V_CMP_EQ_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, false); - return; case O::V_CMPX_EQ_U32: case O::V_CMPX_EQ_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false); return; case O::V_CMP_NE_U32: case O::V_CMP_NE_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, false); - return; case O::V_CMPX_NE_U32: case O::V_CMPX_NE_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false); return; case O::V_CMP_GT_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, false); - return; case O::V_CMPX_GT_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false); return; case O::V_CMP_GE_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false, - false); - return; case O::V_CMPX_GE_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false, - true); + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false); return; case O::V_CMP_LT_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, false); - return; case O::V_CMPX_LT_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false); return; case O::V_CMP_LE_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false, - false); - return; case O::V_CMPX_LE_U32: - EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false); return; case O::V_CMP_GT_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, false); - return; case O::V_CMPX_GT_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false); return; case O::V_CMP_GE_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false, - false); - return; case O::V_CMPX_GE_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false, - true); + EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false); return; case O::V_CMP_LT_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, false); - return; case O::V_CMPX_LT_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false); return; case O::V_CMP_LE_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false, - false); - return; case O::V_CMPX_LE_I32: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false); return; case O::V_CMP_F_I64: - case O::V_CMP_F_U64: EmitCompareConstant(inst, false, false, false); return; - case O::V_CMP_T_I64: - case O::V_CMP_T_U64: EmitCompareConstant(inst, true, false, false); return; + case O::V_CMP_F_U64: case O::V_CMPX_F_I64: - case O::V_CMPX_F_U64: EmitCompareConstant(inst, false, false, true); return; + case O::V_CMPX_F_U64: EmitCompareConstant(inst, false, false); return; + case O::V_CMP_T_I64: + case O::V_CMP_T_U64: case O::V_CMPX_T_I64: - case O::V_CMPX_T_U64: EmitCompareConstant(inst, true, false, true); return; + case O::V_CMPX_T_U64: EmitCompareConstant(inst, true, false); return; case O::V_CMP_EQ_I64: case O::V_CMP_EQ_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false, false); - return; case O::V_CMPX_EQ_I64: case O::V_CMPX_EQ_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false); return; case O::V_CMP_LT_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan64, IR::Type::U64, false, false); - return; case O::V_CMPX_LT_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan64, IR::Type::U64, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan64, IR::Type::U64, false); return; case O::V_CMP_GT_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, false); - return; case O::V_CMPX_GT_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false); return; case O::V_CMP_NE_I64: case O::V_CMP_NE_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, false); - return; case O::V_CMPX_NE_I64: case O::V_CMPX_NE_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false); return; case O::V_CMP_LT_I64: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, false); - return; case O::V_CMPX_LT_I64: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false); return; case O::V_CMP_LE_I64: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, false); - return; case O::V_CMPX_LE_I64: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false); return; // The IR has no signed 64-bit greater-than: a > b is b < a, and a >= b is b <= a. case O::V_CMP_GT_I64: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, false, - true); - return; case O::V_CMPX_GT_I64: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, true, - true); + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, true); return; case O::V_CMP_GE_I64: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, false, - true); - return; case O::V_CMPX_GE_I64: - EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, true, - true); + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, true); return; case O::V_CMP_LE_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, false); - return; case O::V_CMPX_LE_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false); return; case O::V_CMP_GE_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false, false); - return; case O::V_CMPX_GE_U64: - EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false, true); + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false); return; case O::V_CMP_EQ_U16: - EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false); + case O::V_CMPX_EQ_U16: + EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false); return; case O::V_CMP_EQ_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, false); - return; case O::V_CMPX_EQ_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, true); + EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true); return; case O::V_CMP_NE_U16: - EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false); - return; case O::V_CMPX_NE_U16: - EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, true); + EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false); return; case O::V_CMP_NE_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false); - return; case O::V_CMPX_NE_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, true); + EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true); return; case O::V_CMP_GT_U16: - EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, false); - return; case O::V_CMPX_GT_U16: - EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, true); + EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false); return; case O::V_CMP_GE_U16: - EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThanEqual32, false, false); + EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThanEqual32, false); return; case O::V_CMP_LT_U16: - EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, false); - return; case O::V_CMPX_LT_U16: - EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, true); - return; - case O::V_CMPX_EQ_U16: - EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, true); + EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false); return; case O::V_CMP_LE_U16: - EmitInteger16Compare(inst, IR::ValueOpcode::ULessThanEqual32, false, false); + EmitInteger16Compare(inst, IR::ValueOpcode::ULessThanEqual32, false); return; case O::V_CMP_GT_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, false); - return; case O::V_CMPX_GT_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, true); + EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true); return; case O::V_CMP_GE_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, false); - return; case O::V_CMPX_GE_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, true); + EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true); return; case O::V_CMP_LT_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, false); - return; case O::V_CMPX_LT_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, true); + EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true); return; case O::V_CMP_LE_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, false); - return; case O::V_CMPX_LE_I16: - EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, true); + EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true); return; - case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return; - case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return; + case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false); return; + case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false); return; case O::V_CMP_EQ_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, false); + case O::V_CMPX_EQ_F32: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false); return; case O::V_CMP_EQ_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false, false); + case O::V_CMPX_EQ_F64: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false); return; case O::V_CMP_LE_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, false); - return; case O::V_CMPX_LE_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, true); - return; - case O::V_CMPX_GE_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual64, false, true); - return; - case O::V_CMP_F_F64: EmitCompareConstant(inst, false, false, false); return; - case O::V_CMP_TRU_F64: EmitCompareConstant(inst, true, false, false); return; - case O::V_CMPX_F_F64: EmitCompareConstant(inst, false, false, true); return; - case O::V_CMPX_TRU_F64: EmitCompareConstant(inst, true, false, true); return; - case O::V_CMP_LT_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan64, false, false); - return; - case O::V_CMPX_LT_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan64, false, true); - return; - case O::V_CMPX_EQ_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false, true); - return; - case O::V_CMP_GT_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan64, false, false); - return; - case O::V_CMPX_GT_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan64, false, true); - return; - case O::V_CMP_LG_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual64, false, false); - return; - case O::V_CMPX_LG_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual64, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false); return; case O::V_CMP_GE_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual64, false, false); + case O::V_CMPX_GE_F64: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual64, false); + return; + case O::V_CMP_F_F64: + case O::V_CMPX_F_F64: EmitCompareConstant(inst, false, false); return; + case O::V_CMP_TRU_F64: + case O::V_CMPX_TRU_F64: EmitCompareConstant(inst, true, false); return; + case O::V_CMP_LT_F64: + case O::V_CMPX_LT_F64: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan64, false); + return; + case O::V_CMP_GT_F64: + case O::V_CMPX_GT_F64: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan64, false); + return; + case O::V_CMP_LG_F64: + case O::V_CMPX_LG_F64: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual64, false); return; case O::V_CMP_NLG_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual64, false, false); - return; case O::V_CMPX_NLG_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual64, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual64, false); return; case O::V_CMP_NEQ_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual64, false, false); - return; case O::V_CMPX_NEQ_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual64, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual64, false); return; case O::V_CMP_NLE_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan64, false, false); - return; case O::V_CMPX_NLE_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan64, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan64, false); return; case O::V_CMP_NLT_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual64, false, false); - return; case O::V_CMPX_NLT_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual64, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual64, false); return; case O::V_CMP_NGE_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan64, false, false); - return; case O::V_CMPX_NGE_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan64, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan64, false); return; case O::V_CMP_NGT_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual64, false, false); - return; case O::V_CMPX_NGT_F64: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual64, false, true); - return; - case O::V_CMP_O_F64: EmitFloatOrderedCompare(inst, true, false, true); return; - case O::V_CMP_U_F64: EmitFloatOrderedCompare(inst, false, false, true); return; - case O::V_CMPX_O_F64: EmitFloatOrderedCompare(inst, true, true, true); return; - case O::V_CMPX_U_F64: EmitFloatOrderedCompare(inst, false, true, true); return; - case O::V_CMPX_EQ_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual64, false); return; + case O::V_CMP_O_F64: + case O::V_CMPX_O_F64: EmitFloatOrderedCompare(inst, true, true); return; + case O::V_CMP_U_F64: + case O::V_CMPX_U_F64: EmitFloatOrderedCompare(inst, false, true); return; case O::V_CMP_LG_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, false); - return; case O::V_CMPX_LG_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false); return; case O::V_CMP_GT_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, false); - return; case O::V_CMPX_GT_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false); return; case O::V_CMP_GE_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, false); - return; case O::V_CMPX_GE_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false); return; case O::V_CMP_LT_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, false); - return; case O::V_CMPX_LT_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false); return; case O::V_CMP_LE_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, false); - return; case O::V_CMPX_LE_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false); return; case O::V_CMP_NLG_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, false); - return; case O::V_CMPX_NLG_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false); return; case O::V_CMP_NEQ_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, false); - return; case O::V_CMPX_NEQ_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false); return; case O::V_CMP_NLE_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, false); - return; case O::V_CMPX_NLE_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false); return; case O::V_CMP_NLT_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, false); - return; case O::V_CMPX_NLT_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false); return; case O::V_CMP_NGE_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, false); - return; case O::V_CMPX_NGE_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false); return; case O::V_CMP_NGT_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, false); - return; case O::V_CMPX_NGT_F32: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false); return; case O::V_CMP_EQ_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, false); - return; case O::V_CMPX_EQ_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true); return; case O::V_CMP_LG_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, true, false); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, true); return; case O::V_CMP_GT_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, false); - return; case O::V_CMPX_GT_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true); return; case O::V_CMP_GE_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, false); - return; case O::V_CMPX_GE_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true); return; case O::V_CMP_LT_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, false); - return; case O::V_CMPX_LT_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true); return; case O::V_CMP_LE_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, false); - return; case O::V_CMPX_LE_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true); return; case O::V_CMP_NGE_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, true, false); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, true); return; case O::V_CMP_NGT_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, false); - return; case O::V_CMPX_NGT_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true); return; case O::V_CMPX_NLE_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, true, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, true); return; case O::V_CMP_NEQ_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, false); - return; case O::V_CMPX_NEQ_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true); return; case O::V_CMP_NLT_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true, false); - return; case O::V_CMPX_NLT_F16: - EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true, true); + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true); return; - case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true, false); return; - case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false, false); return; - case O::V_CMPX_O_F32: EmitFloatOrderedCompare(inst, true, true); return; - case O::V_CMPX_U_F32: EmitFloatOrderedCompare(inst, false, true); return; - case O::V_CMP_CLASS_F32: EmitFloatClassCompare(inst, false); return; - case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst, true); return; - case O::V_CMP_CLASS_F16: EmitFloatClassCompare(inst, false, true); return; - case O::V_CMPX_CLASS_F16: EmitFloatClassCompare(inst, true, true); return; + case O::V_CMP_O_F32: + case O::V_CMPX_O_F32: EmitFloatOrderedCompare(inst, true); return; + case O::V_CMP_U_F32: + case O::V_CMPX_U_F32: EmitFloatOrderedCompare(inst, false); return; + case O::V_CMP_CLASS_F32: + case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst); return; + case O::V_CMP_CLASS_F16: + case O::V_CMPX_CLASS_F16: EmitFloatClassCompare(inst, true); return; case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return; case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return; diff --git a/src/graphics/shader/recompiler/ir/Serialize.h b/src/graphics/shader/recompiler/ir/Serialize.h index 11156bc1b..33b204a02 100644 --- a/src/graphics/shader/recompiler/ir/Serialize.h +++ b/src/graphics/shader/recompiler/ir/Serialize.h @@ -155,7 +155,7 @@ KYTY_IO_FIELDS(SampledResourcePair, a, b, c) KYTY_IO_FIELDS(StageInput, a, b, c, d, e) KYTY_IO_FIELDS(StageOutput, a, b, c, d) KYTY_IO_FIELDS(ShaderInfo, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, - A, B, C, D, E, F, G, H, I, J) + A, B, C, D, E, F, G, H, I, J, K) KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows, a, b) KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead, a, b, c, d, e, f, g, h, i, j) KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::PointerTable, a, b, c, d) diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 680919c35..9e41d7799 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -547,6 +547,7 @@ struct ShaderInfo { bool subgroup_shuffle = false; bool subgroup_arithmetic = false; bool subgroup_local_invocation_id = false; + bool helper_invocation = false; bool compute_derivatives = false; bool image_gather_extended = false; bool function_lds = false; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index a3ec856f9..c3e6440af 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -219,6 +219,7 @@ VALUE_OPCODE(FPFract64, F64, F64) VALUE_OPCODE(LaneId, U32) VALUE_OPCODE(Ballot, U32x4, U1) +VALUE_OPCODE(LiveMask, U32x4) VALUE_OPCODE(ReadFirstLane, U32, U32, U1) VALUE_OPCODE(ReadLane, U32, U32, U32) VALUE_OPCODE(SubgroupUMinU32, U32, U32) @@ -310,8 +311,8 @@ VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicOr64, Void, U32, U64, U1) VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicMaskedOr32, Void, U32, U32, U32, U1) -VALUE_OPCODE(DataAppend, U32, U32, U1) -VALUE_OPCODE(DataConsume, U32, U32, U1) +VALUE_OPCODE(DataAppend, U32, U32, U32, U32, U1) +VALUE_OPCODE(DataConsume, U32, U32, U32, U32, U1) VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1) VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress) diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index d0e0de093..80471b368 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -905,8 +905,10 @@ private: if (inst.GetOpcode() != ValueOpcode::CompositeExtractU32x4 || !Immediate(Arg(inst, 1), 0u)) return {}; const auto* source = Arg(inst, 0).TryInstruction(); - return source != nullptr && source->GetOpcode() == ValueOpcode::Ballot - ? Arg(*source, 0) : Value {}; + if (source == nullptr) return {}; + // Helper lanes keep the per-lane EXEC, so the live mask projects to true. + if (source->GetOpcode() == ValueOpcode::LiveMask) return Value(true); + return source->GetOpcode() == ValueOpcode::Ballot ? Arg(*source, 0) : Value {}; } bool CanProject(Value value) { diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index 14a624cfe..a566360fe 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -366,7 +366,6 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs info.function_lds |= kind == ResourceKind::Lds && program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh; if (shared_access == SharedAccess::Append || shared_access == SharedAccess::Consume) { - info.subgroup_ballot = true; info.subgroup_shuffle = true; info.subgroup_local_invocation_id = true; } @@ -517,6 +516,7 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs inst.Flags() != CFG::BranchCondition::ScalarInstruction; break; case ValueOpcode::Ballot: info.subgroup_ballot = true; break; + case ValueOpcode::LiveMask: info.subgroup_ballot = info.helper_invocation = true; break; case ValueOpcode::SubgroupUMinU32: case ValueOpcode::SubgroupUMaxU32: info.subgroup_arithmetic = true; break; case ValueOpcode::DppMoveU32: @@ -577,7 +577,7 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) { info.has_bitwise_xor = info.uses_lds = info.uses_gds = info.uses_flattened_srt = info.uses_dispatch_threads = info.uses_swizzle = false; info.bvh = info.subgroup_ballot = info.subgroup_vote = info.subgroup_barrier = - info.subgroup_shuffle = info.subgroup_arithmetic = + info.subgroup_shuffle = info.subgroup_arithmetic = info.helper_invocation = info.subgroup_local_invocation_id = info.compute_derivatives = info.image_gather_extended = info.function_lds = info.function_scratch = info.pixel_valid_mask = info.buffer_int64_atomics = info.buffer_u8 = info.buffer_u16 = info.shared_int64_atomics = info.coherent_buffers = info.float64 = false; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index ecea4d833..29db76ce8 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -1436,6 +1436,7 @@ struct GraphicsCase { u32 pixel_custom_interpolation_mask = 0; u32 width = 1; std::vector object_ids; + std::vector gds_initial; }; struct CompiledShader { @@ -19919,8 +19920,40 @@ public: push_constant_range.size = ShaderRecompiler::IR::NativePushConstantSize; } + using Kind = ShaderRecompiler::IR::DescriptorBindingKind; + const bool uses_gds = !test.gds_initial.empty(); + const auto gds_bit = uint64_t{1} << static_cast(Kind::Gds); + Require(test.name, "graphics", fragment_bind.descriptor_mask == (uses_gds ? gds_bit : 0u), + "a graphics case binds GDS only, and only with initial GDS data"); + Buffer gds_buffer; + vk::DescriptorBufferInfo gds_info{}; + vk::WriteDescriptorSet gds_write{}; + vk::DescriptorSetLayout descriptor_layout = nullptr; + if (uses_gds) { + gds_buffer = CreateStorageBuffer(test.name, test.gds_initial, test.gds_initial.size()); + gds_info = {gds_buffer.buffer, 0, gds_buffer.size}; + const vk::DescriptorSetLayoutBinding binding{ + ShaderRecompiler::IR::NativeBinding(ShaderType::Pixel, Kind::Gds), + vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eFragment}; + vk::DescriptorSetLayoutCreateInfo layout_info{}; + layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; + layout_info.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR; + layout_info.bindingCount = 1; + layout_info.pBindings = &binding; + RequireVk(test.name, "graphics", + m_device.createDescriptorSetLayout(&layout_info, nullptr, &descriptor_layout), + "vkCreateDescriptorSetLayout"); + gds_write.sType = vk::StructureType::eWriteDescriptorSet; + gds_write.dstBinding = binding.binding; + gds_write.descriptorCount = 1; + gds_write.descriptorType = vk::DescriptorType::eStorageBuffer; + gds_write.pBufferInfo = &gds_info; + } + vk::PipelineLayoutCreateInfo pipeline_layout_info{}; pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; + pipeline_layout_info.setLayoutCount = uses_gds ? 1u : 0u; + pipeline_layout_info.pSetLayouts = &descriptor_layout; pipeline_layout_info.pushConstantRangeCount = push_constant_range.size != 0 ? 1u : 0u; pipeline_layout_info.pPushConstantRanges = @@ -20077,6 +20110,10 @@ public: } cmd.beginRendering(rendering); cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline); + if (uses_gds) { + cmd.pushDescriptorSetKHR(vk::PipelineBindPoint::eGraphics, pipeline_layout, 0, 1, + &gds_write); + } if (push_constant_range.size != 0) { ShaderRecompiler::IR::PushData push_data; if (object_id) { @@ -20104,6 +20141,8 @@ public: m_device.destroyPipeline(pipeline, nullptr); m_device.destroyPipelineLayout(pipeline_layout, nullptr); + m_device.destroyDescriptorSetLayout(descriptor_layout, nullptr); + DestroyBuffer(&gds_buffer); m_device.destroyShaderModule(fragment_module, nullptr); m_device.destroyShaderModule(vertex_module, nullptr); DestroyBuffer(&vertex_buffer); @@ -21639,6 +21678,7 @@ private: available_features.drawIndirectFirstInstance && available_features.multiDrawIndirect && available_features.depthBounds && + available_features.fragmentStoresAndAtomics && available_depth_clip.depthClipEnable && available_clip_control.depthClipControl && available_color_write.colorWriteEnable && @@ -21738,6 +21778,7 @@ private: device_features.drawIndirectFirstInstance = m_rasterization_supported; device_features.multiDrawIndirect = m_rasterization_supported; device_features.depthBounds = m_rasterization_supported; + device_features.fragmentStoresAndAtomics = m_rasterization_supported; device_info.pEnabledFeatures = &device_features; std::vector device_extensions{ VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, @@ -44098,6 +44139,92 @@ GraphicsCase GraphicsDsAddtidScratchExport() { O::DS_READ_ADDTID_B32, O::EXP, O::S_ENDPGM}}; } +// Covers only pixel 1 of a 2x1 target, so the rest of its quad runs as helper invocations, and +// starts a GDS counter at 10 in dword 5. +void CoverOnePixelOfQuad(GraphicsCase *test) { + test->width = 2; + test->vertices = {0x00000000u, 0xbf800000u, 0, 0, 0, 0, + 0x40400000u, 0xbf800000u, 0, 0, 0, 0, + 0x00000000u, 0x40400000u, 0, 0, 0, 0}; + test->gds_initial = {0, 0, 0, 0, 0, 10}; +} + +GraphicsCase GraphicsDsAppendSkipsHelperLanes(bool consume) { + using O = ShaderOpcode; + + // The EXEC and VCC words exclude the helper lanes. Exports the first counter value, the second + // minus the first minus s_bcnt1(exec) (reversed for consume), s_bcnt1(exec), and the EXEC plus + // VCC counts after an if/endif on a compare of the interpolant (zero in every lane). + const u32 op = consume ? 0x3d : 0x3e; + std::vector code; + code.push_back(EncodeVintrp(0x00, 7, 0, 0, 0)); + code.push_back(EncodeVintrp(0x01, 7, 0, 0, 1)); + AppendSMovLiteral(&code, 124, 0x100); + code.push_back(EncodeDs0(op, 20, true)); + code.push_back(EncodeDs1(1, 0, 0)); + code.push_back(EncodeDs0(op, 20, true)); + code.push_back(EncodeDs1(2, 0, 0)); + code.push_back(EncodeSop1(0x10, 0, 126)); + code.push_back(EncodeVop2(0x26, 3, Vgpr(consume ? 1 : 2), consume ? 2 : 1)); + code.push_back(EncodeVop2(0x27, 3, 0, 3)); + code.push_back(EncodeVop1(0x06, 4, Vgpr(1))); + code.push_back(EncodeVop1(0x05, 5, Vgpr(3))); + code.push_back(EncodeVop1(0x06, 6, 0)); + code.push_back(EncodeVopc(0xc2, InlineU32(0), 7)); + code.push_back(EncodeSop1(0x24, 2, 106)); + code.push_back(EncodeSop2(0x11, 126, 126, 2)); + code.push_back(EncodeSop1(0x10, 1, 126)); + code.push_back(EncodeSop1(0x10, 4, 106)); + code.push_back(EncodeSop2(0x00, 1, 1, 4)); + code.push_back(EncodeVop1(0x06, 7, 1)); + code.push_back(EncodeExp0(0x00, 0xf)); + code.push_back(EncodeExp1(4, 5, 6, 7)); + AppendEnd(&code); + + GraphicsCase test; + test.name = consume ? "GraphicsDsConsumeSkipsHelperLanes" : "GraphicsDsAppendSkipsHelperLanes"; + test.fragment_code = std::move(code); + CoverOnePixelOfQuad(&test); + test.expected_pixel = {0, 0, 0, 0, 0x41200000u, 0, 0x3f800000u, 0x40000000u}; + test.opcodes = {O::V_INTERP_P1_F32, O::V_INTERP_P2_F32, O::S_MOV_B32, + consume ? O::DS_CONSUME : O::DS_APPEND, O::S_BCNT1_I32_B64, + O::V_SUB_NC_U32, O::V_SUBREV_NC_U32, O::V_CVT_F32_U32, O::V_CVT_F32_I32, + O::V_CMP_EQ_U32, O::S_AND_SAVEEXEC_B64, O::S_OR_B64, O::S_ADD_U32, O::EXP, + O::S_ENDPGM}; + return test; +} + +GraphicsCase GraphicsDsOrderedCountSkipsHelperLanes() { + using O = ShaderOpcode; + + // A live lane adds the first lane's ADDR value (1) for the wave. Exports the first counter + // value and the difference to the second. + std::vector code; + code.push_back(EncodeVintrp(0x00, 7, 0, 0, 0)); + code.push_back(EncodeVintrp(0x01, 7, 0, 0, 1)); + code.push_back(EncodeVop1(0x01, 2, InlineU32(1))); + AppendSMovLiteral(&code, 124, 0x00100000u); // M0 base 0x10; offset0 adds 4: dword 5. + code.push_back(EncodeDs0(0x3f, 7, true)); + code.push_back(EncodeDs1(1, 2, 2)); + code.push_back(EncodeDs0(0x3f, 7, true)); + code.push_back(EncodeDs1(3, 2, 2)); + code.push_back(EncodeVop2(0x26, 4, Vgpr(3), 1)); + code.push_back(EncodeVop1(0x06, 5, Vgpr(1))); + code.push_back(EncodeVop1(0x06, 6, Vgpr(4))); + code.push_back(EncodeExp0(0x00, 0xf)); + code.push_back(EncodeExp1(5, 6, 6, 6)); + AppendEnd(&code); + + GraphicsCase test; + test.name = "GraphicsDsOrderedCountSkipsHelperLanes"; + test.fragment_code = std::move(code); + CoverOnePixelOfQuad(&test); + test.expected_pixel = {0, 0, 0, 0, 0x41200000u, 0x3f800000u, 0x3f800000u, 0x3f800000u}; + test.opcodes = {O::V_INTERP_P1_F32, O::V_INTERP_P2_F32, O::V_MOV_B32, O::S_MOV_B32, + O::DS_ORDERED_COUNT, O::V_SUB_NC_U32, O::V_CVT_F32_U32, O::EXP, O::S_ENDPGM}; + return test; +} + GraphicsCase GraphicsDirectSgprPushConstantExport() { using O = ShaderOpcode; @@ -44970,6 +45097,9 @@ std::vector MakeGraphicsCases() { GraphicsFrontFaceBits(true, true), GraphicsFlatInterpolatorExport(), GraphicsDsAddtidScratchExport(), + GraphicsDsAppendSkipsHelperLanes(false), + GraphicsDsAppendSkipsHelperLanes(true), + GraphicsDsOrderedCountSkipsHelperLanes(), GraphicsDirectSgprPushConstantExport(), GraphicsInlineSrtScalarPromotionExport(), GraphicsNullVmExportDiscardsInactiveExec(), @@ -50967,6 +51097,13 @@ int main(int argc, char **argv) { RunCase(&vulkan, Vop3pIntegerNegationCapturedAndSelectedHalves()); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--helper-lanes-only") == 0) { + VulkanHarness vulkan; + RunGraphicsCase(&vulkan, GraphicsDsAppendSkipsHelperLanes(false)); + RunGraphicsCase(&vulkan, GraphicsDsAppendSkipsHelperLanes(true)); + RunGraphicsCase(&vulkan, GraphicsDsOrderedCountSkipsHelperLanes()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--dpp-only") == 0) { VulkanHarness vulkan; RunGraphicsCase(&vulkan, GraphicsSparseWaveMinimum()); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index d78c1ec9c..eb47d01df 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -13659,6 +13659,34 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { } } + // The wave32 live mask projects to the entry predicate, and VCC keeps its compare predicate + // after WQM. + const uint32_t live_mask_shader[] = { + EncodeSop1(0x03, 0, 126), // s_mov_b32 s0, exec_lo + EncodeSop2(0x0e, 126, 126, 0), // s_and_b32 exec_lo, exec_lo, s0 + EncodeVop1(0x01, 0, 242), // v_mov_b32 v0, 1.0 + EncodeExp0(0x00, 0x1), EncodeExp1(0, 0, 0, 0), + EncodeSopp(0x01), + }; + const uint32_t wqm_vcc_shader[] = { + EncodeSop1(0x09, 126, 126), // s_wqm_b32 exec_lo, exec_lo + EncodeVop2(0x23, 0, 193, 1), // v_mbcnt_lo_u32_b32 v0, -1, v1 + EncodeVopc(0xc1, 129, 0), // v_cmp_lt_u32 vcc_lo, 1, v0 + EncodeVop2(0x01, 2, 128, 0), // v_cndmask_b32 v2, 0, v0, vcc_lo + EncodeExp0(0x00, 0x1), EncodeExp1(2, 2, 2, 2), + EncodeSopp(0x01), + }; + auto wave32_pixel_options = MakeCompileOptions(ShaderType::Pixel); + wave32_pixel_options.wave_size = 32u; + for (const std::span shader : {std::span(live_mask_shader), + std::span(wqm_vcc_shader)}) { + const auto pixel_result = RecompileForTest(shader, wave32_pixel_options); + CheckSpirvBinaryValidates(pixel_result.spirv); + Check(DisassembleSpirvBinary(pixel_result.spirv).find("OpGroupNonUniformBallot") == + std::string::npos, + "wave32 pixel mask rebuilt a known predicate from a ballot"); + } + const uint32_t local_shader[] = { EncodeVopc(0xc1, 5 + 256, 8), // v_cmp_lt_u32 vcc, v5, v8 EncodeSop2(0x0f, 2, 126, 106), // s_and_b64 s[2:3], exec, vcc