diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index 12e87029c..b3ae0b661 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -302,23 +302,95 @@ void Translator::S_MOV_B64(const Decoder::Instruction& inst) { } void Translator::S_WQM(const Decoder::Instruction& inst, bool wide) { - const auto source = wide ? ReadU64(inst.src0) - : ir.ConstructU64(ReadU32(inst.src0), IR::U32(IR::Value(0u))); + const bool source_is_mask = inst.src0.kind == Decoder::OperandKind::Sgpr || + inst.src0.kind == Decoder::OperandKind::ExecLo || + inst.src0.kind == Decoder::OperandKind::VccLo || + (!wide && IsExecOrVcc(inst.src0)); + const auto source_valid = ReadMaskValid(inst.src0); + auto source_live = IR::U1(IR::Value(false)); + if (source_is_mask) { + if (inst.src0.kind == Decoder::OperandKind::VccLo || + inst.src0.kind == Decoder::OperandKind::VccHi) { + source_live = ir.GetVcc(); + } else { + source_live = ReadMask(inst.src0); + } + } + const auto retained_live = ir.LogicalAnd(source_valid, source_live); + + IR::U64 source; + if (wide) { + source = ReadU64(inst.src0); + } else { + source = ir.ConstructU64(ReadU32(inst.src0), IR::U32(IR::Value(0u))); + } const auto result = IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {source})); if (!wide) { const auto low = ir.CompositeExtract(result, 0); + const bool destination_is_mask = IsExecOrVcc(inst.dst); + const bool destination_is_exec = inst.dst.kind == Decoder::OperandKind::ExecLo || + inst.dst.kind == Decoder::OperandKind::ExecHi; + auto old_live = IR::U1(IR::Value(false)); + if (destination_is_mask) { + if (destination_is_exec) { + old_live = ir.GetExec(); + } else { + old_live = ir.GetVcc(); + } + } + // A 32-bit write preserves the other EXEC/VCC half and invalidates any // overlapping SGPR-pair mask provenance through the ordinary scalar path. WriteRawU32(inst.dst, low); + if (destination_is_mask) { + const auto is_high_half = [](const Decoder::Operand& operand) { + return operand.kind == Decoder::OperandKind::ExecHi || + operand.kind == Decoder::OperandKind::VccHi; + }; + const bool destination_is_high = is_high_half(inst.dst); + auto affected_lanes = IR::U1(IR::Value(!destination_is_high)); + if (program.wave_size == 64u) { + const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); + affected_lanes = ir.ULessThan(lane, IR::U32(IR::Value(32u))); + if (destination_is_high) { + affected_lanes = ir.LogicalNot(affected_lanes); + } + } + + IR::U1 live; + if (destination_is_exec) { + live = ir.GetExec(); + } else { + live = ir.GetVcc(); + } + // A cross-half copy refers to different lanes, so only scalar bits apply. + if (is_high_half(inst.src0) == destination_is_high) { + live = ir.LogicalOr(retained_live, live); + } + live = IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {affected_lanes, live, old_live})); + if (destination_is_exec) { + ir.SetExec(live); + } else { + ir.SetVcc(live); + } + } ir.SetScc(ir.INotEqual(low, IR::U32(IR::Value(0u)))); return; } - const auto mask_valid = ReadMaskValid(inst.src0); + + // WQM adds live lanes but never removes a source lane. Keep that fact separate + // from the scalar mask: reconstructing an entry-true predicate from a ballot + // otherwise introduces a spurious dependency into every vector write. + const auto live = ir.LogicalOr(retained_live, ThreadBit(ExtractU64(result))); WriteOperand(DestinationOperand(inst), result); - if (inst.dst.kind == Decoder::OperandKind::Sgpr) { + if (inst.dst.kind == Decoder::OperandKind::ExecLo) { + ir.SetExec(live); + } else if (inst.dst.kind == Decoder::OperandKind::VccLo) { + ir.SetVcc(live); + } else if (inst.dst.kind == Decoder::OperandKind::Sgpr) { const auto dst = static_cast(inst.dst.reg); - ir.SetThreadBitScalarReg(dst, ThreadBit(ExtractU64(result))); - ir.SetScalarMaskTag(dst, mask_valid); + ir.SetThreadBitScalarReg(dst, live); + ir.SetScalarMaskTag(dst, source_valid); } ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 542f28934..ee54d51e7 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -18527,6 +18527,25 @@ TestCase ScalarWqmB32Masks(u32 wave_size) { store_marker(lo, hi); } } + for (u32 base : {106u, 126u}) { + for (u32 half : {0u, 1u}) { + AppendVMovU32(&code, 2, 0); + code.push_back(EncodeSMovB32(107, InlineU32(0))); + code.push_back(EncodeVopc(0xc2, InlineU32(3), 0)); // Only lane 3. + code.push_back(EncodeSop1(0x04, base, 106)); + code.push_back(EncodeSop1(0x09, base + half, base + (half ^ 1u))); + code.push_back(EncodeSMovB32(20, base)); + code.push_back(EncodeSMovB32(21, base + 1)); + code.push_back(EncodeSop1(0x04, 126, base)); + AppendVMovU32(&code, 2, 9); + code.push_back(EncodeSop1(0x04, 126, 193)); + const u32 lo = half == 0 ? 0u : 8u; + const u32 hi = half == 0 ? 0u : 0xfu; + store_scalar(20, lo); + store_scalar(21, hi); + store_marker(lo, hi); + } + } AppendEnd(&code); test.initial.resize(test.expected.size()); test.opcodes = {O::S_WQM_B32, O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32, @@ -18596,8 +18615,21 @@ TestCase ScalarWqmB64PreservesPartialMasks() { code.push_back(EncodeSop1(0x0a, 10, 8)); code.push_back(EncodeSop1(0x04, 126, 10)); AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset); + code.push_back(EncodeSop1(0x04, 126, 16)); + for (u32 source : {253u, 127u, 107u}) { + if (source == 253u) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1))); // SCC=1. + } else { + AppendSMovLiteral(&code, source - 1, ~0u); + code.push_back(EncodeSMovB32(source, InlineU32(1))); + } + code.push_back(EncodeSop1(0x0a, 126, source)); + offset += 64; + AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset); + code.push_back(EncodeSop1(0x04, 126, 16)); + } AppendEnd(&code); - test.initial.assign(256, 0xdeadbeef); + test.initial.assign(448, 0xdeadbeef); test.expected = test.initial; for (u32 lane = 0; lane < 64; ++lane) { for (u32 region = 0; region < 3; ++region) { @@ -18606,9 +18638,12 @@ TestCase ScalarWqmB64PreservesPartialMasks() { if ((lane >= 4 && lane < 8) || (lane >= 32 && lane < 36)) { test.expected[192 + lane] = 9; } + for (u32 region = 4; region < 7; ++region) { + if (lane < 4) test.expected[region * 64 + lane] = 9; + } } test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::S_WQM_B64, O::V_MOV_B32, - O::V_CMP_EQ_U32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, + O::V_CMP_EQ_U32, O::S_CMP_EQ_U32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 64; test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 11155a25f..23f97b2d5 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -11508,6 +11508,27 @@ void TestNewShaderRecompilerVertexExportUsesInvocationExecMask() { } void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { + for (uint32_t wave_size : {32u, 64u}) { + for (uint32_t opcode : {0x09u, 0x0au}) { + for (uint32_t source : {126u, 8u}) { + const uint32_t pixel_shader[] = { + EncodeSop1(0x04, 8, 126), // Save the entry live mask. + EncodeSop1(opcode, 126, source), + EncodeVop1(0x01, 0, 242), // v_mov_b32 v0, 1.0 + EncodeExp0(0x00, 0x1), EncodeExp1(0, 0, 0, 0), + EncodeSopp(0x01), + }; + auto pixel_options = MakeCompileOptions(ShaderType::Pixel); + pixel_options.wave_size = wave_size; + const auto pixel_result = RecompileForTest(pixel_shader, pixel_options); + CheckSpirvBinaryValidates(pixel_result.spirv); + Check(DisassembleSpirvBinary(pixel_result.spirv).find("OpGroupNonUniformBallot") == + std::string::npos, + "entry WQM lost the known live predicate through a scalar ballot"); + } + } + } + const uint32_t local_shader[] = { EncodeVopc(0xc1, 5 + 256, 8), // v_cmp_lt_u32 vcc, v5, v8 EncodeSop2(0x0f, 2, 126, 106), // s_and_b64 s[2:3], exec, vcc