mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-20 10:56:58 +02:00
shader: fold the lane test after a pixel shader's entry s_wqm (#801)
Preserve live lane predicates through RDNA2 whole-quad expansion Keep the source-live predicate when WQM expands a scalar mask, so entry masks remain known live without an instruction pre-scan. Preserve scalar mask values, SCC, half writes, and raw-source behavior. Cover entry pixel masks, cross-half copies, and scalar/high-word sources with existing shader regression suites. Co-authored-by: nmzik <Nmzik@mail.ru> Co-authored-by: Douglas <douglasproducoes21@gmail.com>
This commit is contained in:
Douglas Soares Silva
committed by
GitHub
GitHub
parent
84d015120c
commit
fcf0f3b67e
@@ -302,23 +302,95 @@ void Translator::S_MOV_B64(const Decoder::Instruction& inst) {
|
||||
}
|
||||
|
||||
void Translator::S_WQM(const Decoder::Instruction& inst, bool wide) {
|
||||
const auto source = wide ? ReadU64(inst.src0)
|
||||
: ir.ConstructU64(ReadU32(inst.src0), IR::U32(IR::Value(0u)));
|
||||
const bool source_is_mask = inst.src0.kind == Decoder::OperandKind::Sgpr ||
|
||||
inst.src0.kind == Decoder::OperandKind::ExecLo ||
|
||||
inst.src0.kind == Decoder::OperandKind::VccLo ||
|
||||
(!wide && IsExecOrVcc(inst.src0));
|
||||
const auto source_valid = ReadMaskValid(inst.src0);
|
||||
auto source_live = IR::U1(IR::Value(false));
|
||||
if (source_is_mask) {
|
||||
if (inst.src0.kind == Decoder::OperandKind::VccLo ||
|
||||
inst.src0.kind == Decoder::OperandKind::VccHi) {
|
||||
source_live = ir.GetVcc();
|
||||
} else {
|
||||
source_live = ReadMask(inst.src0);
|
||||
}
|
||||
}
|
||||
const auto retained_live = ir.LogicalAnd(source_valid, source_live);
|
||||
|
||||
IR::U64 source;
|
||||
if (wide) {
|
||||
source = ReadU64(inst.src0);
|
||||
} else {
|
||||
source = ir.ConstructU64(ReadU32(inst.src0), IR::U32(IR::Value(0u)));
|
||||
}
|
||||
const auto result = IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {source}));
|
||||
if (!wide) {
|
||||
const auto low = ir.CompositeExtract(result, 0);
|
||||
const bool destination_is_mask = IsExecOrVcc(inst.dst);
|
||||
const bool destination_is_exec = inst.dst.kind == Decoder::OperandKind::ExecLo ||
|
||||
inst.dst.kind == Decoder::OperandKind::ExecHi;
|
||||
auto old_live = IR::U1(IR::Value(false));
|
||||
if (destination_is_mask) {
|
||||
if (destination_is_exec) {
|
||||
old_live = ir.GetExec();
|
||||
} else {
|
||||
old_live = ir.GetVcc();
|
||||
}
|
||||
}
|
||||
|
||||
// A 32-bit write preserves the other EXEC/VCC half and invalidates any
|
||||
// overlapping SGPR-pair mask provenance through the ordinary scalar path.
|
||||
WriteRawU32(inst.dst, low);
|
||||
if (destination_is_mask) {
|
||||
const auto is_high_half = [](const Decoder::Operand& operand) {
|
||||
return operand.kind == Decoder::OperandKind::ExecHi ||
|
||||
operand.kind == Decoder::OperandKind::VccHi;
|
||||
};
|
||||
const bool destination_is_high = is_high_half(inst.dst);
|
||||
auto affected_lanes = IR::U1(IR::Value(!destination_is_high));
|
||||
if (program.wave_size == 64u) {
|
||||
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
|
||||
affected_lanes = ir.ULessThan(lane, IR::U32(IR::Value(32u)));
|
||||
if (destination_is_high) {
|
||||
affected_lanes = ir.LogicalNot(affected_lanes);
|
||||
}
|
||||
}
|
||||
|
||||
IR::U1 live;
|
||||
if (destination_is_exec) {
|
||||
live = ir.GetExec();
|
||||
} else {
|
||||
live = ir.GetVcc();
|
||||
}
|
||||
// A cross-half copy refers to different lanes, so only scalar bits apply.
|
||||
if (is_high_half(inst.src0) == destination_is_high) {
|
||||
live = ir.LogicalOr(retained_live, live);
|
||||
}
|
||||
live = IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {affected_lanes, live, old_live}));
|
||||
if (destination_is_exec) {
|
||||
ir.SetExec(live);
|
||||
} else {
|
||||
ir.SetVcc(live);
|
||||
}
|
||||
}
|
||||
ir.SetScc(ir.INotEqual(low, IR::U32(IR::Value(0u))));
|
||||
return;
|
||||
}
|
||||
const auto mask_valid = ReadMaskValid(inst.src0);
|
||||
|
||||
// WQM adds live lanes but never removes a source lane. Keep that fact separate
|
||||
// from the scalar mask: reconstructing an entry-true predicate from a ballot
|
||||
// otherwise introduces a spurious dependency into every vector write.
|
||||
const auto live = ir.LogicalOr(retained_live, ThreadBit(ExtractU64(result)));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
|
||||
if (inst.dst.kind == Decoder::OperandKind::ExecLo) {
|
||||
ir.SetExec(live);
|
||||
} else if (inst.dst.kind == Decoder::OperandKind::VccLo) {
|
||||
ir.SetVcc(live);
|
||||
} else if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
|
||||
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
|
||||
ir.SetThreadBitScalarReg(dst, ThreadBit(ExtractU64(result)));
|
||||
ir.SetScalarMaskTag(dst, mask_valid);
|
||||
ir.SetThreadBitScalarReg(dst, live);
|
||||
ir.SetScalarMaskTag(dst, source_valid);
|
||||
}
|
||||
ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
|
||||
}
|
||||
|
||||
@@ -18527,6 +18527,25 @@ TestCase ScalarWqmB32Masks(u32 wave_size) {
|
||||
store_marker(lo, hi);
|
||||
}
|
||||
}
|
||||
for (u32 base : {106u, 126u}) {
|
||||
for (u32 half : {0u, 1u}) {
|
||||
AppendVMovU32(&code, 2, 0);
|
||||
code.push_back(EncodeSMovB32(107, InlineU32(0)));
|
||||
code.push_back(EncodeVopc(0xc2, InlineU32(3), 0)); // Only lane 3.
|
||||
code.push_back(EncodeSop1(0x04, base, 106));
|
||||
code.push_back(EncodeSop1(0x09, base + half, base + (half ^ 1u)));
|
||||
code.push_back(EncodeSMovB32(20, base));
|
||||
code.push_back(EncodeSMovB32(21, base + 1));
|
||||
code.push_back(EncodeSop1(0x04, 126, base));
|
||||
AppendVMovU32(&code, 2, 9);
|
||||
code.push_back(EncodeSop1(0x04, 126, 193));
|
||||
const u32 lo = half == 0 ? 0u : 8u;
|
||||
const u32 hi = half == 0 ? 0u : 0xfu;
|
||||
store_scalar(20, lo);
|
||||
store_scalar(21, hi);
|
||||
store_marker(lo, hi);
|
||||
}
|
||||
}
|
||||
AppendEnd(&code);
|
||||
test.initial.resize(test.expected.size());
|
||||
test.opcodes = {O::S_WQM_B32, O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32,
|
||||
@@ -18596,8 +18615,21 @@ TestCase ScalarWqmB64PreservesPartialMasks() {
|
||||
code.push_back(EncodeSop1(0x0a, 10, 8));
|
||||
code.push_back(EncodeSop1(0x04, 126, 10));
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
|
||||
code.push_back(EncodeSop1(0x04, 126, 16));
|
||||
for (u32 source : {253u, 127u, 107u}) {
|
||||
if (source == 253u) {
|
||||
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1))); // SCC=1.
|
||||
} else {
|
||||
AppendSMovLiteral(&code, source - 1, ~0u);
|
||||
code.push_back(EncodeSMovB32(source, InlineU32(1)));
|
||||
}
|
||||
code.push_back(EncodeSop1(0x0a, 126, source));
|
||||
offset += 64;
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
|
||||
code.push_back(EncodeSop1(0x04, 126, 16));
|
||||
}
|
||||
AppendEnd(&code);
|
||||
test.initial.assign(256, 0xdeadbeef);
|
||||
test.initial.assign(448, 0xdeadbeef);
|
||||
test.expected = test.initial;
|
||||
for (u32 lane = 0; lane < 64; ++lane) {
|
||||
for (u32 region = 0; region < 3; ++region) {
|
||||
@@ -18606,9 +18638,12 @@ TestCase ScalarWqmB64PreservesPartialMasks() {
|
||||
if ((lane >= 4 && lane < 8) || (lane >= 32 && lane < 36)) {
|
||||
test.expected[192 + lane] = 9;
|
||||
}
|
||||
for (u32 region = 4; region < 7; ++region) {
|
||||
if (lane < 4) test.expected[region * 64 + lane] = 9;
|
||||
}
|
||||
}
|
||||
test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::S_WQM_B64, O::V_MOV_B32,
|
||||
O::V_CMP_EQ_U32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD,
|
||||
O::V_CMP_EQ_U32, O::S_CMP_EQ_U32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD,
|
||||
O::S_ENDPGM};
|
||||
test.compute_info.threads_num[0] = 64;
|
||||
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
|
||||
|
||||
@@ -11508,6 +11508,27 @@ void TestNewShaderRecompilerVertexExportUsesInvocationExecMask() {
|
||||
}
|
||||
|
||||
void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
|
||||
for (uint32_t wave_size : {32u, 64u}) {
|
||||
for (uint32_t opcode : {0x09u, 0x0au}) {
|
||||
for (uint32_t source : {126u, 8u}) {
|
||||
const uint32_t pixel_shader[] = {
|
||||
EncodeSop1(0x04, 8, 126), // Save the entry live mask.
|
||||
EncodeSop1(opcode, 126, source),
|
||||
EncodeVop1(0x01, 0, 242), // v_mov_b32 v0, 1.0
|
||||
EncodeExp0(0x00, 0x1), EncodeExp1(0, 0, 0, 0),
|
||||
EncodeSopp(0x01),
|
||||
};
|
||||
auto pixel_options = MakeCompileOptions(ShaderType::Pixel);
|
||||
pixel_options.wave_size = wave_size;
|
||||
const auto pixel_result = RecompileForTest(pixel_shader, pixel_options);
|
||||
CheckSpirvBinaryValidates(pixel_result.spirv);
|
||||
Check(DisassembleSpirvBinary(pixel_result.spirv).find("OpGroupNonUniformBallot") ==
|
||||
std::string::npos,
|
||||
"entry WQM lost the known live predicate through a scalar ballot");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const uint32_t local_shader[] = {
|
||||
EncodeVopc(0xc1, 5 + 256, 8), // v_cmp_lt_u32 vcc, v5, v8
|
||||
EncodeSop2(0x0f, 2, 126, 106), // s_and_b64 s[2:3], exec, vcc
|
||||
|
||||
Reference in New Issue
Block a user