diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 7e7566158..8d26d7e73 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -821,18 +821,28 @@ uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst) { return ctx.Shuffle(inst, 0, ctx.Arg(inst, 1)); } -uint32_t EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst) { +template +static uint32_t EmitSubgroupReduceU32(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; auto value = ctx.Arg(inst, 0); if (ctx.other_half != nullptr) { - value = EmitUMin32(state, ctx.HalfArg(inst, 0, 0), ctx.HalfArg(inst, 0, 1)); + value = Combine(state, ctx.HalfArg(inst, 0, 0), ctx.HalfArg(inst, 0, 1)); } const auto result = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpGroupNonUniformUMin, TypeU32(state), result, - ConstantU32(state, spv::ScopeSubgroup), spv::GroupOperationReduce, value); + state.builder.AddFunction(Reduce, TypeU32(state), result, + ConstantU32(state, spv::ScopeSubgroup), spv::GroupOperationReduce, + value); return result; } +uint32_t EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst) { + return EmitSubgroupReduceU32(ctx, inst); +} + +uint32_t EmitSubgroupUMaxU32(ValueEmitContext& ctx, const IR::Inst& inst) { + return EmitSubgroupReduceU32(ctx, inst); +} + uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; const auto hit = state.builder.AllocateId(); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index d38814fd9..bd23c6bd0 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -234,6 +234,7 @@ uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& in uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitSubgroupUMaxU32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index f70aead14..965fdf3e6 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -235,7 +235,7 @@ void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) { const auto op = inst.GetOpcode(); if (ctx.half != 0 && (op == Op::Ballot || op == Op::ReadFirstLane || op == Op::SubgroupUMinU32 || - op == Op::ReadConstBuffer || op == Op::ReadRealtimeClock || + op == Op::SubgroupUMaxU32 || op == Op::ReadConstBuffer || op == Op::ReadRealtimeClock || (op == Op::LoadAddressU32 && ctx.Memory(inst).kind == IR::ResourceKind::ScalarAddress))) { // Whole-wave results, clocks and SMEM loads (SGPR operands, no EXEC) match in both halves. ctx.Define(inst, ctx.other_half->Result(inst)); diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index cfe27a3ce..a3ec856f9 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -222,6 +222,7 @@ VALUE_OPCODE(Ballot, U32x4, U1) VALUE_OPCODE(ReadFirstLane, U32, U32, U1) VALUE_OPCODE(ReadLane, U32, U32, U32) VALUE_OPCODE(SubgroupUMinU32, U32, U32) +VALUE_OPCODE(SubgroupUMaxU32, U32, U32) VALUE_OPCODE(WriteLane, U32, U32, U32, U32) VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1, U32) VALUE_OPCODE(PermuteU32, U32, U32, U32, U1) diff --git a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp index 52f2c200a..7e444aacd 100644 --- a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp @@ -99,28 +99,33 @@ Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_s return Value(&phi); } -// A full-EXEC prefix scan with both half-wave endpoint reads is a reduction. +// A full-EXEC prefix scan with both half-wave endpoint reads is a reduction. Wave64 first joins +// the two rows of each half; a wave32 half is one row. // Fragment subgroups may have no physical invocation at either endpoint. -class UMinReductionMatcher { +class WaveReductionMatcher { public: - explicit UMinReductionMatcher(Block* block): block(block) {} + WaveReductionMatcher(Block* block, ValueOpcode operation) + : block(block), operation(operation), + identity(operation == ValueOpcode::UMin32 ? UINT32_MAX : 0u) {} - Value Match(const Inst& minimum) { - const auto* low = Get(minimum.Arg(0), ValueOpcode::ReadLane); - const auto* high = Get(minimum.Arg(1), ValueOpcode::ReadLane); + Value Match(const Inst& reduction, uint32_t wave_size) { + const auto* low = Get(reduction.Arg(0), ValueOpcode::ReadLane); + const auto* high = Get(reduction.Arg(1), ValueOpcode::ReadLane); if (low == nullptr || high == nullptr || low->Arg(0).Resolve() != high->Arg(0).Resolve()) { return {}; } - const auto lane = [](Value value, uint32_t expected) { + const auto lane = [&](Value value, uint32_t expected) { value = value.Resolve(); return value.IsImmediate() && value.GetType() == Type::U32 && - value.U32() % 64u == expected; + value.U32() % wave_size == expected; }; - if (!((lane(low->Arg(1), 31) && lane(high->Arg(1), 63)) || - (lane(low->Arg(1), 63) && lane(high->Arg(1), 31)))) { + const auto half = wave_size / 2u; + if (!((lane(low->Arg(1), half - 1u) && lane(high->Arg(1), wave_size - 1u)) || + (lane(low->Arg(1), wave_size - 1u) && lane(high->Arg(1), half - 1u)))) { return {}; } - const auto* joined = Get(Unwrap(low->Arg(0)), ValueOpcode::UMin32); + if (wave_size == 32u) return MatchScan(Unwrap(low->Arg(0))); + const auto* joined = Get(Unwrap(low->Arg(0)), operation); if (joined == nullptr) return {}; for (size_t index = 0; index < 2; index++) { auto attempt = *this; @@ -134,26 +139,32 @@ public: !attempt.FullExec(cross->Arg(3)) || attempt.Unwrap(cross->Arg(0)) != prefix) { continue; } - for (const uint32_t shift: {8u, 4u, 2u, 1u}) { - prefix = attempt.MatchRow(prefix, shift); - } - prefix = attempt.Unwrap(prefix); - if (prefix.IsEmpty() || attempt.saved_exec.IsEmpty()) return prefix; - const auto* seed = attempt.Get(prefix, ValueOpcode::SelectU32); - if (seed != nullptr && seed->Arg(2).Resolve() == Value(UINT32_MAX)) { - const auto predicate = seed->Arg(0).Resolve(); - const auto* subset = attempt.Get(predicate, ValueOpcode::LogicalAnd); - if (predicate == attempt.saved_exec || - (subset != nullptr && (subset->Arg(0).Resolve() == attempt.saved_exec || - subset->Arg(1).Resolve() == attempt.saved_exec))) { - return prefix; - } - } + if (const auto seed = attempt.MatchScan(prefix); !seed.IsEmpty()) return seed; } return {}; } private: + // The row scan's input, whose lanes outside the saved EXEC hold the operation's identity. + Value MatchScan(Value prefix) { + for (const uint32_t shift: {8u, 4u, 2u, 1u}) { + prefix = MatchRow(prefix, shift); + } + prefix = Unwrap(prefix); + if (prefix.IsEmpty() || saved_exec.IsEmpty()) return prefix; + const auto* seed = Get(prefix, ValueOpcode::SelectU32); + if (seed != nullptr && seed->Arg(2).Resolve() == Value(identity)) { + const auto predicate = seed->Arg(0).Resolve(); + const auto* subset = Get(predicate, ValueOpcode::LogicalAnd); + if (predicate == saved_exec || + (subset != nullptr && (subset->Arg(0).Resolve() == saved_exec || + subset->Arg(1).Resolve() == saved_exec))) { + return prefix; + } + } + return {}; + } + Inst* Get(Value value, ValueOpcode opcode) const { auto* inst = value.Resolve().TryInstruction(); return inst != nullptr && inst->Parent() == block && inst->GetOpcode() == opcode @@ -192,19 +203,21 @@ private: Value MatchRow(Value value, uint32_t shift) { const auto* update = Get(Unwrap(value), ValueOpcode::DppUpdateU32); if (update == nullptr || !IsRowShift(*update, shift) || !FullExec(update->Arg(2))) return {}; - const auto* minimum = Get(update->Arg(0), ValueOpcode::UMin32); - if (minimum == nullptr) return {}; + const auto* combined = Get(update->Arg(0), operation); + if (combined == nullptr) return {}; for (size_t index = 0; index < 2; index++) { - const auto* move = Get(minimum->Arg(index), ValueOpcode::DppMoveU32); + const auto* move = Get(combined->Arg(index), ValueOpcode::DppMoveU32); if (move == nullptr || !IsRowShift(*move, shift) || !FullExec(move->Arg(1))) continue; - const auto previous = Unwrap(minimum->Arg(1 - index)); + const auto previous = Unwrap(combined->Arg(1 - index)); if (Unwrap(move->Arg(0)) == previous && Unwrap(update->Arg(1)) == previous) return previous; } return {}; } - Block* block; - Value saved_exec; + Block* block; + ValueOpcode operation; + uint32_t identity; + Value saved_exec; }; } // namespace @@ -217,18 +230,22 @@ ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size) { for (auto* block: program.blocks) { for (auto& inst: *block) { - if (wave_size == 64u && inst.GetOpcode() == ValueOpcode::UMin32) { - const auto seed = UMinReductionMatcher(block).Match(inst); + const auto opcode = inst.GetOpcode(); + if (opcode == ValueOpcode::UMin32 || opcode == ValueOpcode::UMax32) { + const auto seed = WaveReductionMatcher(block, opcode).Match(inst, wave_size); if (!seed.IsEmpty()) { const auto position = std::find_if(block->begin(), block->end(), [&](const Inst& candidate) { return &candidate == &inst; }); - auto& reduction = *block->PrependNewInst(position, ValueOpcode::SubgroupUMinU32, {seed}); + const auto reduce = opcode == ValueOpcode::UMin32 + ? ValueOpcode::SubgroupUMinU32 + : ValueOpcode::SubgroupUMaxU32; + auto& reduction = *block->PrependNewInst(position, reduce, {seed}); inst.ReplaceUsesWith(Value(&reduction)); stats.rewritten_reductions++; } continue; } - if (inst.GetOpcode() != ValueOpcode::ReadLane) { + if (opcode != ValueOpcode::ReadLane) { continue; } const auto selector = inst.Arg(1).Resolve(); diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index 744d84c56..14a624cfe 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -517,7 +517,8 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs inst.Flags() != CFG::BranchCondition::ScalarInstruction; break; case ValueOpcode::Ballot: info.subgroup_ballot = true; break; - case ValueOpcode::SubgroupUMinU32: info.subgroup_arithmetic = true; break; + case ValueOpcode::SubgroupUMinU32: + case ValueOpcode::SubgroupUMaxU32: info.subgroup_arithmetic = true; break; case ValueOpcode::DppMoveU32: case ValueOpcode::ReadFirstLane: case ValueOpcode::ReadLane: { diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index 7e589753a..04b361d46 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -628,17 +628,18 @@ void TestReadLaneElimination() { "dynamic-lane read was rewritten unsafely"); } -void TestCompleteUMinReduction() { +void TestCompleteWaveReduction() { enum class Case { - Masked, Subset, Literal, LiteralMasked, LiteralMove, Wave32, - UnrelatedSeed, UnmaskedSymbolic, WrongNeutral, RowMask, BankMask, + Masked, Subset, Literal, LiteralMasked, LiteralMove, Wave32, Wave32Rows, + Wave32Maximum, UnrelatedSeed, UnmaskedSymbolic, WrongNeutral, RowMask, BankMask, Shift, FetchInactive, BoundControl, Dpp8, MoveExec, UpdateExec, Previous, DppSource, CrossExec, CrossRow, CrossSelectorLow, CrossSelectorHigh, CrossFetch, CrossBound, CrossSource, FinalExec, Endpoint, EndpointSource, DifferentWitness, CrossBlock, WrongOperation }; for (const auto test : {Case::Masked, Case::Subset, Case::Literal, - Case::LiteralMasked, Case::LiteralMove, Case::Wave32, Case::UnrelatedSeed, + Case::LiteralMasked, Case::LiteralMove, Case::Wave32, Case::Wave32Rows, + Case::Wave32Maximum, Case::UnrelatedSeed, Case::UnmaskedSymbolic, Case::WrongNeutral, Case::RowMask, Case::BankMask, Case::Shift, Case::FetchInactive, Case::BoundControl, Case::Dpp8, Case::MoveExec, Case::UpdateExec, Case::Previous, Case::DppSource, @@ -657,13 +658,15 @@ void TestCompleteUMinReduction() { auto seed_mask = test == Case::UnrelatedSeed ? other_mask : mask; if (test == Case::Subset) seed_mask = fixture.Emit(ValueOpcode::LogicalAnd, {other_mask, mask}); + // A UMax scan seeds lanes outside EXEC with zero. + const bool maximum = test == Case::Wave32Maximum; + const uint32_t neutral = test == Case::WrongNeutral || maximum ? 0u : UINT32_MAX; const auto seed = test == Case::Literal || test == Case::UnmaskedSymbolic ? source : - fixture.Emit(ValueOpcode::SelectU32, - {seed_mask, source, Value(test == Case::WrongNeutral ? 0u : UINT32_MAX)}); + fixture.Emit(ValueOpcode::SelectU32, {seed_mask, source, Value(neutral)}); auto prefix = fixture.Emit(ValueOpcode::SelectU32, {enabled, seed, source}); Value preserved; - const auto operation = test == Case::WrongOperation ? ValueOpcode::BitwiseOr32 : - ValueOpcode::UMin32; + auto operation = maximum ? ValueOpcode::UMax32 : ValueOpcode::UMin32; + if (test == Case::WrongOperation) operation = ValueOpcode::BitwiseOr32; for (const uint32_t shift : {1u, 2u, 4u, 8u}) { DppMoveFlags flags{.control = 0x110u + shift}; if (shift == 2) { @@ -704,10 +707,16 @@ void TestCompleteUMinReduction() { const auto final = fixture.Emit(ValueOpcode::SelectU32, {test == Case::FinalExec ? mask : enabled, combined, prefix}); const uint32_t result_block = test == Case::CrossBlock ? 1u : 0u; + // Wave32 reads each half's row endpoint from the scan itself. + const bool rows = test == Case::Wave32Rows || maximum; + const auto endpoints = rows ? prefix : final; + uint32_t low_lane = test == Case::Endpoint ? 30u : 31u; + if (rows) low_lane = 15u; const auto low = fixture.Emit(ValueOpcode::ReadLane, - {final, Value(test == Case::Endpoint ? 30u : 31u)}, 0, result_block); + {endpoints, Value(low_lane)}, 0, result_block); const auto high = fixture.Emit(ValueOpcode::ReadLane, - {test == Case::EndpointSource ? prefix : final, Value(63u)}, 0, result_block); + {test == Case::EndpointSource ? prefix : endpoints, Value(rows ? 31u : 63u)}, 0, + result_block); const auto result = fixture.Emit(operation, {high, low}, 0, result_block); const auto use = fixture.Emit(ValueOpcode::ReferenceU32, {result}, 0, result_block); using Terminator = Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind; @@ -718,16 +727,19 @@ void TestCompleteUMinReduction() { fixture.BlockAt().terminator.true_block = &fixture.BlockAt(1); } ValidateProgram(fixture.program, true); - const auto stats = EliminateReadLane(fixture.program, test == Case::Wave32 ? 32u : 64u); + const auto stats = + EliminateReadLane(fixture.program, test == Case::Wave32 || rows ? 32u : 64u); const bool matches = test == Case::Masked || test == Case::Subset || - test == Case::Literal || test == Case::LiteralMasked || test == Case::LiteralMove; + test == Case::Literal || test == Case::LiteralMasked || test == Case::LiteralMove || + rows; Check(stats.rewritten_reductions == (matches ? 1u : 0u) && stats.rewritten_reads == 0, - ("UMin reduction proof failed case " + std::to_string(static_cast(test))).c_str()); + ("wave reduction proof failed case " + std::to_string(static_cast(test))).c_str()); if (matches) { const auto *reduction = use.Instruction()->Arg(0).ResolveInstruction(); - Check(reduction->GetOpcode() == ValueOpcode::SubgroupUMinU32 && + Check(reduction->GetOpcode() == + (maximum ? ValueOpcode::SubgroupUMaxU32 : ValueOpcode::SubgroupUMinU32) && reduction->Arg(0).Resolve() == seed.Resolve(), - "UMin reduction changed the neutral-masked input"); + "wave reduction changed the neutral-masked input"); RemoveIdentities(fixture.program); EliminateDeadCode(fixture.program.blocks); uint32_t updates = 0; @@ -735,10 +747,10 @@ void TestCompleteUMinReduction() { updates += inst.GetOpcode() == ValueOpcode::DppUpdateU32; Check(inst.GetOpcode() != ValueOpcode::ReadLane && inst.GetOpcode() != ValueOpcode::Permlane16U32, - "UMin reduction retained unused endpoint or cross-row operations"); + "wave reduction retained unused endpoint or cross-row operations"); } Check(updates == 2 && preserved.Instruction()->Arg(0).Instruction()->HasUses(), - "UMin reduction destroyed an independently consumed prefix or kept its dead suffix"); + "wave reduction destroyed an independently consumed prefix or kept its dead suffix"); ValidateProgram(fixture.program, true); } else { Check(use.Instruction()->Arg(0).Resolve() == result, @@ -884,7 +896,7 @@ int main() { TestSharedIntegerRuntimeDependencies(); TestConstantBufferBounds(); TestReadLaneElimination(); - TestCompleteUMinReduction(); + TestCompleteWaveReduction(); TestOptimizationPipeline(); TestControlFlowValueSurvivesReadLaneFolding(); TestDeadPhiCyclesAndPlanningRoots();