raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-26 12:56:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

shader: lower wave32 whole-wave UMin and UMax reductions to subgroup reductions

Browse Source 
Co-authored-by: Senaxx <michelbeld@gmail.com>
Co-authored-by: Mac <me@itsmemac.com>
This commit is contained in:
nmzik
2026-10-26 01:12:50 +01:00
parent be1c84f4d7
commit 6357740fa1
7 changed files with 102 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
+14 -4
View File
@@ -821,18 +821,28 @@ uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst) {
return ctx.Shuffle(inst, 0, ctx.Arg(inst, 1));
}
uint32_t EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst) {
template <spv::Op Reduce, auto Combine>
static uint32_t EmitSubgroupReduceU32(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
auto value = ctx.Arg(inst, 0);
if (ctx.other_half != nullptr) {
value = EmitUMin32(state, ctx.HalfArg(inst, 0, 0), ctx.HalfArg(inst, 0, 1));
value = Combine(state, ctx.HalfArg(inst, 0, 0), ctx.HalfArg(inst, 0, 1));
}
const auto result = state.builder.AllocateId();
state.builder.AddFunction(spv::OpGroupNonUniformUMin, TypeU32(state), result,
ConstantU32(state, spv::ScopeSubgroup), spv::GroupOperationReduce, value);
state.builder.AddFunction(Reduce, TypeU32(state), result,
ConstantU32(state, spv::ScopeSubgroup), spv::GroupOperationReduce,
value);
return result;
}
uint32_t EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst) {
return EmitSubgroupReduceU32<spv::OpGroupNonUniformUMin, EmitUMin32>(ctx, inst);
}
uint32_t EmitSubgroupUMaxU32(ValueEmitContext& ctx, const IR::Inst& inst) {
return EmitSubgroupReduceU32<spv::OpGroupNonUniformUMax, EmitUMax32>(ctx, inst);
}
uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto hit = state.builder.AllocateId();
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h
+1
View File
@@ -234,6 +234,7 @@ uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& in
uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitSubgroupUMaxU32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
+1 -1
View File
@@ -235,7 +235,7 @@ void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto op = inst.GetOpcode();
if (ctx.half != 0 &&
(op == Op::Ballot || op == Op::ReadFirstLane || op == Op::SubgroupUMinU32 ||
op == Op::ReadConstBuffer || op == Op::ReadRealtimeClock ||
op == Op::SubgroupUMaxU32 || op == Op::ReadConstBuffer || op == Op::ReadRealtimeClock ||
(op == Op::LoadAddressU32 && ctx.Memory(inst).kind == IR::ResourceKind::ScalarAddress))) {
// Whole-wave results, clocks and SMEM loads (SGPR operands, no EXEC) match in both halves.
ctx.Define(inst, ctx.other_half->Result(inst));
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc
+1
View File
@@ -222,6 +222,7 @@ VALUE_OPCODE(Ballot, U32x4, U1)
VALUE_OPCODE(ReadFirstLane, U32, U32, U1)
VALUE_OPCODE(ReadLane, U32, U32, U32)
VALUE_OPCODE(SubgroupUMinU32, U32, U32)
VALUE_OPCODE(SubgroupUMaxU32, U32, U32)
VALUE_OPCODE(WriteLane, U32, U32, U32, U32)
VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1, U32)
VALUE_OPCODE(PermuteU32, U32, U32, U32, U1)
src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp
+53 -36
View File
@@ -99,28 +99,33 @@ Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_s
return Value(&phi);
}
// A full-EXEC prefix scan with both half-wave endpoint reads is a reduction.
// A full-EXEC prefix scan with both half-wave endpoint reads is a reduction. Wave64 first joins
// the two rows of each half; a wave32 half is one row.
// Fragment subgroups may have no physical invocation at either endpoint.
class UMinReductionMatcher {
class WaveReductionMatcher {
public:
explicit UMinReductionMatcher(Block* block): block(block) {}
WaveReductionMatcher(Block* block, ValueOpcode operation)
: block(block), operation(operation),
identity(operation == ValueOpcode::UMin32 ? UINT32_MAX : 0u) {}
Value Match(const Inst& minimum) {
const auto* low = Get(minimum.Arg(0), ValueOpcode::ReadLane);
const auto* high = Get(minimum.Arg(1), ValueOpcode::ReadLane);
Value Match(const Inst& reduction, uint32_t wave_size) {
const auto* low = Get(reduction.Arg(0), ValueOpcode::ReadLane);
const auto* high = Get(reduction.Arg(1), ValueOpcode::ReadLane);
if (low == nullptr || high == nullptr || low->Arg(0).Resolve() != high->Arg(0).Resolve()) {
return {};
}
const auto lane = [](Value value, uint32_t expected) {
const auto lane = [&](Value value, uint32_t expected) {
value = value.Resolve();
return value.IsImmediate() && value.GetType() == Type::U32 &&
value.U32() % 64u == expected;
value.U32() % wave_size == expected;
};
if (!((lane(low->Arg(1), 31) && lane(high->Arg(1), 63)) ||
(lane(low->Arg(1), 63) && lane(high->Arg(1), 31)))) {
const auto half = wave_size / 2u;
if (!((lane(low->Arg(1), half - 1u) && lane(high->Arg(1), wave_size - 1u)) ||
(lane(low->Arg(1), wave_size - 1u) && lane(high->Arg(1), half - 1u)))) {
return {};
}
const auto* joined = Get(Unwrap(low->Arg(0)), ValueOpcode::UMin32);
if (wave_size == 32u) return MatchScan(Unwrap(low->Arg(0)));
const auto* joined = Get(Unwrap(low->Arg(0)), operation);
if (joined == nullptr) return {};
for (size_t index = 0; index < 2; index++) {
auto attempt = *this;
@@ -134,26 +139,32 @@ public:
!attempt.FullExec(cross->Arg(3)) || attempt.Unwrap(cross->Arg(0)) != prefix) {
continue;
}
for (const uint32_t shift: {8u, 4u, 2u, 1u}) {
prefix = attempt.MatchRow(prefix, shift);
}
prefix = attempt.Unwrap(prefix);
if (prefix.IsEmpty() || attempt.saved_exec.IsEmpty()) return prefix;
const auto* seed = attempt.Get(prefix, ValueOpcode::SelectU32);
if (seed != nullptr && seed->Arg(2).Resolve() == Value(UINT32_MAX)) {
const auto predicate = seed->Arg(0).Resolve();
const auto* subset = attempt.Get(predicate, ValueOpcode::LogicalAnd);
if (predicate == attempt.saved_exec ||
(subset != nullptr && (subset->Arg(0).Resolve() == attempt.saved_exec ||
subset->Arg(1).Resolve() == attempt.saved_exec))) {
return prefix;
}
}
if (const auto seed = attempt.MatchScan(prefix); !seed.IsEmpty()) return seed;
}
return {};
}
private:
// The row scan's input, whose lanes outside the saved EXEC hold the operation's identity.
Value MatchScan(Value prefix) {
for (const uint32_t shift: {8u, 4u, 2u, 1u}) {
prefix = MatchRow(prefix, shift);
}
prefix = Unwrap(prefix);
if (prefix.IsEmpty() || saved_exec.IsEmpty()) return prefix;
const auto* seed = Get(prefix, ValueOpcode::SelectU32);
if (seed != nullptr && seed->Arg(2).Resolve() == Value(identity)) {
const auto predicate = seed->Arg(0).Resolve();
const auto* subset = Get(predicate, ValueOpcode::LogicalAnd);
if (predicate == saved_exec ||
(subset != nullptr && (subset->Arg(0).Resolve() == saved_exec ||
subset->Arg(1).Resolve() == saved_exec))) {
return prefix;
}
}
return {};
}
Inst* Get(Value value, ValueOpcode opcode) const {
auto* inst = value.Resolve().TryInstruction();
return inst != nullptr && inst->Parent() == block && inst->GetOpcode() == opcode
@@ -192,19 +203,21 @@ private:
Value MatchRow(Value value, uint32_t shift) {
const auto* update = Get(Unwrap(value), ValueOpcode::DppUpdateU32);
if (update == nullptr || !IsRowShift(*update, shift) || !FullExec(update->Arg(2))) return {};
const auto* minimum = Get(update->Arg(0), ValueOpcode::UMin32);
if (minimum == nullptr) return {};
const auto* combined = Get(update->Arg(0), operation);
if (combined == nullptr) return {};
for (size_t index = 0; index < 2; index++) {
const auto* move = Get(minimum->Arg(index), ValueOpcode::DppMoveU32);
const auto* move = Get(combined->Arg(index), ValueOpcode::DppMoveU32);
if (move == nullptr || !IsRowShift(*move, shift) || !FullExec(move->Arg(1))) continue;
const auto previous = Unwrap(minimum->Arg(1 - index));
const auto previous = Unwrap(combined->Arg(1 - index));
if (Unwrap(move->Arg(0)) == previous && Unwrap(update->Arg(1)) == previous) return previous;
}
return {};
}
Block* block;
Value saved_exec;
Block* block;
ValueOpcode operation;
uint32_t identity;
Value saved_exec;
};
} // namespace
@@ -217,18 +230,22 @@ ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size) {
for (auto* block: program.blocks) {
for (auto& inst: *block) {
if (wave_size == 64u && inst.GetOpcode() == ValueOpcode::UMin32) {
const auto seed = UMinReductionMatcher(block).Match(inst);
const auto opcode = inst.GetOpcode();
if (opcode == ValueOpcode::UMin32 || opcode == ValueOpcode::UMax32) {
const auto seed = WaveReductionMatcher(block, opcode).Match(inst, wave_size);
if (!seed.IsEmpty()) {
const auto position = std::find_if(block->begin(), block->end(),
[&](const Inst& candidate) { return &candidate == &inst; });
auto& reduction = *block->PrependNewInst(position, ValueOpcode::SubgroupUMinU32, {seed});
const auto reduce = opcode == ValueOpcode::UMin32
? ValueOpcode::SubgroupUMinU32
: ValueOpcode::SubgroupUMaxU32;
auto& reduction = *block->PrependNewInst(position, reduce, {seed});
inst.ReplaceUsesWith(Value(&reduction));
stats.rewritten_reductions++;
}
continue;
}
if (inst.GetOpcode() != ValueOpcode::ReadLane) {
if (opcode != ValueOpcode::ReadLane) {
continue;
}
const auto selector = inst.Arg(1).Resolve();
src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp
+2 -1
View File
@@ -517,7 +517,8 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
inst.Flags<CFG::BranchCondition>() != CFG::BranchCondition::ScalarInstruction;
break;
case ValueOpcode::Ballot: info.subgroup_ballot = true; break;
case ValueOpcode::SubgroupUMinU32: info.subgroup_arithmetic = true; break;
case ValueOpcode::SubgroupUMinU32:
case ValueOpcode::SubgroupUMaxU32: info.subgroup_arithmetic = true; break;
case ValueOpcode::DppMoveU32:
case ValueOpcode::ReadFirstLane:
case ValueOpcode::ReadLane: {
tests/ScalarProvenanceTests.cpp
+30 -18
View File
@@ -628,17 +628,18 @@ void TestReadLaneElimination() {
"dynamic-lane read was rewritten unsafely");
}
void TestCompleteUMinReduction() {
void TestCompleteWaveReduction() {
enum class Case {
Masked, Subset, Literal, LiteralMasked, LiteralMove, Wave32,
UnrelatedSeed, UnmaskedSymbolic, WrongNeutral, RowMask, BankMask,
Masked, Subset, Literal, LiteralMasked, LiteralMove, Wave32, Wave32Rows,
Wave32Maximum, UnrelatedSeed, UnmaskedSymbolic, WrongNeutral, RowMask, BankMask,
Shift, FetchInactive, BoundControl, Dpp8, MoveExec, UpdateExec,
Previous, DppSource, CrossExec, CrossRow, CrossSelectorLow,
CrossSelectorHigh, CrossFetch, CrossBound, CrossSource, FinalExec,
Endpoint, EndpointSource, DifferentWitness, CrossBlock, WrongOperation
};
for (const auto test : {Case::Masked, Case::Subset, Case::Literal,
Case::LiteralMasked, Case::LiteralMove, Case::Wave32, Case::UnrelatedSeed,
Case::LiteralMasked, Case::LiteralMove, Case::Wave32, Case::Wave32Rows,
Case::Wave32Maximum, Case::UnrelatedSeed,
Case::UnmaskedSymbolic, Case::WrongNeutral, Case::RowMask, Case::BankMask,
Case::Shift, Case::FetchInactive, Case::BoundControl, Case::Dpp8,
Case::MoveExec, Case::UpdateExec, Case::Previous, Case::DppSource,
@@ -657,13 +658,15 @@ void TestCompleteUMinReduction() {
auto seed_mask = test == Case::UnrelatedSeed ? other_mask : mask;
if (test == Case::Subset)
seed_mask = fixture.Emit(ValueOpcode::LogicalAnd, {other_mask, mask});
// A UMax scan seeds lanes outside EXEC with zero.
const bool maximum = test == Case::Wave32Maximum;
const uint32_t neutral = test == Case::WrongNeutral || maximum ? 0u : UINT32_MAX;
const auto seed = test == Case::Literal || test == Case::UnmaskedSymbolic ? source :
fixture.Emit(ValueOpcode::SelectU32,
{seed_mask, source, Value(test == Case::WrongNeutral ? 0u : UINT32_MAX)});
fixture.Emit(ValueOpcode::SelectU32, {seed_mask, source, Value(neutral)});
auto prefix = fixture.Emit(ValueOpcode::SelectU32, {enabled, seed, source});
Value preserved;
const auto operation = test == Case::WrongOperation ? ValueOpcode::BitwiseOr32 :
ValueOpcode::UMin32;
auto operation = maximum ? ValueOpcode::UMax32 : ValueOpcode::UMin32;
if (test == Case::WrongOperation) operation = ValueOpcode::BitwiseOr32;
for (const uint32_t shift : {1u, 2u, 4u, 8u}) {
DppMoveFlags flags{.control = 0x110u + shift};
if (shift == 2) {
@@ -704,10 +707,16 @@ void TestCompleteUMinReduction() {
const auto final = fixture.Emit(ValueOpcode::SelectU32,
{test == Case::FinalExec ? mask : enabled, combined, prefix});
const uint32_t result_block = test == Case::CrossBlock ? 1u : 0u;
// Wave32 reads each half's row endpoint from the scan itself.
const bool rows = test == Case::Wave32Rows || maximum;
const auto endpoints = rows ? prefix : final;
uint32_t low_lane = test == Case::Endpoint ? 30u : 31u;
if (rows) low_lane = 15u;
const auto low = fixture.Emit(ValueOpcode::ReadLane,
{final, Value(test == Case::Endpoint ? 30u : 31u)}, 0, result_block);
{endpoints, Value(low_lane)}, 0, result_block);
const auto high = fixture.Emit(ValueOpcode::ReadLane,
{test == Case::EndpointSource ? prefix : final, Value(63u)}, 0, result_block);
{test == Case::EndpointSource ? prefix : endpoints, Value(rows ? 31u : 63u)}, 0,
result_block);
const auto result = fixture.Emit(operation, {high, low}, 0, result_block);
const auto use = fixture.Emit(ValueOpcode::ReferenceU32, {result}, 0, result_block);
using Terminator = Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind;
@@ -718,16 +727,19 @@ void TestCompleteUMinReduction() {
fixture.BlockAt().terminator.true_block = &fixture.BlockAt(1);
}
ValidateProgram(fixture.program, true);
const auto stats = EliminateReadLane(fixture.program, test == Case::Wave32 ? 32u : 64u);
const auto stats =
EliminateReadLane(fixture.program, test == Case::Wave32 || rows ? 32u : 64u);
const bool matches = test == Case::Masked || test == Case::Subset ||
test == Case::Literal || test == Case::LiteralMasked || test == Case::LiteralMove;
test == Case::Literal || test == Case::LiteralMasked || test == Case::LiteralMove ||
rows;
Check(stats.rewritten_reductions == (matches ? 1u : 0u) && stats.rewritten_reads == 0,
("UMin reduction proof failed case " + std::to_string(static_cast<unsigned>(test))).c_str());
("wave reduction proof failed case " + std::to_string(static_cast<unsigned>(test))).c_str());
if (matches) {
const auto *reduction = use.Instruction()->Arg(0).ResolveInstruction();
Check(reduction->GetOpcode() == ValueOpcode::SubgroupUMinU32 &&
Check(reduction->GetOpcode() ==
(maximum ? ValueOpcode::SubgroupUMaxU32 : ValueOpcode::SubgroupUMinU32) &&
reduction->Arg(0).Resolve() == seed.Resolve(),
"UMin reduction changed the neutral-masked input");
"wave reduction changed the neutral-masked input");
RemoveIdentities(fixture.program);
EliminateDeadCode(fixture.program.blocks);
uint32_t updates = 0;
@@ -735,10 +747,10 @@ void TestCompleteUMinReduction() {
updates += inst.GetOpcode() == ValueOpcode::DppUpdateU32;
Check(inst.GetOpcode() != ValueOpcode::ReadLane &&
inst.GetOpcode() != ValueOpcode::Permlane16U32,
"UMin reduction retained unused endpoint or cross-row operations");
"wave reduction retained unused endpoint or cross-row operations");
}
Check(updates == 2 && preserved.Instruction()->Arg(0).Instruction()->HasUses(),
"UMin reduction destroyed an independently consumed prefix or kept its dead suffix");
"wave reduction destroyed an independently consumed prefix or kept its dead suffix");
ValidateProgram(fixture.program, true);
} else {
Check(use.Instruction()->Arg(0).Resolve() == result,
@@ -884,7 +896,7 @@ int main() {
TestSharedIntegerRuntimeDependencies();
TestConstantBufferBounds();
TestReadLaneElimination();
TestCompleteUMinReduction();
TestCompleteWaveReduction();
TestOptimizationPipeline();
TestControlFlowValueSurvivesReadLaneFolding();
TestDeadPhiCyclesAndPlanningRoots();