diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 3612aa9ae..f7029eebb 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -797,9 +797,9 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO }; simplify(); const auto read_lane_stats = IR::EliminateReadLane(ir, ir.wave_size); - if (read_lane_stats.rewritten_reads != 0) { - LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options), - read_lane_stats.rewritten_reads); + if (read_lane_stats.rewritten_reads != 0 || read_lane_stats.rewritten_reductions != 0) { + LOGF("%s read-lane elimination: reads=%" PRIu32 " reductions=%" PRIu32 "\n", + GetDumpLabel(options), read_lane_stats.rewritten_reads, read_lane_stats.rewritten_reductions); } // Cleanup also exposes profitable folds by removing dead predicate consumers. simplify(); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 33a20fcbd..ae4d6ca43 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -748,6 +748,18 @@ uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst) { return ctx.Shuffle(inst, 0, ctx.Arg(inst, 1)); } +uint32_t EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + auto value = ctx.Arg(inst, 0); + if (ctx.other_half != nullptr) { + value = EmitUMin32(state, ctx.HalfArg(inst, 0, 0), ctx.HalfArg(inst, 0, 1)); + } + const auto result = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpGroupNonUniformUMin, TypeU32(state), result, + ConstantU32(state, spv::ScopeSubgroup), spv::GroupOperationReduce, value); + return result; +} + uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; const auto hit = state.builder.AllocateId(); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index f06c8b2c5..ee7f753b8 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -238,6 +238,7 @@ uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate); uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 1ac889435..c18919898 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -766,7 +766,7 @@ void DefineModule(EmitterState& state) { } if (state.lane_count == 2 || state.program.info.subgroup_barrier || state.program.info.subgroup_ballot || state.program.info.subgroup_shuffle || - state.program.info.subgroup_local_invocation_id) { + state.program.info.subgroup_arithmetic || state.program.info.subgroup_local_invocation_id) { state.builder.RequireCapability(spv::CapabilityGroupNonUniform); } if (state.lane_count == 2 || state.program.info.subgroup_ballot) { @@ -775,6 +775,9 @@ void DefineModule(EmitterState& state) { if (state.program.info.subgroup_shuffle) { state.builder.RequireCapability(spv::CapabilityGroupNonUniformShuffle); } + if (state.program.info.subgroup_arithmetic) { + state.builder.RequireCapability(spv::CapabilityGroupNonUniformArithmetic); + } if (state.program.info.compute_derivatives && state.program.stage == ShaderType::Compute) { state.builder.RequireCapability(spv::CapabilityComputeDerivativeGroupQuadsKHR); state.builder.RequireExtension("SPV_KHR_compute_shader_derivatives"); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 0952ffb9d..72b6b1229 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -226,8 +226,9 @@ void Invoke(Return (*emit)(Context&, Args...), ValueEmitContext& ctx, const IR:: void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) { if (ctx.half != 0 && (inst.GetOpcode() == IR::ValueOpcode::Ballot || - inst.GetOpcode() == IR::ValueOpcode::ReadFirstLane)) { - // Both operations already combine both emulated halves into one whole-wave result. + inst.GetOpcode() == IR::ValueOpcode::ReadFirstLane || + inst.GetOpcode() == IR::ValueOpcode::SubgroupUMinU32)) { + // These operations combine both emulated halves into one whole-wave result. ctx.Define(inst, ctx.other_half->Result(inst)); return; } diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 462aeee7a..3d0cf961f 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -507,6 +507,7 @@ struct ShaderInfo { bool subgroup_ballot = false; bool subgroup_barrier = false; bool subgroup_shuffle = false; + bool subgroup_arithmetic = false; bool subgroup_local_invocation_id = false; bool compute_derivatives = false; bool image_gather_extended = false; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 1f5196495..4d6772f5b 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -226,6 +226,7 @@ VALUE_OPCODE(LaneId, U32) VALUE_OPCODE(Ballot, U32x4, U1) VALUE_OPCODE(ReadFirstLane, U32, U32, U1) VALUE_OPCODE(ReadLane, U32, U32, U32) +VALUE_OPCODE(SubgroupUMinU32, U32, U32) VALUE_OPCODE(WriteLane, U32, U32, U32, U32) VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1) VALUE_OPCODE(PermuteU32, U32, U32, U32, U1) diff --git a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp index 6ba46a853..52f2c200a 100644 --- a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp @@ -99,6 +99,114 @@ Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_s return Value(&phi); } +// A full-EXEC prefix scan with both half-wave endpoint reads is a reduction. +// Fragment subgroups may have no physical invocation at either endpoint. +class UMinReductionMatcher { +public: + explicit UMinReductionMatcher(Block* block): block(block) {} + + Value Match(const Inst& minimum) { + const auto* low = Get(minimum.Arg(0), ValueOpcode::ReadLane); + const auto* high = Get(minimum.Arg(1), ValueOpcode::ReadLane); + if (low == nullptr || high == nullptr || low->Arg(0).Resolve() != high->Arg(0).Resolve()) { + return {}; + } + const auto lane = [](Value value, uint32_t expected) { + value = value.Resolve(); + return value.IsImmediate() && value.GetType() == Type::U32 && + value.U32() % 64u == expected; + }; + if (!((lane(low->Arg(1), 31) && lane(high->Arg(1), 63)) || + (lane(low->Arg(1), 63) && lane(high->Arg(1), 31)))) { + return {}; + } + const auto* joined = Get(Unwrap(low->Arg(0)), ValueOpcode::UMin32); + if (joined == nullptr) return {}; + for (size_t index = 0; index < 2; index++) { + auto attempt = *this; + const auto* cross = attempt.Get(attempt.Unwrap(joined->Arg(index)), ValueOpcode::Permlane16U32); + if (cross == nullptr) continue; + const auto flags = cross->Flags(); + auto prefix = attempt.Unwrap(joined->Arg(1 - index)); + if (!flags.x16 || flags.fetch_inactive || !flags.bound_control || + cross->Arg(1).Resolve() != Value(UINT32_MAX) || + cross->Arg(2).Resolve() != Value(UINT32_MAX) || + !attempt.FullExec(cross->Arg(3)) || attempt.Unwrap(cross->Arg(0)) != prefix) { + continue; + } + for (const uint32_t shift: {8u, 4u, 2u, 1u}) { + prefix = attempt.MatchRow(prefix, shift); + } + prefix = attempt.Unwrap(prefix); + if (prefix.IsEmpty() || attempt.saved_exec.IsEmpty()) return prefix; + const auto* seed = attempt.Get(prefix, ValueOpcode::SelectU32); + if (seed != nullptr && seed->Arg(2).Resolve() == Value(UINT32_MAX)) { + const auto predicate = seed->Arg(0).Resolve(); + const auto* subset = attempt.Get(predicate, ValueOpcode::LogicalAnd); + if (predicate == attempt.saved_exec || + (subset != nullptr && (subset->Arg(0).Resolve() == attempt.saved_exec || + subset->Arg(1).Resolve() == attempt.saved_exec))) { + return prefix; + } + } + } + return {}; + } + +private: + Inst* Get(Value value, ValueOpcode opcode) const { + auto* inst = value.Resolve().TryInstruction(); + return inst != nullptr && inst->Parent() == block && inst->GetOpcode() == opcode + ? inst : nullptr; + } + + bool FullExec(Value predicate) { + predicate = predicate.Resolve(); + if (predicate == Value(true)) return true; + const auto* either = Get(predicate, ValueOpcode::LogicalOr); + if (either == nullptr) return false; + for (size_t index = 0; index < 2; index++) { + const auto* negate = Get(either->Arg(index), ValueOpcode::LogicalNot); + if (negate == nullptr || negate->Arg(0).Resolve() != either->Arg(1 - index).Resolve()) continue; + const auto original = negate->Arg(0).Resolve(); + if (saved_exec.IsEmpty()) saved_exec = original; + return saved_exec == original; + } + return false; + } + + Value Unwrap(Value value) { + while (const auto* select = Get(value, ValueOpcode::SelectU32)) { + if (!FullExec(select->Arg(0))) break; + value = select->Arg(1); + } + return value.Resolve(); + } + + static bool IsRowShift(const Inst& inst, uint32_t shift) { + const auto flags = inst.Flags(); + return flags.control == 0x110u + shift && flags.row_mask == 0xf && + flags.bank_mask == 0xf && !flags.fetch_inactive && !flags.bound_control && !flags.dpp8; + } + + Value MatchRow(Value value, uint32_t shift) { + const auto* update = Get(Unwrap(value), ValueOpcode::DppUpdateU32); + if (update == nullptr || !IsRowShift(*update, shift) || !FullExec(update->Arg(2))) return {}; + const auto* minimum = Get(update->Arg(0), ValueOpcode::UMin32); + if (minimum == nullptr) return {}; + for (size_t index = 0; index < 2; index++) { + const auto* move = Get(minimum->Arg(index), ValueOpcode::DppMoveU32); + if (move == nullptr || !IsRowShift(*move, shift) || !FullExec(move->Arg(1))) continue; + const auto previous = Unwrap(minimum->Arg(1 - index)); + if (Unwrap(move->Arg(0)) == previous && Unwrap(update->Arg(1)) == previous) return previous; + } + return {}; + } + + Block* block; + Value saved_exec; +}; + } // namespace ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size) { @@ -109,6 +217,17 @@ ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size) { for (auto* block: program.blocks) { for (auto& inst: *block) { + if (wave_size == 64u && inst.GetOpcode() == ValueOpcode::UMin32) { + const auto seed = UMinReductionMatcher(block).Match(inst); + if (!seed.IsEmpty()) { + const auto position = std::find_if(block->begin(), block->end(), + [&](const Inst& candidate) { return &candidate == &inst; }); + auto& reduction = *block->PrependNewInst(position, ValueOpcode::SubgroupUMinU32, {seed}); + inst.ReplaceUsesWith(Value(&reduction)); + stats.rewritten_reductions++; + } + continue; + } if (inst.GetOpcode() != ValueOpcode::ReadLane) { continue; } diff --git a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h index 6e6a4671c..0dcff8747 100644 --- a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h +++ b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h @@ -5,7 +5,8 @@ namespace Libs::Graphics::ShaderRecompiler::IR { struct ReadLaneStats { - uint32_t rewritten_reads = 0; + uint32_t rewritten_reads = 0; + uint32_t rewritten_reductions = 0; }; [[nodiscard]] ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size); diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index b23097c51..da004e5bc 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -514,6 +514,7 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs inst.Flags() != CFG::BranchCondition::ScalarInstruction; break; case ValueOpcode::Ballot: info.subgroup_ballot = true; break; + case ValueOpcode::SubgroupUMinU32: info.subgroup_arithmetic = true; break; case ValueOpcode::DppMoveU32: case ValueOpcode::ReadFirstLane: case ValueOpcode::ReadLane: { @@ -574,7 +575,7 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) { info.live_buffers = 0; info.has_bitwise_xor = info.uses_lds = info.uses_gds = info.uses_flattened_srt = info.uses_dispatch_threads = info.uses_swizzle = false; - info.bvh = info.subgroup_ballot = info.subgroup_barrier = info.subgroup_shuffle = + info.bvh = info.subgroup_ballot = info.subgroup_barrier = info.subgroup_shuffle = info.subgroup_arithmetic = info.subgroup_local_invocation_id = info.compute_derivatives = info.image_gather_extended = info.function_lds = info.function_scratch = info.pixel_valid_mask = info.buffer_int64_atomics = info.buffer_u8 = info.buffer_u16 = info.shared_int64_atomics = info.coherent_buffers = info.float64 = false; diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index df9b6835f..21e9265bb 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -619,6 +619,124 @@ void TestReadLaneElimination() { "dynamic-lane read was rewritten unsafely"); } +void TestCompleteUMinReduction() { + enum class Case { + Masked, Subset, Literal, LiteralMasked, LiteralMove, Wave32, + UnrelatedSeed, UnmaskedSymbolic, WrongNeutral, RowMask, BankMask, + Shift, FetchInactive, BoundControl, Dpp8, MoveExec, UpdateExec, + Previous, DppSource, CrossExec, CrossRow, CrossSelectorLow, + CrossSelectorHigh, CrossFetch, CrossBound, CrossSource, FinalExec, + Endpoint, EndpointSource, DifferentWitness, CrossBlock, WrongOperation + }; + for (const auto test : {Case::Masked, Case::Subset, Case::Literal, + Case::LiteralMasked, Case::LiteralMove, Case::Wave32, Case::UnrelatedSeed, + Case::UnmaskedSymbolic, Case::WrongNeutral, Case::RowMask, Case::BankMask, + Case::Shift, Case::FetchInactive, Case::BoundControl, Case::Dpp8, + Case::MoveExec, Case::UpdateExec, Case::Previous, Case::DppSource, + Case::CrossExec, Case::CrossRow, Case::CrossSelectorLow, + Case::CrossSelectorHigh, Case::CrossFetch, Case::CrossBound, + Case::CrossSource, Case::FinalExec, Case::Endpoint, Case::EndpointSource, + Case::DifferentWitness, Case::CrossBlock, Case::WrongOperation}) { + Fixture fixture(test == Case::CrossBlock ? 2 : 1); + const auto source = fixture.Emit(ValueOpcode::LaneId); + const auto mask = fixture.Emit(ValueOpcode::ULessThan32, {source, Value(19u)}); + const auto other_mask = fixture.Emit(ValueOpcode::IEqual32, {source, Value(7u)}); + const auto negated = fixture.Emit(ValueOpcode::LogicalNot, {mask}); + const bool literal = test == Case::Literal || test == Case::LiteralMasked; + const auto enabled = literal ? Value(true) : + fixture.Emit(ValueOpcode::LogicalOr, {negated, mask}); + auto seed_mask = test == Case::UnrelatedSeed ? other_mask : mask; + if (test == Case::Subset) + seed_mask = fixture.Emit(ValueOpcode::LogicalAnd, {other_mask, mask}); + const auto seed = test == Case::Literal || test == Case::UnmaskedSymbolic ? source : + fixture.Emit(ValueOpcode::SelectU32, + {seed_mask, source, Value(test == Case::WrongNeutral ? 0u : UINT32_MAX)}); + auto prefix = fixture.Emit(ValueOpcode::SelectU32, {enabled, seed, source}); + Value preserved; + const auto operation = test == Case::WrongOperation ? ValueOpcode::BitwiseOr32 : + ValueOpcode::UMin32; + for (const uint32_t shift : {1u, 2u, 4u, 8u}) { + DppMoveFlags flags{.control = 0x110u + shift}; + if (shift == 2) { + if (test == Case::RowMask) flags.row_mask = 7; + if (test == Case::BankMask) flags.bank_mask = 7; + if (test == Case::Shift) flags.control = 0x113; + if (test == Case::FetchInactive) flags.fetch_inactive = true; + if (test == Case::BoundControl) flags.bound_control = true; + if (test == Case::Dpp8) flags.dpp8 = true; + } + const auto move = fixture.Emit(ValueOpcode::DppMoveU32, + {test == Case::DppSource ? source : prefix, + test == Case::MoveExec ? mask : test == Case::LiteralMove ? Value(true) : enabled}); + move.Instruction()->SetFlags(flags); + const auto combined = fixture.Emit(operation, {move, prefix}); + const auto update = fixture.Emit(ValueOpcode::DppUpdateU32, + {combined, test == Case::Previous ? source : prefix, + test == Case::UpdateExec ? mask : enabled}); + update.Instruction()->SetFlags(flags); + prefix = update; + if (shift == 2) preserved = fixture.Emit(ValueOpcode::ReferenceU32, {prefix}); + } + auto cross_exec = test == Case::CrossExec ? mask : enabled; + if (test == Case::DifferentWitness) { + const auto other_not = fixture.Emit(ValueOpcode::LogicalNot, {other_mask}); + cross_exec = fixture.Emit(ValueOpcode::LogicalOr, {other_mask, other_not}); + } + const auto cross = fixture.Emit(ValueOpcode::Permlane16U32, + {test == Case::CrossSource ? source : prefix, + Value(test == Case::CrossSelectorLow ? 0xfffffff0u : UINT32_MAX), + Value(test == Case::CrossSelectorHigh ? 0xfffffff0u : UINT32_MAX), cross_exec}); + cross.Instruction()->SetFlags(PermlaneFlags{ + .x16 = test != Case::CrossRow, .fetch_inactive = test == Case::CrossFetch, + .bound_control = test != Case::CrossBound}); + const auto cross_write = fixture.Emit(ValueOpcode::SelectU32, {enabled, cross, source}); + const auto combined = fixture.Emit(operation, {prefix, cross_write}); + const auto final = fixture.Emit(ValueOpcode::SelectU32, + {test == Case::FinalExec ? mask : enabled, combined, prefix}); + const uint32_t result_block = test == Case::CrossBlock ? 1u : 0u; + const auto low = fixture.Emit(ValueOpcode::ReadLane, + {final, Value(test == Case::Endpoint ? 30u : 31u)}, 0, result_block); + const auto high = fixture.Emit(ValueOpcode::ReadLane, + {test == Case::EndpointSource ? prefix : final, Value(63u)}, 0, result_block); + const auto result = fixture.Emit(operation, {high, low}, 0, result_block); + const auto use = fixture.Emit(ValueOpcode::ReferenceU32, {result}, 0, result_block); + using Terminator = Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind; + fixture.BlockAt(result_block).terminator.kind = Terminator::Return; + if (result_block != 0) { + fixture.BlockAt().AddBranch(&fixture.BlockAt(1)); + fixture.BlockAt().terminator.kind = Terminator::Branch; + fixture.BlockAt().terminator.true_block = &fixture.BlockAt(1); + } + ValidateProgram(fixture.program, true); + const auto stats = EliminateReadLane(fixture.program, test == Case::Wave32 ? 32u : 64u); + const bool matches = test == Case::Masked || test == Case::Subset || + test == Case::Literal || test == Case::LiteralMasked || test == Case::LiteralMove; + Check(stats.rewritten_reductions == (matches ? 1u : 0u) && stats.rewritten_reads == 0, + ("UMin reduction proof failed case " + std::to_string(static_cast(test))).c_str()); + if (matches) { + const auto *reduction = use.Instruction()->Arg(0).ResolveInstruction(); + Check(reduction->GetOpcode() == ValueOpcode::SubgroupUMinU32 && + reduction->Arg(0).Resolve() == seed.Resolve(), + "UMin reduction changed the neutral-masked input"); + RemoveIdentities(fixture.program.blocks); + EliminateDeadCode(fixture.program.blocks); + uint32_t updates = 0; + for (const auto &inst : fixture.BlockAt()) { + updates += inst.GetOpcode() == ValueOpcode::DppUpdateU32; + Check(inst.GetOpcode() != ValueOpcode::ReadLane && + inst.GetOpcode() != ValueOpcode::Permlane16U32, + "UMin reduction retained unused endpoint or cross-row operations"); + } + Check(updates == 2 && preserved.Instruction()->Arg(0).Instruction()->HasUses(), + "UMin reduction destroyed an independently consumed prefix or kept its dead suffix"); + ValidateProgram(fixture.program, true); + } else { + Check(use.Instruction()->Arg(0).Resolve() == result, + "rejected UMin proof changed its original result"); + } + } +} + void TestOptimizationPipeline() { Fixture fixture; const auto sum = fixture.Emit(ValueOpcode::IAdd32, {Value(40u), Value(2u)}); @@ -756,6 +874,7 @@ int main() { TestSharedIntegerRuntimeDependencies(); TestConstantBufferBounds(); TestReadLaneElimination(); + TestCompleteUMinReduction(); TestOptimizationPipeline(); TestControlFlowValueSurvivesReadLaneFolding(); TestDeadPhiCyclesAndPlanningRoots(); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index a71ad5341..ecccb775c 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -39536,6 +39536,63 @@ TestCase ImageAtomicGlc0DoesNotReturnOldValue() { return test; } +void AppendWave64Minimum(std::vector &code) { + code.insert(code.end(), {EncodeSop1(0x04, 24, 126), // Save contributing lanes. + EncodeSop1(0x28, 106, 126)}); + AppendVop3(&code, 0x101, 13, 193, Vgpr(13), 24); // Inactive lanes contribute UINT_MAX. + for (u32 shift : {1u, 2u, 4u, 8u}) { + code.push_back(EncodeVop2(0x13, 13, 250, 13)); + code.push_back(EncodeVop2Dpp(13, 0x110u + shift)); + } + // Exact RDNA2 wave64 reduction from PS 964747887d898821, PCs 0x224..0x244. + code.insert(code.end(), {0xd778100cu, 0x0305830du, + EncodeVop2(0x13, 13, Vgpr(13), 12), EncodeSop1(0x04, 126, 106), + 0xd7600006u, 0x00013f0du, 0xd7600007u, 0x00017f0du, + EncodeSop2(0x07, 8, 6, 7), EncodeVop1(0x01, 0, 8)}); +} + +GraphicsCase GraphicsSparseWaveMinimum() { + GraphicsCase test; + test.name = "GraphicsSparseWaveMinimum"; + auto &code = test.fragment_code; + code = {EncodeVintrp(0x02, 13, 0, 0, 2)}; + AppendWave64Minimum(code); + code.insert(code.end(), {EncodeExp0(0x00, 0xf), EncodeExp1(0, 0, 0, 0)}); + AppendEnd(&code); + test.pixel_interpolator_settings = {0x00000400u}; + test.expected_pixel = {0x3e800000u, 0x3e800000u, 0x3e800000u, 0x3e800000u}; + test.opcodes = {ShaderOpcode::V_MIN_U32, ShaderOpcode::V_PERMLANEX16_B32, + ShaderOpcode::V_READLANE_B32, ShaderOpcode::S_MIN_U32}; + return test; +} + +TestCase ComputeWave64Minimum(bool masked) { + TestCase test; + test.name = masked ? "ComputeWave64MinimumMasked" : "ComputeWave64Minimum"; + auto &code = test.code; + code = {EncodeVop2(0x1a, 20, InlineU32(2), 0)}; + AppendVMovU32(&code, 13, 100); + code.push_back(EncodeVop2(0x26, 13, Vgpr(13), 0)); // 100 - lane: minimum is in the upper half. + if (masked) { + AppendSMovLiteral(&code, 126, 0xfffffffeu); + AppendSMovLiteral(&code, 127, 0x7fffffffu); + } + AppendWave64Minimum(code); + AppendBufferStoreDword(&code, 0, 20); + AppendEnd(&code); + test.initial.assign(64, 0xa5a5a5a5u); + test.expected.assign(64, masked ? 38u : 37u); + if (masked) test.expected.front() = test.expected.back() = 0xa5a5a5a5u; + test.compute_info.threads_num[0] = 64; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = 64; + test.has_compute_info = true; + test.opcodes = {ShaderOpcode::V_MIN_U32, ShaderOpcode::V_PERMLANEX16_B32, + ShaderOpcode::V_READLANE_B32, ShaderOpcode::S_MIN_U32}; + return test; +} + GraphicsCase GraphicsInterpolationExport() { using O = ShaderOpcode; @@ -40193,6 +40250,8 @@ std::vector MakeCases() { AddCase(VectorDppRowXmask); cases.push_back(VectorDppRowShare(32)); cases.push_back(VectorDppRowShare(64)); + cases.push_back(ComputeWave64Minimum(false)); + cases.push_back(ComputeWave64Minimum(true)); AddCase(VectorDppBankMaskPreservesDestination); AddCase(VectorDppBoundsControlZeroPreservesDestination); AddCase(Vop3FmacF32NegatedSourceAccumulates); @@ -40586,6 +40645,7 @@ std::vector MakeCases() { std::vector MakeGraphicsCases() { return { + GraphicsSparseWaveMinimum(), GraphicsInterpolationExport(), GraphicsPositionWExport(), GraphicsPackedHalfCentroid(), @@ -45974,6 +46034,9 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--dpp-only") == 0) { VulkanHarness vulkan; + RunGraphicsCase(&vulkan, GraphicsSparseWaveMinimum()); + RunCase(&vulkan, ComputeWave64Minimum(false)); + RunCase(&vulkan, ComputeWave64Minimum(true)); RunCase(&vulkan, VectorDpp8Captured(false)); for (const u32 wave_size : {32u, 64u}) for (const bool fi : {false, true})