diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 2027b3e97..ce7824a6d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -114,6 +114,18 @@ uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t c return EmitInputComponentU32(state, kind, component); } +// Whether EXEC enables a source lane within the caller's 32-lane half. +uint32_t EmitSourceLaneActive(ValueEmitContext& ctx, uint32_t exec, uint32_t lane) { + auto& state = ctx.state; + const auto scope = ConstantU32(state, spv::ScopeSubgroup); + const auto ballot = EmitNative(state, scope, exec); + if (ctx.half != 0) { + lane = EmitAndConstant(state, lane, 31); + } + return EmitNative(state, scope, ballot, + lane); +} + uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags, uint32_t exec) { auto& state = ctx.state; @@ -852,14 +864,11 @@ uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) { ConstantU32(state, 15)); state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, row_value, index); const auto shuffled = ctx.Shuffle(inst, 0, target); - uint32_t result = shuffled; - if (!flags.fetch_inactive) { - const auto source_exec = ctx.Shuffle(inst, 3, target); - result = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpSelect, TypeU32(state), result, source_exec, shuffled, - ConstantU32(state, 0)); + if (flags.fetch_inactive) { + return shuffled; } - return result; + return Select(state, TypeU32(state), EmitSourceLaneActive(ctx, ctx.Arg(inst, 3), target), + shuffled, ctx.Arg(inst, 4)); } uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index 7d06c7279..993630d25 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -465,9 +465,13 @@ void Translator::V_PERMLANE16_B32(const Decoder::Instruction& inst, bool x16) { .fetch_inactive = inst.dst.op_sel, .bound_control = inst.dst.op_sel_hi, }; - const auto result = - ir.Emit(IR::ValueOpcode::Permlane16U32, - {ReadU32(inst.src0), ReadU32(inst.src1), ReadU32(inst.src2), ir.GetExec()}, flags); + // An EXEC-disabled source lane without FI disables the write unless BOUND_CTRL reads zero. + const auto inactive = flags.fetch_inactive || flags.bound_control + ? IR::U32(IR::Value(0u)) + : ir.GetVectorReg(static_cast(inst.dst.reg)); + const auto result = ir.Emit( + IR::ValueOpcode::Permlane16U32, + {ReadU32(inst.src0), ReadU32(inst.src1), ReadU32(inst.src2), ir.GetExec(), inactive}, flags); auto dst = DestinationOperand(inst); dst.op_sel = false; dst.op_sel_hi = false; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 1b0dcb9b2..cfe27a3ce 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -223,7 +223,7 @@ VALUE_OPCODE(ReadFirstLane, U32, U32, U1) VALUE_OPCODE(ReadLane, U32, U32, U32) VALUE_OPCODE(SubgroupUMinU32, U32, U32) VALUE_OPCODE(WriteLane, U32, U32, U32, U32) -VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1) +VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1, U32) VALUE_OPCODE(PermuteU32, U32, U32, U32, U1) VALUE_OPCODE(BpermuteU32, U32, U32, U32, U1) diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index 33dd6457f..7e589753a 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -694,7 +694,8 @@ void TestCompleteUMinReduction() { const auto cross = fixture.Emit(ValueOpcode::Permlane16U32, {test == Case::CrossSource ? source : prefix, Value(test == Case::CrossSelectorLow ? 0xfffffff0u : UINT32_MAX), - Value(test == Case::CrossSelectorHigh ? 0xfffffff0u : UINT32_MAX), cross_exec}); + Value(test == Case::CrossSelectorHigh ? 0xfffffff0u : UINT32_MAX), cross_exec, + Value(0u)}); cross.Instruction()->SetFlags(PermlaneFlags{ .x16 = test != Case::CrossRow, .fetch_inactive = test == Case::CrossFetch, .bound_control = test != Case::CrossBound}); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 492f6dbff..dccaa3796 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -28017,56 +28017,52 @@ TestCase VectorPermlanex16() { return test; } -TestCase VectorPermlane16FetchInactiveZero() { +TestCase VectorPermlane16InactiveSource(u32 op_sel, u32 wave_size) { using O = ShaderOpcode; - - std::vector code = { - EncodeVop1(0x01, 1, 0), - EncodeVop2(0x25, 1, InlineU32(10), 1), - EncodeVopc(0xc2, InlineU32(0), 0), - EncodeSop1(0x04, 126, 106), - EncodeSMovB32(0, InlineU32(1)), - EncodeSMovB32(1, InlineU32(0)), - }; - AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1); - AppendStoreVgpr(&code, 2, 0); + constexpr u32 sentinel = 0xaaaaaaaau; + // Each lane reads its pair neighbour; the halves differ, so each needs its own EXEC. + // OP_SEL is 0 (keep the destination), 1 (FI) or 2 (BOUND_CTRL). + const u32 masks[] = {0x5f5ff5f5u, 0xf5f55f5fu}; + std::vector code; + AppendVMovU32(&code, 1, 100); + code.push_back(EncodeVop2(0x25, 1, Vgpr(0), 1)); + AppendVMovLiteral(&code, 2, sentinel); + AppendSMovLiteral(&code, 0, 0x67452301u); + AppendSMovLiteral(&code, 1, 0xefcdab89u); + AppendSMovLiteral(&code, 126, masks[0]); + if (wave_size == 64) AppendSMovLiteral(&code, 127, masks[1]); + AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1, 0, op_sel); + AppendSMovLiteral(&code, 126, 0xffffffffu); + if (wave_size == 64) AppendSMovLiteral(&code, 127, 0xffffffffu); + AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 0); AppendEnd(&code); TestCase test; - test.name = "VectorPermlane16FetchInactiveZero"; - test.code = code; - test.expected = {0}; - test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32, - O::S_MOV_B64, O::S_MOV_B32, O::V_PERMLANE16_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; -} - -TestCase VectorPermlane16FetchInactiveFi() { - using O = ShaderOpcode; - - std::vector code = { - EncodeVop1(0x01, 1, Vgpr(0)), EncodeVop2(0x25, 1, InlineU32(10), 1), - EncodeVopc(0xc2, InlineU32(0), 0), EncodeSop1(0x04, 126, 106), - EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)), - }; - AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1, 0, 1); - AppendStoreVgpr(&code, 2, 0); - AppendEnd(&code); - - TestCase test; - test.name = "VectorPermlane16FetchInactiveFi"; - test.code = code; - test.expected = {11}; - test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32, - O::S_MOV_B64, O::S_MOV_B32, O::V_PERMLANE16_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.compute_info.threads_num[0] = 4; + const char *names[3][2] = { + {"VectorPermlane16InactiveSourceKeepsWave32", + "VectorPermlane16InactiveSourceKeepsWave64"}, + {"VectorPermlane16InactiveSourceFiWave32", + "VectorPermlane16InactiveSourceFiWave64"}, + {"VectorPermlane16InactiveSourceZeroWave32", + "VectorPermlane16InactiveSourceZeroWave64"}}; + test.name = names[op_sel][wave_size == 64]; + test.code = std::move(code); + test.expected.resize(wave_size, sentinel); + for (u32 lane = 0; lane < wave_size; ++lane) { + const auto mask = masks[lane / 32u]; + if ((mask & (1u << (lane % 32u))) == 0) continue; + const u32 source = lane ^ 1u; + if (op_sel == 1 || (mask & (1u << (source % 32u))) != 0) { + test.expected[lane] = 100u + source; + } else if (op_sel == 2) { + test.expected[lane] = 0; + } + } + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, + O::V_PERMLANE16_B32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; + test.compute_info.wave_size = wave_size; + test.compute_info.threads_num[0] = wave_size; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; test.compute_info.thread_ids_num = 1; @@ -44163,8 +44159,9 @@ std::vector MakeCases() { AddCase(VectorLaneWave32RuntimeSelectorWraps); AddCase(VectorReadlaneSelectsTwoKeysWithinWave); AddCase(VectorPermlanex16); - AddCase(VectorPermlane16FetchInactiveZero); - AddCase(VectorPermlane16FetchInactiveFi); + for (const u32 op_sel : {0u, 1u, 2u}) + for (const u32 wave_size : {32u, 64u}) + cases.push_back(VectorPermlane16InactiveSource(op_sel, wave_size)); cases.push_back(VectorDpp8Captured(false)); for (const u32 wave_size : {32u, 64u}) for (const bool fi : {false, true}) @@ -50618,6 +50615,9 @@ int main(int argc, char **argv) { RunCase(&vulkan, VectorReadFirstLaneExecMasks(64)); RunCase(&vulkan, VectorDppBankMaskPreservesDestination()); RunCase(&vulkan, VectorDppBoundsControlZeroPreservesDestination()); + for (const u32 op_sel : {0u, 1u, 2u}) + for (const u32 wave_size : {32u, 64u}) + RunCase(&vulkan, VectorPermlane16InactiveSource(op_sel, wave_size)); return 0; } if (argc == 2 && std::strcmp(argv[1], "--cmpx-o-f32-only") == 0) {