diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index d8ab138f2..79091601b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -266,7 +266,8 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { requirements.subgroup_local_invocation_id = true; break; } - case IR::ValueOpcode::SwizzleU32: { + case IR::ValueOpcode::SwizzleU32: + case IR::ValueOpcode::BpermuteU32: { MarkBallot(); requirements.subgroup_shuffle = true; requirements.subgroup_local_invocation_id = true; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 6085c03b6..8ee262650 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -30,6 +30,19 @@ uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) { return Binary(state, OpLogicalAnd, TypeBool(state), lhs, rhs); } +uint32_t EmitDsMaskedLaneRead(EmitterState& state, uint32_t source, uint32_t target, + uint32_t exec) { + const auto shuffled = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled, + ConstantU32(state, ScopeSubgroup), source, target}); + const auto source_exec = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformShuffle, TypeBool(state), source_exec, + ConstantU32(state, ScopeSubgroup), exec, target}); + const auto source_active = + AndCondition(state, source_exec, EmitSubgroupLaneActiveBool(state, target)); + return Select(state, TypeU32(state), source_active, shuffled, ConstantU32(state, 0)); +} + uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, uint32_t index, uint32_t offset, uint32_t soffset) { auto& state = ctx.state; @@ -1181,24 +1194,27 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Define(inst, AppendConsume(ctx, inst, op == IR::ValueOpcode::DataAppend)); return true; } - if (op == IR::ValueOpcode::SwizzleU32) { - ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0)}); - const auto source = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction( - {OpLoad, TypeU32(ctx.state), source, ctx.scratch_u32_variable}); - const auto target = - EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state), - inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0); - const auto shuffled = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled, - ConstantU32(state, ScopeSubgroup), source, target}); - const auto source_exec = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformShuffle, TypeBool(state), source_exec, - ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 2), target}); - const auto source_active = - AndCondition(state, source_exec, EmitSubgroupLaneActiveBool(state, target)); - ctx.Define(inst, - Select(state, TypeU32(state), source_active, shuffled, ConstantU32(state, 0))); + if (op == IR::ValueOpcode::SwizzleU32 || op == IR::ValueOpcode::BpermuteU32) { + uint32_t source = ctx.Arg(inst, 0); + uint32_t target = 0; + if (op == IR::ValueOpcode::SwizzleU32) { + state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, source}); + source = state.builder.AllocateId(); + state.builder.AddFunction({OpLoad, TypeU32(state), source, ctx.scratch_u32_variable}); + target = EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state), + inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0); + } else { + const auto index = Binary( + state, OpBitwiseAnd, TypeU32(state), + Binary(state, OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1), + ConstantU32(state, 2)), + ConstantU32(state, 31)); + const auto base = + Binary(state, OpBitwiseAnd, TypeU32(state), EmitSubgroupLocalInvocationId(state), + ConstantU32(state, ~31u)); + target = Binary(state, OpBitwiseOr, TypeU32(state), base, index); + } + ctx.Define(inst, EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2))); return true; } return false; diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index d25a94db5..263d851c3 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -107,6 +107,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = { {0xa6u, Opcode::DS_READ_U16_D16, 1, 16}, {0xa7u, Opcode::DS_READ_U16_D16_HI, 1, 16}, {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32}, + {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32}, {0xdeu, Opcode::DS_WRITE_B96, 3, 32}, {0xdfu, Opcode::DS_WRITE_B128, 4, 32}, {0xfeu, Opcode::DS_READ_B96, 3, 32}, {0xffu, Opcode::DS_READ_B128, 4, 32}, }; @@ -195,6 +196,7 @@ uint32_t DsSourceCount(Opcode opcode) { case Opcode::DS_WRITE2ST64_B64: case Opcode::DS_MIN_F32: case Opcode::DS_MAX_F32: return 3u; + case Opcode::DS_BPERMUTE_B32: return 2u; case Opcode::DS_READ_ADDTID_B32: case Opcode::DS_CONSUME: case Opcode::DS_APPEND: return 0u; @@ -395,9 +397,10 @@ void DecodeDs(uint32_t pc, std::span code, uint32_t word_index, SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented"); } if (inst.gds && - (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_WRITE_ADDTID_B32 || + (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_BPERMUTE_B32 || + inst.opcode == Opcode::DS_WRITE_ADDTID_B32 || inst.opcode == Opcode::DS_READ_ADDTID_B32)) { - SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS"); + SetUnsupported(inst, Family::DS, opcode, "DS lane operation is available only for LDS"); } if (inst.opcode == Opcode::DS_WRITE_ADDTID_B32 && data1 != 0u) { SetUnsupported(inst, Family::DS, opcode, diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index bd816435e..b3e996068 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -661,6 +661,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::DS_MIN_F32: case Opcode::DS_MAX_F32: case Opcode::DS_SWIZZLE_B32: + case Opcode::DS_BPERMUTE_B32: case Opcode::DS_READ_I8: case Opcode::DS_READ_U8: case Opcode::DS_READ_I16: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index b4635ee65..89a82a9d6 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -492,6 +492,7 @@ enum class Opcode { DS_MIN_F32, DS_MAX_F32, DS_SWIZZLE_B32, + DS_BPERMUTE_B32, DS_CONSUME, DS_APPEND, DS_READ_I8, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index f41e4b0bf..0af641c86 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -878,6 +878,13 @@ bool Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) { return true; } +bool Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) { + const auto address = ir.IAdd(ReadU32(inst.src0), IR::U32(IR::Value(inst.offset))); + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::BpermuteU32, + {ReadU32(inst.src1), address, ir.GetExec()})); + return true; +} + bool Translator::EmitMemory(const Decoder::Instruction& inst, std::string* error) { switch (inst.opcode) { case Decoder::Opcode::S_LOAD_DWORD: @@ -1036,6 +1043,7 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst, std::string* error case Decoder::Opcode::DS_MAX_F32: return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMax32); case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst); + case Decoder::Opcode::DS_BPERMUTE_B32: return DS_BPERMUTE_B32(inst); case Decoder::Opcode::DS_CONSUME: return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataConsume); case Decoder::Opcode::DS_APPEND: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index e59360c57..3f3a3bb5f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -109,6 +109,7 @@ private: bool DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode); bool DS_ADDTID(const Decoder::Instruction& inst, bool write); bool DS_SWIZZLE_B32(const Decoder::Instruction& inst); + bool DS_BPERMUTE_B32(const Decoder::Instruction& inst); IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value); IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper, diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 1c9fa3829..4e540cabc 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -188,6 +188,7 @@ VALUE_OPCODE(ReadFirstLane, U32, U32, U1) VALUE_OPCODE(ReadLane, U32, U32, U32) VALUE_OPCODE(WriteLane, U32, U32, U32, U32) VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1) +VALUE_OPCODE(BpermuteU32, U32, U32, U32, U1) // Typed shader resources and memory operations. Descriptor state is represented by SSA values; // dense bindings are assigned by resource tracking. diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index f5172058a..8176320ac 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -12029,6 +12029,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, case Opcode::DS_XOR_RTN_B32: case Opcode::DS_WRXCHG_RTN_B32: case Opcode::DS_SWIZZLE_B32: + case Opcode::DS_BPERMUTE_B32: case Opcode::DS_READ_I8: case Opcode::DS_READ_U8: case Opcode::DS_READ_I16: @@ -18558,6 +18559,89 @@ TestCase DsSwizzleInvalidSourceLaneZero() { return test; } +TestCase DsBpermuteCapturedExecOffsetAndWrap() { + using O = ShaderOpcode; + + std::vector code; + code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0)); + AppendBufferLoadDword(&code, 17, 30); + AppendVMovLiteral(&code, 1, 0xdeadbeefu); + AppendVMovU32(&code, 3, 100); + code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3)); + code.push_back(EncodeSop1(0x04, 4, 126)); + code.push_back(EncodeVop2(0x1b, 18, InlineU32(1), 0)); + code.push_back(EncodeVopc(0xc2, InlineU32(1), 18)); + code.push_back(EncodeSop1(0x04, 126, 106)); + code.push_back(0xdacc0000u); + code.push_back(0x01000311u); // ds_bpermute_b32 v1, v17, v3 + code.push_back(EncodeSop1(0x04, 126, 4)); + AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 4); + + AppendVMovU32(&code, 17, 124); + code.push_back(EncodeDs0(0xb3, 4)); + code.push_back(EncodeDs1(2, 3, 17)); + AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 8); + AppendEnd(&code); + + TestCase test; + test.name = "DsBpermuteCapturedExecOffsetAndWrap"; + test.code = code; + test.initial = {0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0}; + test.expected = {0, 0, 12, 4, + 0xdeadbeefu, 0, 0xdeadbeefu, 101, + 100, 100, 100, 100}; + test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD, + O::V_MOV_B32, O::V_ADD_NC_U32, + O::V_AND_B32, O::V_CMP_EQ_U32, + O::S_MOV_B64, + O::DS_BPERMUTE_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; + test.required_spirv = {"OpGroupNonUniformShuffle", + "OpGroupNonUniformBallot"}; + test.forbidden_spirv = {" Workgroup", "OpControlBarrier", + "OpMemoryBarrier"}; + test.decoded_counts = {{"DS_BPERMUTE_B32", 2}}; + test.ir_counts = {{"BpermuteU32", 2}}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.wave_size = 32; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + +TestCase DsBpermuteWave64UsesIndependentHalves() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 3, 100); + code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3)); + AppendVMovU32(&code, 17, 128); + code.push_back(0xdacc0000u); + code.push_back(0x01000311u); // ds_bpermute_b32 v1, v17, v3 + AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0); + AppendEnd(&code); + + TestCase test; + test.name = "DsBpermuteWave64UsesIndependentHalves"; + test.code = code; + test.initial = std::vector(64, 0); + test.expected = std::vector(32, 100); + test.expected.insert(test.expected.end(), 32, 132); + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, + O::DS_BPERMUTE_B32, O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpGroupNonUniformShuffle"}; + test.compute_info.threads_num[0] = 64; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.wave_size = 64; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + TestCase BufferAtomicVariants() { using O = ShaderOpcode; @@ -20746,6 +20830,8 @@ std::vector MakeCases() { AddCase(DsMiscVariants); AddCase(DsFloatMinMaxUsesSeparateCompareOperand); AddCase(DsSwizzleInvalidSourceLaneZero); + AddCase(DsBpermuteCapturedExecOffsetAndWrap); + AddCase(DsBpermuteWave64UsesIndependentHalves); AddCase(BufferAtomicVariants); AddCase(BufferAtomicGlc0DoesNotReturnOldValue); AddCase(BufferAtomicFMinExactRawGlcModes); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 62aa00173..c72f521ac 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -4123,6 +4123,18 @@ void TestNewShaderDecoderArchitecture() { Check(boot.opcode == Opcode::DS_SWIZZLE_B32 && boot.offset == 0xc480u, "DS decoder rejected a captured boot-shader instruction"); + const uint32_t bpermute_ds[] = {0xdacc0000u, 0x01000311u}; + Instruction bpermute; + Check(DecodeInstruction(bpermute_ds, 0u, bpermute, &error), error.c_str()); + Check(bpermute.family == Family::DS && + bpermute.opcode == Opcode::DS_BPERMUTE_B32 && + bpermute.word_count == 2u && bpermute.src_count == 2u && + bpermute.data_dwords == 1u && bpermute.data_bits == 32u && + bpermute.offset == 0u && !bpermute.gds && + bpermute.dst.reg == 1u && bpermute.src0.reg == 17u && + bpermute.src1.reg == 3u, + "DS decoder rejected or misdecoded captured DS_BPERMUTE_B32"); + const uint32_t d16_hi_ds[] = {0xda9c0000u, 0x05000006u}; Instruction d16_hi; Check(DecodeInstruction(d16_hi_ds, 0u, d16_hi, &error), error.c_str()); @@ -12083,6 +12095,12 @@ void TestNewShaderRecompilerUnsupportedMemoryDecode() { CheckNewDecoderUnsupported(mtbuf_unknown, static_cast(std::size(mtbuf_unknown)), "MTBUF", "opcode=0x08"); + + const uint32_t bpermute_gds[] = { + EncodeDs0(0xb3) | (1u << 17u), EncodeDs1(1, 3, 17), 0xbf810000u}; + CheckNewDecoderUnsupported(bpermute_gds, + static_cast(std::size(bpermute_gds)), + "DS", "available only for LDS"); } void TestNewShaderRecompilerFlatUserPointerUsesDma() {