diff --git a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp index 6b60ed667..f9641ede8 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp @@ -24,7 +24,8 @@ constexpr OpcodeMap SOP2_OPCODE_LIST[] = { {0x1cu, Opcode::S_XNOR_B32}, {0x1du, Opcode::S_XNOR_B64}, {0x1eu, Opcode::S_LSHL_B32}, {0x1fu, Opcode::S_LSHL_B64}, {0x20u, Opcode::S_LSHR_B32}, {0x21u, Opcode::S_LSHR_B64}, - {0x22u, Opcode::S_ASHR_I32}, {0x24u, Opcode::S_BFM_B32}, + {0x22u, Opcode::S_ASHR_I32}, {0x23u, Opcode::S_ASHR_I64}, + {0x24u, Opcode::S_BFM_B32}, {0x25u, Opcode::S_BFM_B64}, {0x26u, Opcode::S_MUL_I32}, {0x27u, Opcode::S_BFE_U32}, {0x28u, Opcode::S_BFE_I32}, {0x29u, Opcode::S_BFE_U64}, {0x2cu, Opcode::S_ABSDIFF_I32}, diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 08b0a3059..79b31d92c 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -106,6 +106,7 @@ enum class Opcode { S_LSHR_B32, S_LSHR_B64, S_ASHR_I32, + S_ASHR_I64, S_MUL_I32, S_MUL_HI_U32, S_MUL_HI_I32, diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp index 183028e66..fb6eeb48e 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -168,6 +168,21 @@ bool Translator::SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode return true; } +bool Translator::S_ASHR_I64(const Decoder::Instruction& inst) { + // Signed 64-bit sources sign-extend literals; generic B64 operands zero-extend them. + const auto source = + inst.src0.kind == Decoder::OperandKind::LiteralConstant + ? ir.ConstructU64( + ReadU32(inst.src0), + IR::U32(IR::Value((inst.src0.value & 0x80000000u) ? 0xffffffffu : 0u))) + : ReadU64(inst.src0); + const auto result = + ir.Emit(IR::ValueOpcode::ShiftRightArithmetic64, {source, ReadU32(inst.src1)}); + WriteOperand(inst.dst, result); + ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); + return true; +} + bool Translator::ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool negate_rhs, bool negate_result, bool update_scc) { const auto lhs = ReadU32(inst.src0); diff --git a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp index fa554dd7a..850734b72 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp @@ -179,6 +179,7 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) { case O::S_LSHR_B64: return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical64, IR::Type::U64, false, false, true); + case O::S_ASHR_I64: return S_ASHR_I64(inst); case O::S_ANDN2_B32: return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseAnd32, true, false, true); diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 1ee1cace4..5c54ca8b9 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -176,6 +176,7 @@ private: bool negate_result); bool SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type, bool reverse, bool mask_shift_count, bool update_scc); + bool S_ASHR_I64(const Decoder::Instruction& inst); bool ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool negate_rhs, bool negate_result, bool update_scc); bool V_AND_OR_B32(const Decoder::Instruction& inst); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index bacbf5b93..5d8b92724 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -17197,6 +17197,251 @@ TestCase Scalar64BitOps() { O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +uint64_t ReferenceAshrI64(uint64_t bits, u32 count) { + const auto shift = count & 63u; + // Complementing negative values expresses sign fill using unsigned shifts. + return (bits & (uint64_t{1} << 63u)) != 0 ? ~(~bits >> shift) : bits >> shift; +} + +TestCase ScalarAshrI64Edges(bool dynamic) { + using O = ShaderOpcode; + TestCase test; + test.name = + dynamic ? "ScalarAshrI64DynamicEdges" : "ScalarAshrI64ConstantEdges"; + constexpr uint64_t sources[] = {0, + 1, + 0x80000000ull, + 0x100000000ull, + 0x7fffffffffffffffull, + 0x8000000000000000ull, + 0xffffffffffffffffull, + 0x0123456789abcdefull, + 0xfedcba9876543210ull}; + constexpr u32 counts[] = {0, 1, 7, 31, 32, 33, 63, + 64, 65, 95, 96, 127, 0xffffffffu}; + test.initial.assign(std::begin(counts), std::end(counts)); + for (const auto source : sources) { + test.initial.push_back(static_cast(source)); + test.initial.push_back(static_cast(source >> 32u)); + } + test.expected = test.initial; + auto &code = test.code; + const auto load_scalar = [&](u32 reg, u32 index) { + AppendVMovU32(&code, 30, index * sizeof(u32)); + AppendBufferLoadDword(&code, 1, 30); + code.push_back(EncodeVop1(0x02, reg, Vgpr(1))); + }; + for (u32 source_index = 0; source_index < std::size(sources); + ++source_index) { + const auto source = sources[source_index]; + // Runtime sources exercise both constant-count and dynamic-count emission. + load_scalar(20, static_cast(std::size(counts)) + source_index * 2u); + load_scalar(21, + static_cast(std::size(counts)) + source_index * 2u + 1u); + for (u32 index = 0; index < std::size(counts); ++index) { + const auto expected = ReferenceAshrI64(source, counts[index]); + if (dynamic) { + load_scalar(22, index); + } + // Start SCC at the opposite value so every result must update it. + code.push_back( + EncodeSopc(0x06, InlineU32(0), InlineU32(expected != 0 ? 1u : 0u))); + const auto count_src = dynamic ? 22u + : counts[index] <= 64u ? InlineU32(counts[index]) + : 255u; + code.push_back(EncodeSop2(0x23, 24, 20, count_src)); + if (!dynamic && count_src == 255u) { + code.push_back(counts[index]); + } + code.push_back(EncodeSMovB32(26, 253)); + const auto output = static_cast(test.expected.size()); + AppendStoreSgprPair(&code, 24, output); + AppendStoreSgpr(&code, 26, output + 2u); + test.expected.push_back(static_cast(expected)); + test.expected.push_back(static_cast(expected >> 32u)); + test.expected.push_back(expected != 0 ? 1u : 0u); + } + } + AppendEnd(&code); + test.initial.resize(test.expected.size()); + test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, + O::S_MOV_B32, O::S_CMP_EQ_U32, + O::S_ASHR_I64, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.decoded_counts = { + {"S_ASHR_I64", std::size(sources) * std::size(counts)}}; + test.required_spirv = {"OpShiftRightArithmetic"}; + test.forbidden_spirv = {"OpTypeInt 64"}; + return test; +} + +TestCase ScalarAshrI64OperandsAndAliases() { + using O = ShaderOpcode; + TestCase test; + test.name = "ScalarAshrI64OperandsAndAliases"; + test.initial = {33}; // Runtime shift for signed literal source tests. + test.expected = test.initial; + auto &code = test.code; + AppendVMovU32(&code, 30, 0); + AppendBufferLoadDword(&code, 1, 30); + code.push_back(EncodeVop1(0x02, 22, Vgpr(1))); + size_t instructions = 0; + const auto check_result = [&](u32 dst, u32 src0, u32 src1, uint64_t value, + u32 count, std::optional literal = {}) { + code.push_back(EncodeSop2(0x23, dst, src0, src1)); + if (literal) { + code.push_back(*literal); + } + code.push_back(EncodeSMovB32(26, 253)); + const auto output = static_cast(test.expected.size()); + AppendStoreSgprPair(&code, dst, output); + AppendStoreSgpr(&code, 26, output + 2u); + const auto expected = ReferenceAshrI64(value, count); + test.expected.push_back(static_cast(expected)); + test.expected.push_back(static_cast(expected >> 32u)); + test.expected.push_back(expected != 0 ? 1u : 0u); + ++instructions; + }; + for (const u32 literal : {0u, 0x7fffffffu, 0x80000000u, 0xffffffffu}) { + const uint64_t value = + literal | ((literal & 0x80000000u) != 0 ? 0xffffffff00000000ull : 0ull); + for (const u32 count : {0u, 1u, 31u, 32u, 63u, 64u}) { + check_result(24, 255, InlineU32(count), value, count, literal); + } + check_result(24, 255, 22, value, 33, literal); + } + for (const u32 source : {128u, 129u, 192u, 193u, 208u, 125u}) { + const uint64_t value = source == 125u ? 0ull + : source <= 192u ? source - 128u + : uint64_t{192} - source; + for (const u32 count : {0u, 1u, 32u, 63u}) { + check_result(24, source, InlineU32(count), value, count); + } + } + for (const u32 scc : {0u, 1u}) { + // SCC may be either source and must be read before it is overwritten. + code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1))); + check_result(24, 253, InlineU32(1), scc, 1); + AppendSMovLiteral(&code, 20, 1); + AppendSMovLiteral(&code, 21, 0); + code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1))); + check_result(24, 20, 253, 1, scc); + } + constexpr uint64_t value = 0xfedcba9889abcdefull; + for (const u32 count : {0u, 1u, 31u, 32u, 33u, 63u, 65u}) { + AppendSMovLiteral(&code, 20, static_cast(value)); + AppendSMovLiteral(&code, 21, static_cast(value >> 32u)); + AppendSMovLiteral(&code, 23, count); + check_result(20, 20, 23, value, count); // In-place source/destination pair. + for (const u32 count_reg : {24u, 25u}) { + AppendSMovLiteral(&code, 20, static_cast(value)); + AppendSMovLiteral(&code, 21, static_cast(value >> 32u)); + AppendSMovLiteral(&code, count_reg, count); + check_result(24, 20, count_reg, value, + count); // Count aliases destination. + } + } + for (const u32 count_reg : {20u, 21u}) { + const uint64_t alias_value = + count_reg == 20u ? 0x8000000000000021ull : 0x0000002100000001ull; + AppendSMovLiteral(&code, 20, static_cast(alias_value)); + AppendSMovLiteral(&code, 21, static_cast(alias_value >> 32u)); + check_result(20, 20, count_reg, alias_value, 33); + } + // Both operands may refer to the single literal word in the instruction. + check_result(24, 255, 255, 0xffffffffffffffffull, 0xffffffffu, 0xffffffffu); + for (const u32 source : {0u, 1u}) { + // Discarding the destination must still update SCC and leave M0 untouched. + AppendSMovLiteral(&code, 124, 0x5a13abcdu); + AppendSMovLiteral(&code, 20, 0); + AppendSMovLiteral(&code, 21, source); + code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(source))); + code.push_back(EncodeSop2(0x23, 125, 20, InlineU32(0))); + code.push_back(EncodeSMovB32(26, 253)); + const auto output = static_cast(test.expected.size()); + AppendStoreSgpr(&code, 26, output); + AppendStoreSgpr(&code, 124, output + 1u); + AppendStoreSgprPair(&code, 20, output + 2u); + test.expected.insert(test.expected.end(), {source, 0x5a13abcdu, 0, source}); + ++instructions; + } + AppendEnd(&code); + test.initial.resize(test.expected.size()); + test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, + O::S_MOV_B32, O::S_CMP_EQ_U32, + O::S_ASHR_I64, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.decoded_counts = {{"S_ASHR_I64", instructions}}; + test.forbidden_spirv = {"OpTypeInt 64"}; + return test; +} + +TestCase ScalarAshrI64Masks(u32 wave_size) { + using O = ShaderOpcode; + TestCase test; + test.name = + wave_size == 64 ? "ScalarAshrI64MasksWave64" : "ScalarAshrI64MasksWave32"; + auto &code = test.code; + struct Case { + uint64_t source; + u32 count; + }; + constexpr Case cases[] = { + {0x8000000100000001ull, 1}, {0x8000000100000001ull, 32}, + {0x8000000100000001ull, 63}, {0x0000000100000000ull, 0}, + {0x0000000100000000ull, 32}, {1, 1}}; + for (const u32 mask_reg : {106u, 126u}) { + for (const auto &item : cases) { + const auto result = ReferenceAshrI64(item.source, item.count); + AppendVMovU32(&code, 1, 1); + AppendVMovU32(&code, 2, 0); + AppendSMovLiteral(&code, mask_reg, static_cast(item.source)); + AppendSMovLiteral(&code, mask_reg + 1u, + static_cast(item.source >> 32u)); + code.push_back( + EncodeSopc(0x06, InlineU32(0), InlineU32(result != 0 ? 1u : 0u))); + code.push_back( + EncodeSop2(0x23, mask_reg, mask_reg, InlineU32(item.count))); + code.push_back(EncodeSMovB32(20, mask_reg)); + code.push_back(EncodeSMovB32(21, mask_reg + 1u)); + code.push_back(EncodeSMovB32(22, 253)); + if (mask_reg == 106u) { + code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1)); + } else { + code.push_back(EncodeVop1(0x01, 2, InlineU32(1))); + } + code.push_back(EncodeSop1(0x04, 126, 193)); // Restore EXEC for readback. + const u32 expected[] = {static_cast(result), + static_cast(result >> 32u), + result != 0 ? 1u : 0u}; + for (u32 i = 0; i < std::size(expected); ++i) { + AppendStoreSgprAtLaneDwordOffset( + &code, 20 + i, 0, static_cast(test.expected.size())); + test.expected.insert(test.expected.end(), wave_size, expected[i]); + } + AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, + static_cast(test.expected.size())); + for (u32 lane = 0; lane < wave_size; ++lane) { + test.expected.push_back(static_cast((result >> lane) & 1u)); + } + } + } + AppendEnd(&code); + test.initial.resize(test.expected.size()); + test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32, + O::S_ASHR_I64, O::V_MOV_B32, O::V_CNDMASK_B32, + O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; + test.decoded_counts = {{"S_ASHR_I64 vcc_lo, vcc_lo", std::size(cases)}, + {"S_ASHR_I64 exec_lo, exec_lo", std::size(cases)}}; + test.forbidden_spirv = {"OpTypeInt 64"}; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.has_compute_info = true; + return test; +} TestCase ScalarConditionalMoveB64() { using O = ShaderOpcode; TestCase test; @@ -26411,6 +26656,11 @@ std::vector MakeCases() { AddCase(ScalarBfeI32CapturedRawSignExtends); AddCase(BitfieldExtractWidthPastEndEdges); AddCase(Scalar64BitOps); + cases.push_back(ScalarAshrI64Edges(false)); + cases.push_back(ScalarAshrI64Edges(true)); + AddCase(ScalarAshrI64OperandsAndAliases); + cases.push_back(ScalarAshrI64Masks(32)); + cases.push_back(ScalarAshrI64Masks(64)); AddCase(VectorDynamicU64ShiftEdges); AddCase(ScalarConditionalMoveB64); AddCase(ScalarConditionalMoveB64PreservesMasks); @@ -31174,6 +31424,15 @@ int main(int argc, char **argv) { std::setvbuf(stdout, nullptr, _IONBF, 0); EnsureConfigInitialized(); CheckLeastRecentlyUsedCacheOrdering(); + if (argc == 2 && std::strcmp(argv[1], "--s-ashr-i64-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, ScalarAshrI64Edges(false)); + RunCase(&vulkan, ScalarAshrI64Edges(true)); + RunCase(&vulkan, ScalarAshrI64OperandsAndAliases()); + RunCase(&vulkan, ScalarAshrI64Masks(32)); + RunCase(&vulkan, ScalarAshrI64Masks(64)); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--tessellation-only") == 0) { CheckTessellationPrograms(); CheckEmbeddedFetchVertexOffset(); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index ad6cd1b73..2495669f5 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -4130,6 +4130,56 @@ void CheckNewDecoderUnsupported(const uint32_t *shader, uint32_t words, #endif } +void TestScalarAshrI64Decoder() { + using namespace ShaderRecompiler::Decoder; + + const uint32_t shader[] = { + 0x91860204u, // s_ashr_i64 s[6:7], s[4:5], s2 + EncodeSop2(0x23, 8, 255, 129), 0xfffffffeu, + EncodeSop2(0x23, 10, 8, 255), 0xffffffffu, + EncodeSop2(0x23, 12, 255, 255), 0xffffffc0u, + EncodeSopp(0x01), + }; + Program program; + ShaderRecompiler::Decoder::DecodeProgram(shader, program); + Check(program.instructions.size() == 5u, + "S_ASHR_I64 decoder lost instruction boundaries around literals"); + const auto ® = program.instructions[0]; + Check(reg.family == Family::SOP2 && reg.opcode == Opcode::S_ASHR_I64 && + reg.opcode_id == 0x23u && reg.pc == 0u && reg.word_count == 1u && + reg.src_count == 2u && reg.dst.kind == OperandKind::Sgpr && + reg.dst.reg == 6u && reg.src0.kind == OperandKind::Sgpr && + reg.src0.reg == 4u && reg.src1.kind == OperandKind::Sgpr && + reg.src1.reg == 2u, + "S_ASHR_I64 register encoding was decoded incorrectly"); + const auto &source_literal = program.instructions[1]; + const auto &count_literal = program.instructions[2]; + const auto &shared_literal = program.instructions[3]; + for (const auto *inst : {&source_literal, &count_literal, &shared_literal}) { + Check(inst->opcode == Opcode::S_ASHR_I64 && inst->word_count == 2u && + inst->src_count == 2u, + "S_ASHR_I64 literal encoding was decoded incorrectly"); + } + Check(source_literal.pc == 4u && + source_literal.src0.kind == OperandKind::LiteralConstant && + source_literal.src0.value == 0xfffffffeu && + source_literal.src1.kind == OperandKind::IntegerInlineConstant && + source_literal.src1.value == 1u && count_literal.pc == 12u && + count_literal.src1.kind == OperandKind::LiteralConstant && + count_literal.src1.value == 0xffffffffu && + shared_literal.pc == 20u && + shared_literal.src0.kind == OperandKind::LiteralConstant && + shared_literal.src1.kind == OperandKind::LiteralConstant && + shared_literal.src0.value == 0xffffffc0u && + shared_literal.src1.value == 0xffffffc0u && + program.instructions[4].pc == 28u && + program.instructions[4].opcode == Opcode::S_ENDPGM, + "S_ASHR_I64 decoder mishandled source, count, or shared literals"); + Check(Common::ContainsStr(ProgramToString(program), + "S_ASHR_I64 s6, s4, s2"), + "S_ASHR_I64 is missing from the decoded dump"); +} + void TestNewShaderDecoderArchitecture() { using namespace ShaderRecompiler::Decoder; @@ -13493,6 +13543,7 @@ int main() { // Opcode semantics and optimized SPIR-V are exercised by // ShaderRecompilerComputeTests; keep the distinct decoder contract checks // here. + TestScalarAshrI64Decoder(); TestNewShaderDecoderArchitecture(); TestImageAddressOperands(); TestSopkCompareImmediateExtension();