diff --git a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp index 2b476bca9..31acabd86 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp @@ -11,15 +11,13 @@ bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) { : TranslateComposedInteger(inst); } -bool Translator::TranslateInstruction(const IR::Instruction& inst, - const ScalarMemorySourceValues* scalar_source_snapshot) { +bool Translator::TranslateInstruction(const IR::Instruction& inst) { switch (IR::GetOpcodeInfo(inst.op).lowering_class) { case IR::LoweringClass::Control: return TranslateControlOperation(inst); case IR::LoweringClass::Move: return TranslateMove(inst); case IR::LoweringClass::Lane: return TranslateLaneOperation(inst); case IR::LoweringClass::State: return TranslateStateOperation(inst); - case IR::LoweringClass::Memory: - return TranslateMemoryOperation(inst, scalar_source_snapshot); + case IR::LoweringClass::Memory: return TranslateMemoryOperation(inst); case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst); case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst); case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst); diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index dedefbb41..f90b5e8b0 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -181,19 +181,7 @@ IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode) { } } -bool Translator::IsScalarMemoryLoadOperation(IR::Opcode opcode) { - return opcode == IR::Opcode::SLoadDword || opcode == IR::Opcode::SBufferLoadDword; -} - -ScalarMemorySourceValues Translator::ReadScalarMemorySource(const IR::Instruction& inst) { - const auto resource = inst.op == IR::Opcode::SLoadDword - ? GetScalarAddressResource(inst.memory.resource) - : GetBufferResource(inst.memory); - return {resource, ReadU32(inst.src[0])}; -} - -bool Translator::TranslateScalarMemory(const IR::Instruction& inst, - const ScalarMemorySourceValues* source_snapshot) { +bool Translator::TranslateScalarMemory(const IR::Instruction& inst) { switch (inst.op) { case IR::Opcode::LoadSrtDword: { const auto resource = ir.Emit(IR::ValueOpcode::GetSrtResource); @@ -201,20 +189,30 @@ bool Translator::TranslateScalarMemory(const IR::Instruction& inst, ir.Emit(IR::ValueOpcode::ReadConst, {resource, ReadU32(inst.src[0])})); return true; } - case IR::Opcode::SBufferLoadDword: { - const auto source = - source_snapshot != nullptr ? *source_snapshot : ReadScalarMemorySource(inst); - WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ReadConstBuffer, - {source.resource, source.offset}, AddMemoryInfo(inst))); - return true; - } + case IR::Opcode::SBufferLoadDword: case IR::Opcode::SLoadDword: { - const auto source = - source_snapshot != nullptr ? *source_snapshot : ReadScalarMemorySource(inst); - WriteOperand(inst.dst, - ir.Emit(IR::ValueOpcode::LoadAddressU32, - {source.resource, source.offset, IR::Value(0u), IR::Value(true)}, - AddMemoryInfo(inst))); + const auto resource = inst.op == IR::Opcode::SLoadDword + ? GetScalarAddressResource(inst.memory.resource) + : GetBufferResource(inst.memory); + const auto offset = ReadU32(inst.src[0]); + std::array loaded {}; + for (uint32_t component = 0; component < inst.memory.data_dwords; component++) { + auto scalar = inst; + scalar.memory.offset += component * sizeof(uint32_t); + scalar.memory.data_dwords = 1u; + scalar.memory.component_index = component; + if (inst.op == IR::Opcode::SBufferLoadDword) { + loaded[component] = ir.Emit(IR::ValueOpcode::ReadConstBuffer, + {resource, offset}, AddMemoryInfo(scalar)); + } else { + loaded[component] = ir.Emit( + IR::ValueOpcode::LoadAddressU32, + {resource, offset, IR::Value(0u), IR::Value(true)}, AddMemoryInfo(scalar)); + } + } + for (uint32_t component = 0; component < inst.memory.data_dwords; component++) { + WriteOperand(ScalarDestinationOperand(inst.dst, component), loaded[component]); + } return true; } default: return false; @@ -681,13 +679,12 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) { } } -bool Translator::TranslateMemoryOperation(const IR::Instruction& inst, - const ScalarMemorySourceValues* scalar_source_snapshot) { +bool Translator::TranslateMemoryOperation(const IR::Instruction& inst) { switch (inst.op) { case IR::Opcode::LoadSrtDword: case IR::Opcode::SLoadDword: case IR::Opcode::SBufferLoadDword: - return TranslateScalarMemory(inst, scalar_source_snapshot); + return TranslateScalarMemory(inst); case IR::Opcode::BufferLoadUbyte: case IR::Opcode::BufferLoadSbyte: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 6c0510b81..783be909b 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -37,6 +37,29 @@ IR::Operand Translator::OffsetOperand(const IR::Operand& operand, uint32_t offse return result; } +IR::Operand Translator::ScalarDestinationOperand(const IR::Operand& operand, uint32_t offset) { + EXIT_IF(operand.kind != IR::OperandKind::Register); + uint32_t code = 0; + switch (operand.reg.file) { + case IR::RegisterFile::Scalar: code = operand.reg.index; break; + case IR::RegisterFile::Vcc: code = 106u + operand.reg.index; break; + default: EXIT("invalid scalar-memory destination"); + } + code += offset; + IR::Operand result; + result.kind = IR::OperandKind::Register; + if (code < 106u) { + result.reg = {IR::RegisterFile::Scalar, code}; + } else { + switch (code) { + case 106u: result.reg = {IR::RegisterFile::Vcc, 0u}; break; + case 107u: result.reg = {IR::RegisterFile::Vcc, 1u}; break; + default: EXIT("scalar-memory destination crosses an invalid register"); + } + } + return result; +} + IR::Operand Translator::PlainOperand(const IR::Operand& operand) { auto result = operand; result.sdwa_sel = 6; @@ -653,28 +676,13 @@ void Translator::WriteCompareResult(const IR::Operand& operand, IR::U1 value) { } bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error) { - ScalarMemorySourceValues scalar_source_snapshot {}; - uint32_t scalar_source_pc = 0; - bool has_scalar_source_snapshot = false; for (const auto& source_inst: source.instructions) { current_opcode = source_inst.op; current_pc = source_inst.pc; - const bool grouped_scalar_load = - IsScalarMemoryLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u; - if (grouped_scalar_load && - (!has_scalar_source_snapshot || scalar_source_pc != source_inst.pc || - source_inst.memory.component_index == 0u)) { - scalar_source_snapshot = ReadScalarMemorySource(source_inst); - scalar_source_pc = source_inst.pc; - has_scalar_source_snapshot = true; - } else if (!grouped_scalar_load) { - has_scalar_source_snapshot = false; - } if (static_cast(source_inst.op) >= static_cast(IR::Opcode::Count)) { return Fail(error, "value IR input opcode is out of range"); } - if (TranslateInstruction(source_inst, - grouped_scalar_load ? &scalar_source_snapshot : nullptr)) { + if (TranslateInstruction(source_inst)) { continue; } return Fail(error, fmt::format("opcode {} at 0x{:08x} has no typed Value IR lowering", diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 5cb43361e..ba62cb608 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -7,11 +7,6 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { -struct ScalarMemorySourceValues { - IR::Value resource; - IR::U32 offset; -}; - class Translator { public: Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit, @@ -25,6 +20,7 @@ public: private: IR::Operand OffsetOperand(const IR::Operand& operand, uint32_t offset); + IR::Operand ScalarDestinationOperand(const IR::Operand& operand, uint32_t offset); IR::Operand PlainOperand(const IR::Operand& operand); IR::U32 ReadPcRelativeU32(const IR::Operand& operand); std::array BallotMask(IR::U1 value); @@ -85,10 +81,7 @@ private: IR::Value NarrowSubdword(IR::U32 value, uint32_t bits); IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode); IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode); - bool IsScalarMemoryLoadOperation(IR::Opcode opcode); - ScalarMemorySourceValues ReadScalarMemorySource(const IR::Instruction& inst); - bool TranslateScalarMemory(const IR::Instruction& inst, - const ScalarMemorySourceValues* source_snapshot); + bool TranslateScalarMemory(const IR::Instruction& inst); bool TranslateBufferLoad(const IR::Instruction& inst); bool TranslateBufferStore(const IR::Instruction& inst); bool TranslateAtomicMemory(const IR::Instruction& inst); @@ -105,15 +98,13 @@ private: uint32_t upper_result); IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high); - bool TranslateInstruction(const IR::Instruction& inst, - const ScalarMemorySourceValues* scalar_source_snapshot); + bool TranslateInstruction(const IR::Instruction& inst); bool TranslateStateOperation(const IR::Instruction& inst); bool TranslateControlOperation(const IR::Instruction& inst); bool TranslateMove(const IR::Instruction& inst); bool TranslateLaneOperation(const IR::Instruction& inst); bool TranslateAttributeOperation(const IR::Instruction& inst); - bool TranslateMemoryOperation(const IR::Instruction& inst, - const ScalarMemorySourceValues* scalar_source_snapshot); + bool TranslateMemoryOperation(const IR::Instruction& inst); bool TranslateIntegerCompare(const IR::Instruction& inst); bool TranslateInteger16Compare(const IR::Instruction& inst); bool TranslateFloatCompare(const IR::Instruction& inst); diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.cpp b/src/graphics/shader/recompiler/ir/ShaderIR.cpp index 6c15fce5c..abe1c1ba8 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIR.cpp @@ -273,7 +273,7 @@ void ClearRegisterOffsetModifiers(Decoder::Operand& operand) { operand.dpp = false; } -bool TryGetScalarDestinationCode(const Decoder::Operand& operand, uint32_t& code) { +bool TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code) { switch (operand.kind) { case Decoder::OperandKind::Sgpr: code = operand.reg; return true; case Decoder::OperandKind::VccLo: code = SCALAR_DST_VCC_LO; return true; @@ -290,7 +290,7 @@ bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index, // aliases such as VCC and EXEC sit beside ordinary SGPRs. Re-decode after adding the // dword index so s_buffer_load_dwordx2 vcc_lo writes vcc_lo, then vcc_hi. uint32_t code = 0; - return TryGetScalarDestinationCode(operand, code) && + return TryGetEncodedScalarCode(operand, code) && Decoder::DecodeScalarDestination(code + index, 0, decoded, nullptr); } diff --git a/src/graphics/shader/recompiler/ir/ShaderIRInternal.h b/src/graphics/shader/recompiler/ir/ShaderIRInternal.h index 9229ec0d5..0db895f0c 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIRInternal.h +++ b/src/graphics/shader/recompiler/ir/ShaderIRInternal.h @@ -30,7 +30,7 @@ uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand); MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind); uint32_t RawScalarLoadBase(const Decoder::Operand& operand); void ClearRegisterOffsetModifiers(Decoder::Operand& operand); -bool TryGetScalarDestinationCode(const Decoder::Operand& operand, uint32_t& code); +bool TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code); bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index, Decoder::Operand& result); Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index); diff --git a/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp b/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp index 852675c1e..3981bc192 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp @@ -77,24 +77,40 @@ Opcode LoweredImageOpcode(Decoder::Opcode opcode) { bool LowerScalarMemoryLoadDword(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op, std::string* error) { - for (uint32_t i = 0; i < decoded.data_dwords; i++) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = op; - inst.src_count = 1; - const auto kind = - op == Opcode::SLoadDword ? ResourceKind::ScalarAddress : ResourceKind::ScalarBuffer; - inst.memory = OffsetMemoryInfo(decoded, kind, i); - // Raw s_load uses a two-SGPR pointer base; s_buffer_load uses a four-SGPR - // descriptor resource index. - inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0) - : ResourceIndexFromOperand(decoded.src0); - if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) || - !LowerSourceOperand(decoded.src1, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); + uint32_t destination_code = 0; + const auto alignment = decoded.data_dwords == 1u ? 1u + : decoded.data_dwords == 2u ? 2u + : 4u; + if (!TryGetEncodedScalarCode(decoded.dst, destination_code) || + destination_code % alignment != 0u || + destination_code + decoded.data_dwords > 108u) { + SetError(error, "scalar-memory destination has an invalid aligned SGPR span"); + return false; } + uint32_t base_code = 0; + const auto base_alignment = op == Opcode::SLoadDword ? 2u : 4u; + const auto base_width = op == Opcode::SLoadDword ? 2u : 4u; + if (!TryGetEncodedScalarCode(decoded.src0, base_code) || + base_code % base_alignment != 0u || base_code + base_width > 108u) { + SetError(error, "scalar-memory base has an invalid aligned SGPR span"); + return false; + } + Instruction inst; + inst.pc = decoded.pc; + inst.op = op; + inst.src_count = 1; + const auto kind = + op == Opcode::SLoadDword ? ResourceKind::ScalarAddress : ResourceKind::ScalarBuffer; + inst.memory = MemoryInfoFromDecoded(decoded, kind); + // Raw s_load uses a two-SGPR pointer base; s_buffer_load uses a four-SGPR + // descriptor resource index. + inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0) + : ResourceIndexFromOperand(decoded.src0); + if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || + !LowerSourceOperand(decoded.src1, inst.src[0], error)) { + return false; + } + block.instructions.push_back(inst); return true; } diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.cpp b/src/graphics/shader/recompiler/ir/ValueProgram.cpp index 2f7ec7d65..d30b7e719 100644 --- a/src/graphics/shader/recompiler/ir/ValueProgram.cpp +++ b/src/graphics/shader/recompiler/ir/ValueProgram.cpp @@ -311,6 +311,33 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st ValueOpcodeName(inst.GetOpcode()), inst.NumArgs(), NumArgsOf(inst.GetOpcode()))); } + if (IsRuntimeRead(inst.GetOpcode())) { + const auto memory_index = inst.Flags().index; + if (memory_index >= program.memory_info.size()) { + return Fail(error, fmt::format("{} has an invalid memory-info index", + ValueOpcodeName(inst.GetOpcode()))); + } + const auto& memory = program.memory_info[memory_index]; + const bool scalar_buffer = inst.GetOpcode() == ValueOpcode::ReadConstBuffer; + const bool scalar_address = memory.kind == ResourceKind::ScalarAddress; + if ((scalar_buffer && memory.kind != ResourceKind::ScalarBuffer) || + (!scalar_buffer && !scalar_address && memory.kind != ResourceKind::Flat && + memory.kind != ResourceKind::Global && memory.kind != ResourceKind::Scratch)) { + return Fail(error, fmt::format("{} has an invalid scalar-memory resource kind", + ValueOpcodeName(inst.GetOpcode()))); + } + const bool scalar_memory = scalar_buffer || scalar_address; + const bool valid_group_width = + scalar_memory ? memory.component_count == 1u || memory.component_count == 2u || + memory.component_count == 4u || memory.component_count == 8u || + memory.component_count == 16u + : memory.component_count >= 1u && memory.component_count <= 4u; + if (memory.data_bits != 32u || memory.data_dwords != 1u || + !valid_group_width || memory.component_index >= memory.component_count) { + return Fail(error, fmt::format("{} has inconsistent scalar-memory metadata", + ValueOpcodeName(inst.GetOpcode()))); + } + } const auto buffer_components = BufferComponentCount(inst.GetOpcode()); if (buffer_components != 0u) { const auto memory_index = inst.Flags().index; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index add8843e3..09eec211b 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -14601,12 +14601,12 @@ TestCase ScalarMemoryLoadVariants() { }; const Load s_loads[] = { - {0x00, 20, 0, 1}, {0x01, 21, 4, 2}, {0x02, 23, 12, 4}, - {0x03, 27, 28, 8}, {0x04, 40, 60, 16}, + {0x00, 20, 0, 1}, {0x01, 22, 4, 2}, {0x02, 24, 12, 4}, + {0x03, 28, 28, 8}, {0x04, 36, 60, 16}, }; const Load s_buffer_loads[] = { - {0x08, 56, 0, 1}, {0x09, 57, 4, 2}, {0x0a, 59, 12, 4}, - {0x0b, 63, 28, 8}, {0x0c, 71, 60, 16}, + {0x08, 52, 0, 1}, {0x09, 54, 4, 2}, {0x0a, 56, 12, 4}, + {0x0b, 60, 28, 8}, {0x0c, 68, 60, 16}, }; std::vector code; diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 6186cf6c0..eb4102df4 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -5497,121 +5497,77 @@ void TestGraphicsCreateInterpolantMapping() { "interpolant identity tail was not filled"); } -void TestNewShaderRecompilerWideMemoryLowering() { +void TestNewShaderRecompilerNativeWideScalarMemoryIr() { const uint32_t shader[] = { - EncodeSmem0(0x02, 4, 4), - 0u, // s_load_dwordx4 s[4:7] - EncodeSmem0(0x0a, 8, 4), - 0u, // s_buffer_load_dwordx4 s[8:11] - EncodeSmem0(0x09, 106, 4), - 0u, // s_buffer_load_dwordx2 vcc_lo - EncodeMubuf0(0x0d, 16), - EncodeMubuf1(20, 0, 1), // buffer_load_dwordx2 v[20:21] - EncodeMubuf0(0x0f, 32), - EncodeMubuf1(24, 0, 1), // buffer_load_dwordx3 v[24:26] - EncodeMubuf0(0x0e, 48), - EncodeMubuf1(28, 0, 1), // buffer_load_dwordx4 v[28:31] - EncodeMubuf0(0x1d, 64), - EncodeMubuf1(32, 0, 1), // buffer_store_dwordx2 v[32:33] - EncodeMubuf0(0x1f, 80), - EncodeMubuf1(36, 0, 1), // buffer_store_dwordx3 v[36:38] - EncodeMubuf0(0x1e, 96), - EncodeMubuf1(40, 0, 1), // buffer_store_dwordx4 v[40:43] - EncodeMubuf0(0x08, 2), - EncodeMubuf1(44, 0, 1), // buffer_load_ubyte v44 - EncodeMubuf0(0x0a, 4), - EncodeMubuf1(45, 0, 1), // buffer_load_ushort v45 - EncodeFlat0(0x0c, 0, 4), - EncodeFlat1(50, 0x7d, 0, 1), // flat_load_dword - EncodeFlat0(0x0d, 1, 8), - EncodeFlat1(52, 0x7d, 0, 1), // scratch_load_dwordx2 - EncodeFlat0(0x0e, 2, 12), - EncodeFlat1(56, 0x7d, 0, 1), // global_load_dwordx4 - 0xbf810000u, + EncodeSmem0(0x00, 0, 4), 125u << 25u, + EncodeSmem0(0x01, 2, 4), 125u << 25u, + EncodeSmem0(0x02, 4, 4), 125u << 25u, + EncodeSmem0(0x03, 8, 4), 125u << 25u, + EncodeSmem0(0x04, 92, 4), 125u << 25u, + EncodeSmem0(0x08, 32, 4), 125u << 25u, + EncodeSmem0(0x09, 106, 4), 125u << 25u, + EncodeSmem0(0x0a, 36, 4), 125u << 25u, + EncodeSmem0(0x0b, 40, 4), 125u << 25u, + EncodeSmem0(0x0c, 48, 4), 125u << 25u, + EncodeSopp(0x01), }; - - auto options = MakeCompileOptions(ShaderType::Compute); - options.dump_ir = true; - options.flat_memory_base = 0; - - ShaderRecompiler::CompileResult result; + ShaderRecompiler::Decoder::Program decoded; + ShaderRecompiler::CFG::Graph graph; + ShaderRecompiler::IR::Program ir; std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + Check(ShaderRecompiler::Decoder::DecodeProgram(shader, decoded, &error) && + ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) && + ShaderRecompiler::IR::LowerProgram(decoded, graph, + ShaderType::Compute, 64, ir, + &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_load_dwordx4"), - "new decoder did not decode SMEM x4 load"); - Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dwordx4"), - "new decoder did not decode scalar-buffer x4 load"); - Check( - Common::ContainsStr(result.decoded_dump, "s_buffer_load_dwordx2 vcc_lo"), - "new decoder did not decode scalar-buffer x2 load into VCC"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx2"), - "new decoder did not decode buffer x2 load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx3"), - "new decoder did not decode buffer x3 load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx4"), - "new decoder did not decode buffer x4 load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_store_dwordx4"), - "new decoder did not decode buffer x4 store"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_ubyte"), - "new decoder did not decode buffer ubyte load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_ushort"), - "new decoder did not decode buffer ushort load"); - Check(Common::ContainsStr(result.decoded_dump, "flat_load_dword"), - "new decoder did not decode flat dword load"); - Check(Common::ContainsStr(result.decoded_dump, "segment=1"), - "new decoder did not preserve scratch segment metadata"); - Check(Common::ContainsStr(result.decoded_dump, "segment=2"), - "new decoder did not preserve global segment metadata"); - Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), - "new decoder did not expose width metadata for wide memory ops"); - Check(Common::ContainsStr(result.ir_dump, "SLoadDword s4"), - "s_load_dwordx4 did not expand to first scalar dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "SLoadDword s7"), - "s_load_dwordx4 did not expand to last scalar dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s8"), - "s_buffer_load_dwordx4 did not expand to first scalar-buffer IR load"); - Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s11"), - "s_buffer_load_dwordx4 did not expand to last scalar-buffer IR load"); - Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_lo"), - "s_buffer_load_dwordx2 did not expand to VCC low dword"); - Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_hi"), - "s_buffer_load_dwordx2 did not expand to VCC high dword"); - Check( - CountSourceOccurrences(result.ir_dump, "BufferLoadDword v20") == 1u && - CountSourceOccurrences(result.ir_dump, "BufferLoadDword v24") == 1u && - CountSourceOccurrences(result.ir_dump, "BufferLoadDword v28") == 1u && - !Common::ContainsStr(result.ir_dump, "BufferLoadDword v31"), - "wide buffer loads were not preserved as one native IR operation"); - Check(CountSourceOccurrences(result.ir_dump, "BufferStoreDword null, v32") == - 1u && - CountSourceOccurrences(result.ir_dump, - "BufferStoreDword null, v36") == 1u && - CountSourceOccurrences(result.ir_dump, - "BufferStoreDword null, v40") == 1u && - !Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v43"), - "wide buffer stores were not preserved as one native IR operation"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadUbyte v44"), - "buffer_load_ubyte did not lower to sub-dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadUshort v45"), - "buffer_load_ushort did not lower to sub-dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "FlatLoadDword v50"), - "flat_load_dword did not lower to flat dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "scratch"), - "scratch segment load did not lower with scratch metadata"); - Check(Common::ContainsStr(result.ir_dump, "global"), - "global segment load did not lower with global metadata"); - Check(SpirvContainsOpcode(result.spirv, 65), - "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 62), - "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 194), - "SPIR-V binary does not contain OpShiftRightLogical"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical"); - Check(SpirvContainsOpcode(result.spirv, 199), - "SPIR-V binary does not contain OpBitwiseAnd"); - CheckSpirvBinaryValidates(result.spirv); + std::vector address_widths; + std::vector buffer_widths; + bool buffer_x2_targets_vcc = false; + for (const auto &block : ir.blocks) { + for (const auto &inst : block.instructions) { + if (inst.op == ShaderRecompiler::IR::Opcode::SLoadDword) { + address_widths.push_back(inst.memory.data_dwords); + } else if (inst.op == + ShaderRecompiler::IR::Opcode::SBufferLoadDword) { + buffer_widths.push_back(inst.memory.data_dwords); + if (inst.memory.data_dwords == 2u) { + buffer_x2_targets_vcc = + inst.dst.reg.file == ShaderRecompiler::IR::RegisterFile::Vcc && + inst.dst.reg.index == 0u; + } + } + } + } + const std::vector expected{1u, 2u, 4u, 8u, 16u}; + Check(address_widths == expected && buffer_widths == expected, + "SMEM x1/x2/x4/x8/x16 did not remain one width-bearing ShaderIR operation each"); + Check(buffer_x2_targets_vcc, + "s_buffer_load_dwordx2 did not preserve its VCC destination span"); + + const auto reject = [&](uint32_t opcode, uint32_t dst, uint32_t sbase, + const char *reason) { + const uint32_t invalid[] = {EncodeSmem0(opcode, dst, sbase), + 125u << 25u, EncodeSopp(0x01)}; + ShaderRecompiler::Decoder::Program invalid_decoded; + ShaderRecompiler::CFG::Graph invalid_graph; + ShaderRecompiler::IR::Program invalid_ir; + std::string invalid_error; + Check(ShaderRecompiler::Decoder::DecodeProgram(invalid, invalid_decoded, + &invalid_error) && + ShaderRecompiler::CFG::BuildGraph(invalid_decoded, invalid_graph, + &invalid_error) && + !ShaderRecompiler::IR::LowerProgram( + invalid_decoded, invalid_graph, ShaderType::Compute, 64, + invalid_ir, &invalid_error) && + Common::ContainsStr(invalid_error, reason), + "invalid SMEM register span was not rejected"); + }; + reject(0x01, 1, 4, "destination"); + reject(0x02, 2, 4, "destination"); + reject(0x02, 124, 4, "destination"); + reject(0x00, 0, 63, "base"); + reject(0x08, 0, 1, "base"); } void TestNewShaderRecompilerNativeWideBufferIr() { @@ -8619,6 +8575,22 @@ void TestNativeWideValueValidation() { ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(0u), Value(true)}, flags); }; + const auto append_scalar_read = [](ValueProgram &program, + ValueOpcode opcode, + MemoryFlags flags) { + IREmitter ir(program.blocks.front()); + if (opcode == ValueOpcode::ReadConstBuffer) { + const auto resource = ir.Emit( + ValueOpcode::GetBufferResource, + {Value(0u), Value(0u), Value(0u), Value(0u)}); + ir.Emit(opcode, {resource, Value(0u)}, flags); + } else { + const auto resource = + ir.Emit(ValueOpcode::GetAddressResource, {Value(0u), Value(0u)}); + ir.Emit(opcode, + {resource, Value(0u), Value(0u), Value(true)}, flags); + } + }; const auto check_rejected = [](const ValueProgram &program, const char *expected) { std::string error; @@ -8627,6 +8599,58 @@ void TestNativeWideValueValidation() { "malformed native-wide value operation was not rejected"); }; + { + auto program = make_program(); + append_scalar_read(program, ValueOpcode::ReadConstBuffer, + MemoryFlags{.index = 1}); + check_rejected(program, "invalid memory-info index"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::ScalarAddress; + program.memory_info.push_back(memory); + append_scalar_read(program, ValueOpcode::ReadConstBuffer, {}); + check_rejected(program, "invalid scalar-memory resource kind"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::ScalarAddress; + memory.data_bits = 16; + program.memory_info.push_back(memory); + append_scalar_read(program, ValueOpcode::LoadAddressU32, {}); + check_rejected(program, "inconsistent scalar-memory metadata"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::ScalarBuffer; + memory.data_dwords = 2; + program.memory_info.push_back(memory); + append_scalar_read(program, ValueOpcode::ReadConstBuffer, {}); + check_rejected(program, "inconsistent scalar-memory metadata"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::ScalarBuffer; + memory.component_count = 3; + program.memory_info.push_back(memory); + append_scalar_read(program, ValueOpcode::ReadConstBuffer, {}); + check_rejected(program, "inconsistent scalar-memory metadata"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::ScalarAddress; + memory.component_count = 4; + memory.component_index = 4; + program.memory_info.push_back(memory); + append_scalar_read(program, ValueOpcode::LoadAddressU32, {}); + check_rejected(program, "inconsistent scalar-memory metadata"); + } + { auto program = make_program(); MemoryInfo memory; @@ -9890,7 +9914,7 @@ void TestSrtWalkerRealSBufferLowering() { "real S_BUFFER_LOAD walk accepted a negative immediate"); } -void TestScalarMemoryLoadsSnapshotOverlappingOperands() { +void TestScalarMemorySourcesCapturedBeforeWrites() { const auto CheckOverlap = [](uint32_t opcode, bool overlap_offset) { const uint32_t base_field = overlap_offset ? 4u : 0u; const uint32_t soffset = overlap_offset ? 0u : 125u; @@ -9932,7 +9956,7 @@ void TestScalarMemoryLoadsSnapshotOverlappingOperands() { error.c_str()); Check(flat.size() == table.size() && std::equal(flat.begin(), flat.end(), table.begin()), - "overlapping scalar-memory load did not snapshot its sources"); + "overlapping scalar-memory load did not capture its sources before writes"); CheckFlattenedReadSlots( ir, 4, "overlapping scalar-memory patch used the wrong flat offsets"); }; @@ -9976,6 +10000,39 @@ void TestScalarMemoryLoadCrossesIntoVcc() { } } +void TestScalarMemoryUnusedTailDce() { + const uint32_t shader[] = { + EncodeSmem0(0x0c, 16, 0), + 4u << 25u, // s_buffer_load_dwordx16 s[16:31], s[0:3], s4 + EncodeVop1(0x01, 0, 16), // v_mov_b32 v0, s16 + EncodeMubuf0(0x1c), + EncodeMubuf1(0, 2, 1), // keep only the first loaded component live + EncodeSopp(0x01), + }; + ShaderRecompiler::IR::Program ir; + std::string error; + Check(LowerTypedPlanning(shader, static_cast(std::size(shader)), + ir, &error), + error.c_str()); + uint32_t reads = 0; + for (const auto *block : ir.values->blocks) { + for (const auto &inst : *block) { + if (inst.GetOpcode() != + ShaderRecompiler::IR::ValueOpcode::ReadConstBuffer) { + continue; + } + reads++; + const auto flags = inst.Flags(); + Check(flags.index < ir.values->memory_info.size() && + ir.values->memory_info[flags.index].component_index == 0u && + ir.values->memory_info[flags.index].component_count == 16u, + "live x16 SMEM component lost its native-width metadata"); + } + } + Check(reads == 1u, + "grouped x16 SMEM prevented dead component reads from being eliminated"); +} + void TestResourceTrackingRealDensePatching() { const uint32_t shader[] = { EncodeMubuf0(0x0c), @@ -10735,6 +10792,7 @@ int main() { TestDemandDrivenSpirvDeclarations(); TestNativeSubgroupPolicy(); TestNewShaderRecompilerSMovB32(); + TestNewShaderRecompilerNativeWideScalarMemoryIr(); TestNewShaderRecompilerNativeWideBufferIr(); TestNewShaderRecompilerMubufFormatLowering(); TestNewShaderRecompilerTypedBufferLowering(); @@ -10809,8 +10867,9 @@ int main() { TestSrtWalkerRealSmemLowering(); TestSrtWalkerVccBaseLowering(); TestSrtWalkerRealSBufferLowering(); - TestScalarMemoryLoadsSnapshotOverlappingOperands(); + TestScalarMemorySourcesCapturedBeforeWrites(); TestScalarMemoryLoadCrossesIntoVcc(); + TestScalarMemoryUnusedTailDce(); TestResourceTrackingRealDensePatching(); TestLowerProgramResetsAnalysisState(); TestNewShaderRecompilerNativeBindingPlan();