mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-20 02:47:02 +02:00
refactor(shader): preserve native scalar memory operations
This commit is contained in:
nmzik
@@ -11,15 +11,13 @@ bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
|
||||
: TranslateComposedInteger(inst);
|
||||
}
|
||||
|
||||
bool Translator::TranslateInstruction(const IR::Instruction& inst,
|
||||
const ScalarMemorySourceValues* scalar_source_snapshot) {
|
||||
bool Translator::TranslateInstruction(const IR::Instruction& inst) {
|
||||
switch (IR::GetOpcodeInfo(inst.op).lowering_class) {
|
||||
case IR::LoweringClass::Control: return TranslateControlOperation(inst);
|
||||
case IR::LoweringClass::Move: return TranslateMove(inst);
|
||||
case IR::LoweringClass::Lane: return TranslateLaneOperation(inst);
|
||||
case IR::LoweringClass::State: return TranslateStateOperation(inst);
|
||||
case IR::LoweringClass::Memory:
|
||||
return TranslateMemoryOperation(inst, scalar_source_snapshot);
|
||||
case IR::LoweringClass::Memory: return TranslateMemoryOperation(inst);
|
||||
case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst);
|
||||
case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst);
|
||||
case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst);
|
||||
|
||||
@@ -181,19 +181,7 @@ IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode) {
|
||||
}
|
||||
}
|
||||
|
||||
bool Translator::IsScalarMemoryLoadOperation(IR::Opcode opcode) {
|
||||
return opcode == IR::Opcode::SLoadDword || opcode == IR::Opcode::SBufferLoadDword;
|
||||
}
|
||||
|
||||
ScalarMemorySourceValues Translator::ReadScalarMemorySource(const IR::Instruction& inst) {
|
||||
const auto resource = inst.op == IR::Opcode::SLoadDword
|
||||
? GetScalarAddressResource(inst.memory.resource)
|
||||
: GetBufferResource(inst.memory);
|
||||
return {resource, ReadU32(inst.src[0])};
|
||||
}
|
||||
|
||||
bool Translator::TranslateScalarMemory(const IR::Instruction& inst,
|
||||
const ScalarMemorySourceValues* source_snapshot) {
|
||||
bool Translator::TranslateScalarMemory(const IR::Instruction& inst) {
|
||||
switch (inst.op) {
|
||||
case IR::Opcode::LoadSrtDword: {
|
||||
const auto resource = ir.Emit(IR::ValueOpcode::GetSrtResource);
|
||||
@@ -201,20 +189,30 @@ bool Translator::TranslateScalarMemory(const IR::Instruction& inst,
|
||||
ir.Emit(IR::ValueOpcode::ReadConst, {resource, ReadU32(inst.src[0])}));
|
||||
return true;
|
||||
}
|
||||
case IR::Opcode::SBufferLoadDword: {
|
||||
const auto source =
|
||||
source_snapshot != nullptr ? *source_snapshot : ReadScalarMemorySource(inst);
|
||||
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ReadConstBuffer,
|
||||
{source.resource, source.offset}, AddMemoryInfo(inst)));
|
||||
return true;
|
||||
}
|
||||
case IR::Opcode::SBufferLoadDword:
|
||||
case IR::Opcode::SLoadDword: {
|
||||
const auto source =
|
||||
source_snapshot != nullptr ? *source_snapshot : ReadScalarMemorySource(inst);
|
||||
WriteOperand(inst.dst,
|
||||
ir.Emit(IR::ValueOpcode::LoadAddressU32,
|
||||
{source.resource, source.offset, IR::Value(0u), IR::Value(true)},
|
||||
AddMemoryInfo(inst)));
|
||||
const auto resource = inst.op == IR::Opcode::SLoadDword
|
||||
? GetScalarAddressResource(inst.memory.resource)
|
||||
: GetBufferResource(inst.memory);
|
||||
const auto offset = ReadU32(inst.src[0]);
|
||||
std::array<IR::Value, 16> loaded {};
|
||||
for (uint32_t component = 0; component < inst.memory.data_dwords; component++) {
|
||||
auto scalar = inst;
|
||||
scalar.memory.offset += component * sizeof(uint32_t);
|
||||
scalar.memory.data_dwords = 1u;
|
||||
scalar.memory.component_index = component;
|
||||
if (inst.op == IR::Opcode::SBufferLoadDword) {
|
||||
loaded[component] = ir.Emit(IR::ValueOpcode::ReadConstBuffer,
|
||||
{resource, offset}, AddMemoryInfo(scalar));
|
||||
} else {
|
||||
loaded[component] = ir.Emit(
|
||||
IR::ValueOpcode::LoadAddressU32,
|
||||
{resource, offset, IR::Value(0u), IR::Value(true)}, AddMemoryInfo(scalar));
|
||||
}
|
||||
}
|
||||
for (uint32_t component = 0; component < inst.memory.data_dwords; component++) {
|
||||
WriteOperand(ScalarDestinationOperand(inst.dst, component), loaded[component]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
default: return false;
|
||||
@@ -681,13 +679,12 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
|
||||
}
|
||||
}
|
||||
|
||||
bool Translator::TranslateMemoryOperation(const IR::Instruction& inst,
|
||||
const ScalarMemorySourceValues* scalar_source_snapshot) {
|
||||
bool Translator::TranslateMemoryOperation(const IR::Instruction& inst) {
|
||||
switch (inst.op) {
|
||||
case IR::Opcode::LoadSrtDword:
|
||||
case IR::Opcode::SLoadDword:
|
||||
case IR::Opcode::SBufferLoadDword:
|
||||
return TranslateScalarMemory(inst, scalar_source_snapshot);
|
||||
return TranslateScalarMemory(inst);
|
||||
|
||||
case IR::Opcode::BufferLoadUbyte:
|
||||
case IR::Opcode::BufferLoadSbyte:
|
||||
|
||||
@@ -37,6 +37,29 @@ IR::Operand Translator::OffsetOperand(const IR::Operand& operand, uint32_t offse
|
||||
return result;
|
||||
}
|
||||
|
||||
IR::Operand Translator::ScalarDestinationOperand(const IR::Operand& operand, uint32_t offset) {
|
||||
EXIT_IF(operand.kind != IR::OperandKind::Register);
|
||||
uint32_t code = 0;
|
||||
switch (operand.reg.file) {
|
||||
case IR::RegisterFile::Scalar: code = operand.reg.index; break;
|
||||
case IR::RegisterFile::Vcc: code = 106u + operand.reg.index; break;
|
||||
default: EXIT("invalid scalar-memory destination");
|
||||
}
|
||||
code += offset;
|
||||
IR::Operand result;
|
||||
result.kind = IR::OperandKind::Register;
|
||||
if (code < 106u) {
|
||||
result.reg = {IR::RegisterFile::Scalar, code};
|
||||
} else {
|
||||
switch (code) {
|
||||
case 106u: result.reg = {IR::RegisterFile::Vcc, 0u}; break;
|
||||
case 107u: result.reg = {IR::RegisterFile::Vcc, 1u}; break;
|
||||
default: EXIT("scalar-memory destination crosses an invalid register");
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
IR::Operand Translator::PlainOperand(const IR::Operand& operand) {
|
||||
auto result = operand;
|
||||
result.sdwa_sel = 6;
|
||||
@@ -653,28 +676,13 @@ void Translator::WriteCompareResult(const IR::Operand& operand, IR::U1 value) {
|
||||
}
|
||||
|
||||
bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error) {
|
||||
ScalarMemorySourceValues scalar_source_snapshot {};
|
||||
uint32_t scalar_source_pc = 0;
|
||||
bool has_scalar_source_snapshot = false;
|
||||
for (const auto& source_inst: source.instructions) {
|
||||
current_opcode = source_inst.op;
|
||||
current_pc = source_inst.pc;
|
||||
const bool grouped_scalar_load =
|
||||
IsScalarMemoryLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u;
|
||||
if (grouped_scalar_load &&
|
||||
(!has_scalar_source_snapshot || scalar_source_pc != source_inst.pc ||
|
||||
source_inst.memory.component_index == 0u)) {
|
||||
scalar_source_snapshot = ReadScalarMemorySource(source_inst);
|
||||
scalar_source_pc = source_inst.pc;
|
||||
has_scalar_source_snapshot = true;
|
||||
} else if (!grouped_scalar_load) {
|
||||
has_scalar_source_snapshot = false;
|
||||
}
|
||||
if (static_cast<size_t>(source_inst.op) >= static_cast<size_t>(IR::Opcode::Count)) {
|
||||
return Fail(error, "value IR input opcode is out of range");
|
||||
}
|
||||
if (TranslateInstruction(source_inst,
|
||||
grouped_scalar_load ? &scalar_source_snapshot : nullptr)) {
|
||||
if (TranslateInstruction(source_inst)) {
|
||||
continue;
|
||||
}
|
||||
return Fail(error, fmt::format("opcode {} at 0x{:08x} has no typed Value IR lowering",
|
||||
|
||||
@@ -7,11 +7,6 @@
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
|
||||
|
||||
struct ScalarMemorySourceValues {
|
||||
IR::Value resource;
|
||||
IR::U32 offset;
|
||||
};
|
||||
|
||||
class Translator {
|
||||
public:
|
||||
Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit,
|
||||
@@ -25,6 +20,7 @@ public:
|
||||
|
||||
private:
|
||||
IR::Operand OffsetOperand(const IR::Operand& operand, uint32_t offset);
|
||||
IR::Operand ScalarDestinationOperand(const IR::Operand& operand, uint32_t offset);
|
||||
IR::Operand PlainOperand(const IR::Operand& operand);
|
||||
IR::U32 ReadPcRelativeU32(const IR::Operand& operand);
|
||||
std::array<IR::U32, 2> BallotMask(IR::U1 value);
|
||||
@@ -85,10 +81,7 @@ private:
|
||||
IR::Value NarrowSubdword(IR::U32 value, uint32_t bits);
|
||||
IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode);
|
||||
IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode);
|
||||
bool IsScalarMemoryLoadOperation(IR::Opcode opcode);
|
||||
ScalarMemorySourceValues ReadScalarMemorySource(const IR::Instruction& inst);
|
||||
bool TranslateScalarMemory(const IR::Instruction& inst,
|
||||
const ScalarMemorySourceValues* source_snapshot);
|
||||
bool TranslateScalarMemory(const IR::Instruction& inst);
|
||||
bool TranslateBufferLoad(const IR::Instruction& inst);
|
||||
bool TranslateBufferStore(const IR::Instruction& inst);
|
||||
bool TranslateAtomicMemory(const IR::Instruction& inst);
|
||||
@@ -105,15 +98,13 @@ private:
|
||||
uint32_t upper_result);
|
||||
IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
|
||||
|
||||
bool TranslateInstruction(const IR::Instruction& inst,
|
||||
const ScalarMemorySourceValues* scalar_source_snapshot);
|
||||
bool TranslateInstruction(const IR::Instruction& inst);
|
||||
bool TranslateStateOperation(const IR::Instruction& inst);
|
||||
bool TranslateControlOperation(const IR::Instruction& inst);
|
||||
bool TranslateMove(const IR::Instruction& inst);
|
||||
bool TranslateLaneOperation(const IR::Instruction& inst);
|
||||
bool TranslateAttributeOperation(const IR::Instruction& inst);
|
||||
bool TranslateMemoryOperation(const IR::Instruction& inst,
|
||||
const ScalarMemorySourceValues* scalar_source_snapshot);
|
||||
bool TranslateMemoryOperation(const IR::Instruction& inst);
|
||||
bool TranslateIntegerCompare(const IR::Instruction& inst);
|
||||
bool TranslateInteger16Compare(const IR::Instruction& inst);
|
||||
bool TranslateFloatCompare(const IR::Instruction& inst);
|
||||
|
||||
@@ -273,7 +273,7 @@ void ClearRegisterOffsetModifiers(Decoder::Operand& operand) {
|
||||
operand.dpp = false;
|
||||
}
|
||||
|
||||
bool TryGetScalarDestinationCode(const Decoder::Operand& operand, uint32_t& code) {
|
||||
bool TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code) {
|
||||
switch (operand.kind) {
|
||||
case Decoder::OperandKind::Sgpr: code = operand.reg; return true;
|
||||
case Decoder::OperandKind::VccLo: code = SCALAR_DST_VCC_LO; return true;
|
||||
@@ -290,7 +290,7 @@ bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index,
|
||||
// aliases such as VCC and EXEC sit beside ordinary SGPRs. Re-decode after adding the
|
||||
// dword index so s_buffer_load_dwordx2 vcc_lo writes vcc_lo, then vcc_hi.
|
||||
uint32_t code = 0;
|
||||
return TryGetScalarDestinationCode(operand, code) &&
|
||||
return TryGetEncodedScalarCode(operand, code) &&
|
||||
Decoder::DecodeScalarDestination(code + index, 0, decoded, nullptr);
|
||||
}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand);
|
||||
MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind);
|
||||
uint32_t RawScalarLoadBase(const Decoder::Operand& operand);
|
||||
void ClearRegisterOffsetModifiers(Decoder::Operand& operand);
|
||||
bool TryGetScalarDestinationCode(const Decoder::Operand& operand, uint32_t& code);
|
||||
bool TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code);
|
||||
bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index,
|
||||
Decoder::Operand& result);
|
||||
Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index);
|
||||
|
||||
@@ -77,24 +77,40 @@ Opcode LoweredImageOpcode(Decoder::Opcode opcode) {
|
||||
|
||||
bool LowerScalarMemoryLoadDword(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
|
||||
std::string* error) {
|
||||
for (uint32_t i = 0; i < decoded.data_dwords; i++) {
|
||||
Instruction inst;
|
||||
inst.pc = decoded.pc;
|
||||
inst.op = op;
|
||||
inst.src_count = 1;
|
||||
const auto kind =
|
||||
op == Opcode::SLoadDword ? ResourceKind::ScalarAddress : ResourceKind::ScalarBuffer;
|
||||
inst.memory = OffsetMemoryInfo(decoded, kind, i);
|
||||
// Raw s_load uses a two-SGPR pointer base; s_buffer_load uses a four-SGPR
|
||||
// descriptor resource index.
|
||||
inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0)
|
||||
: ResourceIndexFromOperand(decoded.src0);
|
||||
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
|
||||
!LowerSourceOperand(decoded.src1, inst.src[0], error)) {
|
||||
return false;
|
||||
}
|
||||
block.instructions.push_back(inst);
|
||||
uint32_t destination_code = 0;
|
||||
const auto alignment = decoded.data_dwords == 1u ? 1u
|
||||
: decoded.data_dwords == 2u ? 2u
|
||||
: 4u;
|
||||
if (!TryGetEncodedScalarCode(decoded.dst, destination_code) ||
|
||||
destination_code % alignment != 0u ||
|
||||
destination_code + decoded.data_dwords > 108u) {
|
||||
SetError(error, "scalar-memory destination has an invalid aligned SGPR span");
|
||||
return false;
|
||||
}
|
||||
uint32_t base_code = 0;
|
||||
const auto base_alignment = op == Opcode::SLoadDword ? 2u : 4u;
|
||||
const auto base_width = op == Opcode::SLoadDword ? 2u : 4u;
|
||||
if (!TryGetEncodedScalarCode(decoded.src0, base_code) ||
|
||||
base_code % base_alignment != 0u || base_code + base_width > 108u) {
|
||||
SetError(error, "scalar-memory base has an invalid aligned SGPR span");
|
||||
return false;
|
||||
}
|
||||
Instruction inst;
|
||||
inst.pc = decoded.pc;
|
||||
inst.op = op;
|
||||
inst.src_count = 1;
|
||||
const auto kind =
|
||||
op == Opcode::SLoadDword ? ResourceKind::ScalarAddress : ResourceKind::ScalarBuffer;
|
||||
inst.memory = MemoryInfoFromDecoded(decoded, kind);
|
||||
// Raw s_load uses a two-SGPR pointer base; s_buffer_load uses a four-SGPR
|
||||
// descriptor resource index.
|
||||
inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0)
|
||||
: ResourceIndexFromOperand(decoded.src0);
|
||||
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
|
||||
!LowerSourceOperand(decoded.src1, inst.src[0], error)) {
|
||||
return false;
|
||||
}
|
||||
block.instructions.push_back(inst);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -311,6 +311,33 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st
|
||||
ValueOpcodeName(inst.GetOpcode()), inst.NumArgs(),
|
||||
NumArgsOf(inst.GetOpcode())));
|
||||
}
|
||||
if (IsRuntimeRead(inst.GetOpcode())) {
|
||||
const auto memory_index = inst.Flags<MemoryFlags>().index;
|
||||
if (memory_index >= program.memory_info.size()) {
|
||||
return Fail(error, fmt::format("{} has an invalid memory-info index",
|
||||
ValueOpcodeName(inst.GetOpcode())));
|
||||
}
|
||||
const auto& memory = program.memory_info[memory_index];
|
||||
const bool scalar_buffer = inst.GetOpcode() == ValueOpcode::ReadConstBuffer;
|
||||
const bool scalar_address = memory.kind == ResourceKind::ScalarAddress;
|
||||
if ((scalar_buffer && memory.kind != ResourceKind::ScalarBuffer) ||
|
||||
(!scalar_buffer && !scalar_address && memory.kind != ResourceKind::Flat &&
|
||||
memory.kind != ResourceKind::Global && memory.kind != ResourceKind::Scratch)) {
|
||||
return Fail(error, fmt::format("{} has an invalid scalar-memory resource kind",
|
||||
ValueOpcodeName(inst.GetOpcode())));
|
||||
}
|
||||
const bool scalar_memory = scalar_buffer || scalar_address;
|
||||
const bool valid_group_width =
|
||||
scalar_memory ? memory.component_count == 1u || memory.component_count == 2u ||
|
||||
memory.component_count == 4u || memory.component_count == 8u ||
|
||||
memory.component_count == 16u
|
||||
: memory.component_count >= 1u && memory.component_count <= 4u;
|
||||
if (memory.data_bits != 32u || memory.data_dwords != 1u ||
|
||||
!valid_group_width || memory.component_index >= memory.component_count) {
|
||||
return Fail(error, fmt::format("{} has inconsistent scalar-memory metadata",
|
||||
ValueOpcodeName(inst.GetOpcode())));
|
||||
}
|
||||
}
|
||||
const auto buffer_components = BufferComponentCount(inst.GetOpcode());
|
||||
if (buffer_components != 0u) {
|
||||
const auto memory_index = inst.Flags<MemoryFlags>().index;
|
||||
|
||||
@@ -14601,12 +14601,12 @@ TestCase ScalarMemoryLoadVariants() {
|
||||
};
|
||||
|
||||
const Load s_loads[] = {
|
||||
{0x00, 20, 0, 1}, {0x01, 21, 4, 2}, {0x02, 23, 12, 4},
|
||||
{0x03, 27, 28, 8}, {0x04, 40, 60, 16},
|
||||
{0x00, 20, 0, 1}, {0x01, 22, 4, 2}, {0x02, 24, 12, 4},
|
||||
{0x03, 28, 28, 8}, {0x04, 36, 60, 16},
|
||||
};
|
||||
const Load s_buffer_loads[] = {
|
||||
{0x08, 56, 0, 1}, {0x09, 57, 4, 2}, {0x0a, 59, 12, 4},
|
||||
{0x0b, 63, 28, 8}, {0x0c, 71, 60, 16},
|
||||
{0x08, 52, 0, 1}, {0x09, 54, 4, 2}, {0x0a, 56, 12, 4},
|
||||
{0x0b, 60, 28, 8}, {0x0c, 68, 60, 16},
|
||||
};
|
||||
|
||||
std::vector<u32> code;
|
||||
|
||||
+173
-114
@@ -5497,121 +5497,77 @@ void TestGraphicsCreateInterpolantMapping() {
|
||||
"interpolant identity tail was not filled");
|
||||
}
|
||||
|
||||
void TestNewShaderRecompilerWideMemoryLowering() {
|
||||
void TestNewShaderRecompilerNativeWideScalarMemoryIr() {
|
||||
const uint32_t shader[] = {
|
||||
EncodeSmem0(0x02, 4, 4),
|
||||
0u, // s_load_dwordx4 s[4:7]
|
||||
EncodeSmem0(0x0a, 8, 4),
|
||||
0u, // s_buffer_load_dwordx4 s[8:11]
|
||||
EncodeSmem0(0x09, 106, 4),
|
||||
0u, // s_buffer_load_dwordx2 vcc_lo
|
||||
EncodeMubuf0(0x0d, 16),
|
||||
EncodeMubuf1(20, 0, 1), // buffer_load_dwordx2 v[20:21]
|
||||
EncodeMubuf0(0x0f, 32),
|
||||
EncodeMubuf1(24, 0, 1), // buffer_load_dwordx3 v[24:26]
|
||||
EncodeMubuf0(0x0e, 48),
|
||||
EncodeMubuf1(28, 0, 1), // buffer_load_dwordx4 v[28:31]
|
||||
EncodeMubuf0(0x1d, 64),
|
||||
EncodeMubuf1(32, 0, 1), // buffer_store_dwordx2 v[32:33]
|
||||
EncodeMubuf0(0x1f, 80),
|
||||
EncodeMubuf1(36, 0, 1), // buffer_store_dwordx3 v[36:38]
|
||||
EncodeMubuf0(0x1e, 96),
|
||||
EncodeMubuf1(40, 0, 1), // buffer_store_dwordx4 v[40:43]
|
||||
EncodeMubuf0(0x08, 2),
|
||||
EncodeMubuf1(44, 0, 1), // buffer_load_ubyte v44
|
||||
EncodeMubuf0(0x0a, 4),
|
||||
EncodeMubuf1(45, 0, 1), // buffer_load_ushort v45
|
||||
EncodeFlat0(0x0c, 0, 4),
|
||||
EncodeFlat1(50, 0x7d, 0, 1), // flat_load_dword
|
||||
EncodeFlat0(0x0d, 1, 8),
|
||||
EncodeFlat1(52, 0x7d, 0, 1), // scratch_load_dwordx2
|
||||
EncodeFlat0(0x0e, 2, 12),
|
||||
EncodeFlat1(56, 0x7d, 0, 1), // global_load_dwordx4
|
||||
0xbf810000u,
|
||||
EncodeSmem0(0x00, 0, 4), 125u << 25u,
|
||||
EncodeSmem0(0x01, 2, 4), 125u << 25u,
|
||||
EncodeSmem0(0x02, 4, 4), 125u << 25u,
|
||||
EncodeSmem0(0x03, 8, 4), 125u << 25u,
|
||||
EncodeSmem0(0x04, 92, 4), 125u << 25u,
|
||||
EncodeSmem0(0x08, 32, 4), 125u << 25u,
|
||||
EncodeSmem0(0x09, 106, 4), 125u << 25u,
|
||||
EncodeSmem0(0x0a, 36, 4), 125u << 25u,
|
||||
EncodeSmem0(0x0b, 40, 4), 125u << 25u,
|
||||
EncodeSmem0(0x0c, 48, 4), 125u << 25u,
|
||||
EncodeSopp(0x01),
|
||||
};
|
||||
|
||||
auto options = MakeCompileOptions(ShaderType::Compute);
|
||||
options.dump_ir = true;
|
||||
options.flat_memory_base = 0;
|
||||
|
||||
ShaderRecompiler::CompileResult result;
|
||||
ShaderRecompiler::Decoder::Program decoded;
|
||||
ShaderRecompiler::CFG::Graph graph;
|
||||
ShaderRecompiler::IR::Program ir;
|
||||
std::string error;
|
||||
Check(ShaderRecompiler::TryRecompile(shader, options, result, &error),
|
||||
Check(ShaderRecompiler::Decoder::DecodeProgram(shader, decoded, &error) &&
|
||||
ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) &&
|
||||
ShaderRecompiler::IR::LowerProgram(decoded, graph,
|
||||
ShaderType::Compute, 64, ir,
|
||||
&error),
|
||||
error.c_str());
|
||||
Check(Common::ContainsStr(result.decoded_dump, "s_load_dwordx4"),
|
||||
"new decoder did not decode SMEM x4 load");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dwordx4"),
|
||||
"new decoder did not decode scalar-buffer x4 load");
|
||||
Check(
|
||||
Common::ContainsStr(result.decoded_dump, "s_buffer_load_dwordx2 vcc_lo"),
|
||||
"new decoder did not decode scalar-buffer x2 load into VCC");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx2"),
|
||||
"new decoder did not decode buffer x2 load");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx3"),
|
||||
"new decoder did not decode buffer x3 load");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx4"),
|
||||
"new decoder did not decode buffer x4 load");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "buffer_store_dwordx4"),
|
||||
"new decoder did not decode buffer x4 store");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "buffer_load_ubyte"),
|
||||
"new decoder did not decode buffer ubyte load");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "buffer_load_ushort"),
|
||||
"new decoder did not decode buffer ushort load");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "flat_load_dword"),
|
||||
"new decoder did not decode flat dword load");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "segment=1"),
|
||||
"new decoder did not preserve scratch segment metadata");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "segment=2"),
|
||||
"new decoder did not preserve global segment metadata");
|
||||
Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"),
|
||||
"new decoder did not expose width metadata for wide memory ops");
|
||||
Check(Common::ContainsStr(result.ir_dump, "SLoadDword s4"),
|
||||
"s_load_dwordx4 did not expand to first scalar dword IR load");
|
||||
Check(Common::ContainsStr(result.ir_dump, "SLoadDword s7"),
|
||||
"s_load_dwordx4 did not expand to last scalar dword IR load");
|
||||
Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s8"),
|
||||
"s_buffer_load_dwordx4 did not expand to first scalar-buffer IR load");
|
||||
Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s11"),
|
||||
"s_buffer_load_dwordx4 did not expand to last scalar-buffer IR load");
|
||||
Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_lo"),
|
||||
"s_buffer_load_dwordx2 did not expand to VCC low dword");
|
||||
Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_hi"),
|
||||
"s_buffer_load_dwordx2 did not expand to VCC high dword");
|
||||
Check(
|
||||
CountSourceOccurrences(result.ir_dump, "BufferLoadDword v20") == 1u &&
|
||||
CountSourceOccurrences(result.ir_dump, "BufferLoadDword v24") == 1u &&
|
||||
CountSourceOccurrences(result.ir_dump, "BufferLoadDword v28") == 1u &&
|
||||
!Common::ContainsStr(result.ir_dump, "BufferLoadDword v31"),
|
||||
"wide buffer loads were not preserved as one native IR operation");
|
||||
Check(CountSourceOccurrences(result.ir_dump, "BufferStoreDword null, v32") ==
|
||||
1u &&
|
||||
CountSourceOccurrences(result.ir_dump,
|
||||
"BufferStoreDword null, v36") == 1u &&
|
||||
CountSourceOccurrences(result.ir_dump,
|
||||
"BufferStoreDword null, v40") == 1u &&
|
||||
!Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v43"),
|
||||
"wide buffer stores were not preserved as one native IR operation");
|
||||
Check(Common::ContainsStr(result.ir_dump, "BufferLoadUbyte v44"),
|
||||
"buffer_load_ubyte did not lower to sub-dword IR load");
|
||||
Check(Common::ContainsStr(result.ir_dump, "BufferLoadUshort v45"),
|
||||
"buffer_load_ushort did not lower to sub-dword IR load");
|
||||
Check(Common::ContainsStr(result.ir_dump, "FlatLoadDword v50"),
|
||||
"flat_load_dword did not lower to flat dword IR load");
|
||||
Check(Common::ContainsStr(result.ir_dump, "scratch"),
|
||||
"scratch segment load did not lower with scratch metadata");
|
||||
Check(Common::ContainsStr(result.ir_dump, "global"),
|
||||
"global segment load did not lower with global metadata");
|
||||
Check(SpirvContainsOpcode(result.spirv, 65),
|
||||
"SPIR-V binary does not contain OpAccessChain");
|
||||
Check(SpirvContainsOpcode(result.spirv, 62),
|
||||
"SPIR-V binary does not contain OpStore");
|
||||
Check(SpirvContainsOpcode(result.spirv, 194),
|
||||
"SPIR-V binary does not contain OpShiftRightLogical");
|
||||
Check(SpirvContainsOpcode(result.spirv, 196),
|
||||
"SPIR-V binary does not contain OpShiftLeftLogical");
|
||||
Check(SpirvContainsOpcode(result.spirv, 199),
|
||||
"SPIR-V binary does not contain OpBitwiseAnd");
|
||||
CheckSpirvBinaryValidates(result.spirv);
|
||||
std::vector<uint32_t> address_widths;
|
||||
std::vector<uint32_t> buffer_widths;
|
||||
bool buffer_x2_targets_vcc = false;
|
||||
for (const auto &block : ir.blocks) {
|
||||
for (const auto &inst : block.instructions) {
|
||||
if (inst.op == ShaderRecompiler::IR::Opcode::SLoadDword) {
|
||||
address_widths.push_back(inst.memory.data_dwords);
|
||||
} else if (inst.op ==
|
||||
ShaderRecompiler::IR::Opcode::SBufferLoadDword) {
|
||||
buffer_widths.push_back(inst.memory.data_dwords);
|
||||
if (inst.memory.data_dwords == 2u) {
|
||||
buffer_x2_targets_vcc =
|
||||
inst.dst.reg.file == ShaderRecompiler::IR::RegisterFile::Vcc &&
|
||||
inst.dst.reg.index == 0u;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const std::vector<uint32_t> expected{1u, 2u, 4u, 8u, 16u};
|
||||
Check(address_widths == expected && buffer_widths == expected,
|
||||
"SMEM x1/x2/x4/x8/x16 did not remain one width-bearing ShaderIR operation each");
|
||||
Check(buffer_x2_targets_vcc,
|
||||
"s_buffer_load_dwordx2 did not preserve its VCC destination span");
|
||||
|
||||
const auto reject = [&](uint32_t opcode, uint32_t dst, uint32_t sbase,
|
||||
const char *reason) {
|
||||
const uint32_t invalid[] = {EncodeSmem0(opcode, dst, sbase),
|
||||
125u << 25u, EncodeSopp(0x01)};
|
||||
ShaderRecompiler::Decoder::Program invalid_decoded;
|
||||
ShaderRecompiler::CFG::Graph invalid_graph;
|
||||
ShaderRecompiler::IR::Program invalid_ir;
|
||||
std::string invalid_error;
|
||||
Check(ShaderRecompiler::Decoder::DecodeProgram(invalid, invalid_decoded,
|
||||
&invalid_error) &&
|
||||
ShaderRecompiler::CFG::BuildGraph(invalid_decoded, invalid_graph,
|
||||
&invalid_error) &&
|
||||
!ShaderRecompiler::IR::LowerProgram(
|
||||
invalid_decoded, invalid_graph, ShaderType::Compute, 64,
|
||||
invalid_ir, &invalid_error) &&
|
||||
Common::ContainsStr(invalid_error, reason),
|
||||
"invalid SMEM register span was not rejected");
|
||||
};
|
||||
reject(0x01, 1, 4, "destination");
|
||||
reject(0x02, 2, 4, "destination");
|
||||
reject(0x02, 124, 4, "destination");
|
||||
reject(0x00, 0, 63, "base");
|
||||
reject(0x08, 0, 1, "base");
|
||||
}
|
||||
|
||||
void TestNewShaderRecompilerNativeWideBufferIr() {
|
||||
@@ -8619,6 +8575,22 @@ void TestNativeWideValueValidation() {
|
||||
ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(0u), Value(true)},
|
||||
flags);
|
||||
};
|
||||
const auto append_scalar_read = [](ValueProgram &program,
|
||||
ValueOpcode opcode,
|
||||
MemoryFlags flags) {
|
||||
IREmitter ir(program.blocks.front());
|
||||
if (opcode == ValueOpcode::ReadConstBuffer) {
|
||||
const auto resource = ir.Emit(
|
||||
ValueOpcode::GetBufferResource,
|
||||
{Value(0u), Value(0u), Value(0u), Value(0u)});
|
||||
ir.Emit(opcode, {resource, Value(0u)}, flags);
|
||||
} else {
|
||||
const auto resource =
|
||||
ir.Emit(ValueOpcode::GetAddressResource, {Value(0u), Value(0u)});
|
||||
ir.Emit(opcode,
|
||||
{resource, Value(0u), Value(0u), Value(true)}, flags);
|
||||
}
|
||||
};
|
||||
const auto check_rejected = [](const ValueProgram &program,
|
||||
const char *expected) {
|
||||
std::string error;
|
||||
@@ -8627,6 +8599,58 @@ void TestNativeWideValueValidation() {
|
||||
"malformed native-wide value operation was not rejected");
|
||||
};
|
||||
|
||||
{
|
||||
auto program = make_program();
|
||||
append_scalar_read(program, ValueOpcode::ReadConstBuffer,
|
||||
MemoryFlags{.index = 1});
|
||||
check_rejected(program, "invalid memory-info index");
|
||||
}
|
||||
{
|
||||
auto program = make_program();
|
||||
MemoryInfo memory;
|
||||
memory.kind = ResourceKind::ScalarAddress;
|
||||
program.memory_info.push_back(memory);
|
||||
append_scalar_read(program, ValueOpcode::ReadConstBuffer, {});
|
||||
check_rejected(program, "invalid scalar-memory resource kind");
|
||||
}
|
||||
{
|
||||
auto program = make_program();
|
||||
MemoryInfo memory;
|
||||
memory.kind = ResourceKind::ScalarAddress;
|
||||
memory.data_bits = 16;
|
||||
program.memory_info.push_back(memory);
|
||||
append_scalar_read(program, ValueOpcode::LoadAddressU32, {});
|
||||
check_rejected(program, "inconsistent scalar-memory metadata");
|
||||
}
|
||||
{
|
||||
auto program = make_program();
|
||||
MemoryInfo memory;
|
||||
memory.kind = ResourceKind::ScalarBuffer;
|
||||
memory.data_dwords = 2;
|
||||
program.memory_info.push_back(memory);
|
||||
append_scalar_read(program, ValueOpcode::ReadConstBuffer, {});
|
||||
check_rejected(program, "inconsistent scalar-memory metadata");
|
||||
}
|
||||
{
|
||||
auto program = make_program();
|
||||
MemoryInfo memory;
|
||||
memory.kind = ResourceKind::ScalarBuffer;
|
||||
memory.component_count = 3;
|
||||
program.memory_info.push_back(memory);
|
||||
append_scalar_read(program, ValueOpcode::ReadConstBuffer, {});
|
||||
check_rejected(program, "inconsistent scalar-memory metadata");
|
||||
}
|
||||
{
|
||||
auto program = make_program();
|
||||
MemoryInfo memory;
|
||||
memory.kind = ResourceKind::ScalarAddress;
|
||||
memory.component_count = 4;
|
||||
memory.component_index = 4;
|
||||
program.memory_info.push_back(memory);
|
||||
append_scalar_read(program, ValueOpcode::LoadAddressU32, {});
|
||||
check_rejected(program, "inconsistent scalar-memory metadata");
|
||||
}
|
||||
|
||||
{
|
||||
auto program = make_program();
|
||||
MemoryInfo memory;
|
||||
@@ -9890,7 +9914,7 @@ void TestSrtWalkerRealSBufferLowering() {
|
||||
"real S_BUFFER_LOAD walk accepted a negative immediate");
|
||||
}
|
||||
|
||||
void TestScalarMemoryLoadsSnapshotOverlappingOperands() {
|
||||
void TestScalarMemorySourcesCapturedBeforeWrites() {
|
||||
const auto CheckOverlap = [](uint32_t opcode, bool overlap_offset) {
|
||||
const uint32_t base_field = overlap_offset ? 4u : 0u;
|
||||
const uint32_t soffset = overlap_offset ? 0u : 125u;
|
||||
@@ -9932,7 +9956,7 @@ void TestScalarMemoryLoadsSnapshotOverlappingOperands() {
|
||||
error.c_str());
|
||||
Check(flat.size() == table.size() &&
|
||||
std::equal(flat.begin(), flat.end(), table.begin()),
|
||||
"overlapping scalar-memory load did not snapshot its sources");
|
||||
"overlapping scalar-memory load did not capture its sources before writes");
|
||||
CheckFlattenedReadSlots(
|
||||
ir, 4, "overlapping scalar-memory patch used the wrong flat offsets");
|
||||
};
|
||||
@@ -9976,6 +10000,39 @@ void TestScalarMemoryLoadCrossesIntoVcc() {
|
||||
}
|
||||
}
|
||||
|
||||
void TestScalarMemoryUnusedTailDce() {
|
||||
const uint32_t shader[] = {
|
||||
EncodeSmem0(0x0c, 16, 0),
|
||||
4u << 25u, // s_buffer_load_dwordx16 s[16:31], s[0:3], s4
|
||||
EncodeVop1(0x01, 0, 16), // v_mov_b32 v0, s16
|
||||
EncodeMubuf0(0x1c),
|
||||
EncodeMubuf1(0, 2, 1), // keep only the first loaded component live
|
||||
EncodeSopp(0x01),
|
||||
};
|
||||
ShaderRecompiler::IR::Program ir;
|
||||
std::string error;
|
||||
Check(LowerTypedPlanning(shader, static_cast<uint32_t>(std::size(shader)),
|
||||
ir, &error),
|
||||
error.c_str());
|
||||
uint32_t reads = 0;
|
||||
for (const auto *block : ir.values->blocks) {
|
||||
for (const auto &inst : *block) {
|
||||
if (inst.GetOpcode() !=
|
||||
ShaderRecompiler::IR::ValueOpcode::ReadConstBuffer) {
|
||||
continue;
|
||||
}
|
||||
reads++;
|
||||
const auto flags = inst.Flags<ShaderRecompiler::IR::MemoryFlags>();
|
||||
Check(flags.index < ir.values->memory_info.size() &&
|
||||
ir.values->memory_info[flags.index].component_index == 0u &&
|
||||
ir.values->memory_info[flags.index].component_count == 16u,
|
||||
"live x16 SMEM component lost its native-width metadata");
|
||||
}
|
||||
}
|
||||
Check(reads == 1u,
|
||||
"grouped x16 SMEM prevented dead component reads from being eliminated");
|
||||
}
|
||||
|
||||
void TestResourceTrackingRealDensePatching() {
|
||||
const uint32_t shader[] = {
|
||||
EncodeMubuf0(0x0c),
|
||||
@@ -10735,6 +10792,7 @@ int main() {
|
||||
TestDemandDrivenSpirvDeclarations();
|
||||
TestNativeSubgroupPolicy();
|
||||
TestNewShaderRecompilerSMovB32();
|
||||
TestNewShaderRecompilerNativeWideScalarMemoryIr();
|
||||
TestNewShaderRecompilerNativeWideBufferIr();
|
||||
TestNewShaderRecompilerMubufFormatLowering();
|
||||
TestNewShaderRecompilerTypedBufferLowering();
|
||||
@@ -10809,8 +10867,9 @@ int main() {
|
||||
TestSrtWalkerRealSmemLowering();
|
||||
TestSrtWalkerVccBaseLowering();
|
||||
TestSrtWalkerRealSBufferLowering();
|
||||
TestScalarMemoryLoadsSnapshotOverlappingOperands();
|
||||
TestScalarMemorySourcesCapturedBeforeWrites();
|
||||
TestScalarMemoryLoadCrossesIntoVcc();
|
||||
TestScalarMemoryUnusedTailDce();
|
||||
TestResourceTrackingRealDensePatching();
|
||||
TestLowerProgramResetsAnalysisState();
|
||||
TestNewShaderRecompilerNativeBindingPlan();
|
||||
|
||||
Reference in New Issue
Block a user