refactor(shader): preserve native wide buffer operations

This commit is contained in:
nmzik
2026-08-19 07:51:11 +02:00
parent 3ea2099245
commit 5383972e96
25 changed files with 1100 additions and 350 deletions
@@ -140,7 +140,14 @@ U64 IREmitter::PackUint2x32(U32 low, U32 high) {
}
U32 IREmitter::CompositeExtract(Value composite, uint32_t index) {
return U32(Emit(ValueOpcode::CompositeExtractU32x2, {composite, Value(index)}));
ValueOpcode opcode;
switch (composite.GetType()) {
case Type::U32x2: opcode = ValueOpcode::CompositeExtractU32x2; break;
case Type::U32x3: opcode = ValueOpcode::CompositeExtractU32x3; break;
case Type::U32x4: opcode = ValueOpcode::CompositeExtractU32x4; break;
default: EXIT("invalid U32 composite type\n");
}
return U32(Emit(opcode, {composite, Value(index)}));
}
U32 IREmitter::IAdd(U32 lhs, U32 rhs) {
@@ -328,54 +328,6 @@ MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind ki
return mem;
}
uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded,
uint32_t component_index) {
if (!decoded.typed || !decoded.formatted) {
return component_index * 4u;
}
const auto format = Format::DecodeTBufferFormat(decoded.data_format, decoded.number_format);
if (!Format::IsKnownFormat(format)) {
return component_index * 4u;
}
return Format::GetFormatComponentByteOffset(format, component_index);
}
uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded) {
if (!decoded.typed || !decoded.formatted) {
return decoded.data_dwords;
}
const auto format = Format::DecodeTBufferFormat(decoded.data_format, decoded.number_format);
if (!Format::IsKnownFormat(format)) {
return decoded.data_dwords;
}
return Format::GetFormatComponentCount(format);
}
bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value,
BasicBlock& block, std::string* error) {
Instruction inst;
inst.pc = pc;
inst.op = Opcode::MoveU32;
inst.src[0] = MakeImmediateU32(value);
inst.src_count = 1;
if (!LowerRegisterOperand(dst, inst.dst, error)) {
return false;
}
block.instructions.push_back(inst);
return true;
}
MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, uint32_t component_index) {
auto mem = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
mem.offset += TypedBufferComponentOffsetBytes(decoded, component_index);
mem.data_dwords = 1;
mem.component_index = component_index;
mem.component_count = decoded.data_dwords;
return mem;
}
MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t byte_offset) {
auto mem = MemoryInfoFromDecoded(decoded, kind);
+3 -2
View File
@@ -123,8 +123,9 @@ struct ExportInfo {
};
struct InputInfo {
uint32_t attr = 0;
uint32_t chan = 0;
uint32_t attr = 0;
uint32_t chan = 0;
uint32_t component_count = 1;
bool operator==(const InputInfo& other) const = default;
};
@@ -36,15 +36,9 @@ bool TryOffsetScalarDestination(const Decoder::Operand& operand, uin
Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index);
MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t dword_index);
uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded,
uint32_t component_index);
uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded);
bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value,
BasicBlock& block, std::string* error);
MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, uint32_t component_index);
MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t byte_offset);
ResourceKind FlatSegmentResourceKind(uint32_t segment);
MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t byte_offset);
ResourceKind FlatSegmentResourceKind(uint32_t segment);
const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index);
Decoder::Operand M0Operand();
@@ -130,7 +130,8 @@ std::string InputInfoToString(Opcode op, const InputInfo& input) {
if (op != Opcode::LoadInputF32) {
return "";
}
return fmt::format(" ; input_attr={} input_chan={}", input.attr, input.chan);
return fmt::format(" ; input_attr={} input_chan={} input_components={}", input.attr, input.chan,
input.component_count);
}
std::string TerminatorToString(const CFG::Terminator& term) {
@@ -121,52 +121,29 @@ bool LowerBufferAddressSources(const Decoder::Instruction& decoded, Instruction&
}
bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto ir_op = BufferLoadOpcode(decoded);
const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
const auto typed_format_components = TypedBufferFormatComponentCount(decoded);
for (uint32_t i = 0; i < count; i++) {
if (i >= typed_format_components) {
if (!LowerMoveImmediateU32(decoded.pc, OffsetDecodedRegister(decoded.dst, i), 0, block,
error)) {
return false;
}
continue;
}
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.memory = OffsetBufferMemoryInfo(decoded, i);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
!LowerBufferAddressSources(decoded, inst, 0, error)) {
return false;
}
block.instructions.push_back(inst);
Instruction inst;
inst.pc = decoded.pc;
inst.op = BufferLoadOpcode(decoded);
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerBufferAddressSources(decoded, inst, 0, error)) {
return false;
}
block.instructions.push_back(inst);
return true;
}
bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto ir_op = BufferStoreOpcode(decoded.data_bits);
auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
if (decoded.data_bits == 32u) {
const auto typed_format_components = TypedBufferFormatComponentCount(decoded);
if (typed_format_components < count) {
count = typed_format_components;
}
}
for (uint32_t i = 0; i < count; i++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.memory = OffsetBufferMemoryInfo(decoded, i);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, i), inst.src[0], error) ||
!LowerBufferAddressSources(decoded, inst, 1, error)) {
return false;
}
block.instructions.push_back(inst);
Instruction inst;
inst.pc = decoded.pc;
inst.op = BufferStoreOpcode(decoded.data_bits);
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(decoded.dst, inst.src[0], error) ||
!LowerBufferAddressSources(decoded, inst, 1, error)) {
return false;
}
block.instructions.push_back(inst);
return true;
}
@@ -304,6 +304,46 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st
return Fail(error, fmt::format("untyped opcode {} survived translation",
ValueOpcodeName(inst.GetOpcode())));
}
const auto buffer_components = BufferComponentCount(inst.GetOpcode());
if (buffer_components != 0u) {
const auto memory_index = inst.Flags<MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) {
return Fail(error, fmt::format("{} has an invalid memory-info index",
ValueOpcodeName(inst.GetOpcode())));
}
const auto& memory = program.memory_info[memory_index];
if (memory.kind != ResourceKind::Buffer &&
memory.kind != ResourceKind::ScalarBuffer) {
return Fail(error, fmt::format("{} has a non-buffer resource kind",
ValueOpcodeName(inst.GetOpcode())));
}
if (buffer_components > 1u &&
(memory.kind != ResourceKind::Buffer || memory.data_bits != 32u ||
memory.data_dwords != buffer_components || memory.component_index != 0u)) {
return Fail(error, fmt::format("{} has inconsistent native-wide metadata",
ValueOpcodeName(inst.GetOpcode())));
}
if (buffer_components == 1u &&
(inst.GetOpcode() == ValueOpcode::LoadBufferU32 ||
inst.GetOpcode() == ValueOpcode::StoreBufferU32) &&
memory.data_dwords != 1u) {
return Fail(error, fmt::format("{} retains scalar-sibling width metadata",
ValueOpcodeName(inst.GetOpcode())));
}
}
uint32_t composite_components = 0u;
switch (inst.GetOpcode()) {
case ValueOpcode::CompositeExtractU32x2: composite_components = 2u; break;
case ValueOpcode::CompositeExtractU32x3: composite_components = 3u; break;
case ValueOpcode::CompositeExtractU32x4: composite_components = 4u; break;
default: break;
}
if (composite_components != 0u &&
(!inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32 ||
inst.Arg(1).U32() >= composite_components)) {
return Fail(error, fmt::format("{} has an invalid component index",
ValueOpcodeName(inst.GetOpcode())));
}
for (size_t arg_index = 0; arg_index < inst.NumArgs(); arg_index++) {
const auto arg = inst.Arg(arg_index);
if (arg.IsEmpty()) {
@@ -34,6 +34,7 @@ constexpr Type U64 = Type::U64;
constexpr Type F16 = Type::F16;
constexpr Type F32 = Type::F32;
constexpr Type U32x2 = Type::U32x2;
constexpr Type U32x3 = Type::U32x3;
constexpr Type U32x4 = Type::U32x4;
constexpr Type F32x2 = Type::F32x2;
constexpr Type SrtResource = Type::SrtResource;
@@ -133,10 +134,16 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
case ValueOpcode::ReadConstBuffer:
case ValueOpcode::LoadBufferU8:
case ValueOpcode::LoadBufferU16:
case ValueOpcode::LoadBufferU32: return BufferAccess::Read;
case ValueOpcode::LoadBufferU32:
case ValueOpcode::LoadBufferU32x2:
case ValueOpcode::LoadBufferU32x3:
case ValueOpcode::LoadBufferU32x4: return BufferAccess::Read;
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32: return BufferAccess::Write;
case ValueOpcode::StoreBufferU32:
case ValueOpcode::StoreBufferU32x2:
case ValueOpcode::StoreBufferU32x3:
case ValueOpcode::StoreBufferU32x4: return BufferAccess::Write;
case ValueOpcode::BufferAtomicSwap32:
case ValueOpcode::BufferAtomicIAdd32:
case ValueOpcode::BufferAtomicISub32:
@@ -153,4 +160,16 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
}
}
uint32_t BufferComponentCount(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::LoadBufferU32x2:
case ValueOpcode::StoreBufferU32x2: return 2u;
case ValueOpcode::LoadBufferU32x3:
case ValueOpcode::StoreBufferU32x3: return 3u;
case ValueOpcode::LoadBufferU32x4:
case ValueOpcode::StoreBufferU32x4: return 4u;
default: return BufferAccessOf(opcode) == BufferAccess::None ? 0u : 1u;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -55,6 +55,7 @@ static_assert(std::is_trivially_copyable_v<ExportFlags>);
[[nodiscard]] size_t NumArgsOf(ValueOpcode opcode);
[[nodiscard]] bool HasSideEffects(ValueOpcode opcode);
[[nodiscard]] BufferAccess BufferAccessOf(ValueOpcode opcode);
[[nodiscard]] uint32_t BufferComponentCount(ValueOpcode opcode);
[[nodiscard]] std::string_view ValueOpcodeName(ValueOpcode opcode);
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -57,9 +57,11 @@ VALUE_OPCODE(ConvertU32F32, U32, F32)
VALUE_OPCODE(ConvertF32S32, F32, U32)
VALUE_OPCODE(ConvertF32U32, F32, U32)
VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32)
VALUE_OPCODE(CompositeConstructU32x3, U32x3, U32, U32, U32)
VALUE_OPCODE(CompositeConstructF32x2, F32x2, F32, F32)
VALUE_OPCODE(CompositeConstructU32x4, U32x4, U32, U32, U32, U32)
VALUE_OPCODE(CompositeExtractU32x2, U32, U32x2, U32)
VALUE_OPCODE(CompositeExtractU32x3, U32, U32x3, U32)
VALUE_OPCODE(CompositeExtractU32x4, U32, U32x4, U32)
VALUE_OPCODE(PackUint2x32, U64, U32x2)
VALUE_OPCODE(UnpackUint2x32, U32x2, U64)
@@ -215,9 +217,15 @@ VALUE_OPCODE(StoreAddressU32, Void, AddressResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU8, U8, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU16, U16, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU32, U32, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU32x2, U32x2, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU32x3, U32x3, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU32x4, U32x4, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(StoreBufferU8, Void, BufferResource, U32, U32, U32, U8, U1)
VALUE_OPCODE(StoreBufferU16, Void, BufferResource, U32, U32, U32, U16, U1)
VALUE_OPCODE(StoreBufferU32, Void, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(StoreBufferU32x2, Void, BufferResource, U32, U32, U32, U32x2, U1)
VALUE_OPCODE(StoreBufferU32x3, Void, BufferResource, U32, U32, U32, U32x3, U1)
VALUE_OPCODE(StoreBufferU32x4, Void, BufferResource, U32, U32, U32, U32x4, U1)
VALUE_OPCODE(BufferAtomicSwap32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicIAdd32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1)
@@ -287,6 +287,9 @@ void FoldInstruction(Inst& inst) {
case ValueOpcode::CompositeExtractU32x2:
FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x2, 2);
return;
case ValueOpcode::CompositeExtractU32x3:
FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x3, 3);
return;
case ValueOpcode::CompositeExtractU32x4:
FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x4, 4);
return;