shader: implement BUFFER_ATOMIC_FMIN, unhandled IR opcode now fails

This commit is contained in:
nmzik
2026-08-05 02:14:48 +02:00
parent 0a2f4814ac
commit df44ad3fac
14 changed files with 187 additions and 5 deletions
@@ -84,6 +84,7 @@ bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) {
case IR::Opcode::AtomicAndU32:
case IR::Opcode::AtomicOrU32:
case IR::Opcode::AtomicXorU32:
case IR::Opcode::AtomicFMinF32:
case IR::Opcode::FlatLoadUbyte:
case IR::Opcode::FlatLoadSbyte:
case IR::Opcode::FlatLoadUshort:
@@ -57,6 +57,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPS[] = {
{0x39u, Opcode::BufferAtomicAnd, 1, 32},
{0x3au, Opcode::BufferAtomicOr, 1, 32},
{0x3bu, Opcode::BufferAtomicXor, 1, 32},
{0x3fu, Opcode::BufferAtomicFMin, 1, 32},
};
constexpr MemoryOpcodeInfo MTBUF_OPS[] = {
@@ -835,6 +835,7 @@ std::string OpcodeToString(Opcode opcode) {
case Opcode::BufferAtomicAnd: return "buffer_atomic_and";
case Opcode::BufferAtomicOr: return "buffer_atomic_or";
case Opcode::BufferAtomicXor: return "buffer_atomic_xor";
case Opcode::BufferAtomicFMin: return "buffer_atomic_fmin";
case Opcode::FlatLoadUbyte: return "flat_load_ubyte";
case Opcode::FlatLoadSbyte: return "flat_load_sbyte";
case Opcode::FlatLoadUshort: return "flat_load_ushort";
@@ -1125,6 +1126,7 @@ std::string InstructionToString(const Instruction& inst) {
case Opcode::BufferAtomicAnd:
case Opcode::BufferAtomicOr:
case Opcode::BufferAtomicXor:
case Opcode::BufferAtomicFMin:
case Opcode::BufferLoadSbyte:
case Opcode::BufferLoadSshort:
case Opcode::FlatLoadUbyte:
@@ -434,6 +434,7 @@ enum class Opcode {
BufferAtomicAnd,
BufferAtomicOr,
BufferAtomicXor,
BufferAtomicFMin,
FlatLoadUbyte,
FlatLoadSbyte,
FlatLoadUshort,
@@ -131,7 +131,8 @@ bool IsAtomic(IR::Opcode op) {
case IR::Opcode::AtomicUMaxU32:
case IR::Opcode::AtomicAndU32:
case IR::Opcode::AtomicOrU32:
case IR::Opcode::AtomicXorU32: return true;
case IR::Opcode::AtomicXorU32:
case IR::Opcode::AtomicFMinF32: return true;
default: return false;
}
}
@@ -775,6 +775,9 @@ void EmitInstruction(EmitterState& state, const IR::Instruction& inst) {
case IR::Opcode::AtomicXorU32:
EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicXor); });
break;
case IR::Opcode::AtomicFMinF32:
EmitGuardedByExec(state, [&]() { EmitAtomicFMinF32(state, inst); });
break;
case IR::Opcode::FlatLoadUbyte: EmitFlatLoadUbyte(state, inst); break;
case IR::Opcode::FlatLoadSbyte: EmitFlatLoadSbyte(state, inst); break;
case IR::Opcode::FlatLoadUshort: EmitFlatLoadUshort(state, inst); break;
@@ -1011,6 +1011,8 @@ void EmitDeviceAtomicMemoryBarrier(EmitterState& state);
void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
void EmitAtomicFMinF32(EmitterState& state, const IR::Instruction& inst);
void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst);
void EmitLoadSrtDword(EmitterState& state, const IR::Instruction& inst);
@@ -1573,19 +1575,22 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&
}
const auto then_label = state.builder.AllocateId();
const auto then_exit = state.builder.AllocateId();
const auto else_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId();
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state.builder.AddFunction({OpBranchConditional, condition, then_label, else_label});
state.builder.AddFunction({OpLabel, then_label});
const auto then_value = fn();
state.builder.AddFunction({OpBranch, then_exit});
state.builder.AddFunction({OpLabel, then_exit});
state.builder.AddFunction({OpBranch, merge_label});
state.builder.AddFunction({OpLabel, else_label});
state.builder.AddFunction({OpBranch, merge_label});
state.builder.AddFunction({OpLabel, merge_label});
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpPhi, state.uint_type, value, then_value, then_label, ConstantU32(state, 0), else_label});
{OpPhi, state.uint_type, value, then_value, then_exit, ConstantU32(state, 0), else_label});
return value;
}
@@ -640,8 +640,8 @@ void EmitMemoryStoreU32(EmitterState& state, const IR::Instruction& inst, IR::Re
}
template <typename Fn>
void EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind kind,
Fn&& desired_value) {
uint32_t EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind kind,
Fn&& desired_value) {
const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
const auto memory = kind == IR::ResourceKind::Lds ? MemorySemanticsWorkgroupMemory
: MemorySemanticsUniformMemory;
@@ -675,6 +675,7 @@ void EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind
const auto semantics = MemorySemanticsAcquireRelease | memory;
state.builder.AddFunction(
{OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)});
return observed;
}
void EmitMemoryStoreSubDwordU32(EmitterState& state, const IR::Instruction& inst,
@@ -1181,6 +1182,68 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
EmitStoreU32(state, inst.dst, old);
}
namespace {
uint32_t EmitF32BitsOrderedLessThan(EmitterState& state, uint32_t lhs_bits, uint32_t rhs_bits) {
struct ClassifiedBits {
uint32_t nan = 0;
uint32_t zero = 0;
uint32_t key = 0;
};
const auto Classify = [&](uint32_t bits) {
ClassifiedBits cls;
const auto abs_bits = EmitAndConstant(state, bits, 0x7fffffffu);
const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
const auto exponent_max =
EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
const auto negative = EmitCompareU32Constant(state, OpINotEqual,
EmitAndConstant(state, bits, 0x80000000u), 0);
const auto negative_key = state.builder.AllocateId();
const auto positive_key = state.builder.AllocateId();
// Map all non-NaN IEEE-754 encodings to monotonically increasing unsigned keys.
state.builder.AddFunction({OpNot, state.uint_type, negative_key, bits});
state.builder.AddFunction(
{OpBitwiseXor, state.uint_type, positive_key, bits, ConstantU32(state, 0x80000000u)});
cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
cls.key = EmitSelectValueU32(state, negative, negative_key, positive_key);
return cls;
};
const auto lhs = Classify(lhs_bits);
const auto rhs = Classify(rhs_bits);
const auto any_nan = EmitLogicalOrBool(state, lhs.nan, rhs.nan);
// The key transform orders -0 below +0, while the floating comparison treats them as equal.
const auto both_zero = EmitLogicalAndBool(state, lhs.zero, rhs.zero);
const auto ordered_nonzero =
EmitLogicalNotBool(state, EmitLogicalOrBool(state, any_nan, both_zero));
const auto less = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, state.bool_type, less, lhs.key, rhs.key});
return EmitLogicalAndBool(state, ordered_nonzero, less);
}
} // namespace
void EmitAtomicFMinF32(EmitterState& state, const IR::Instruction& inst) {
const auto index =
EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1));
const auto in_bounds = EmitStorageBufferElementInBounds(state, inst.memory, index, inst.pc);
const auto src_u32 = EmitValueLoad(state, inst.src[0]);
const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() {
const auto pointer = EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc);
return EmitAtomicUpdateU32(state, pointer, inst.memory.kind, [&](uint32_t old_u32) {
const auto replace_old = state.builder.AllocateId();
const auto less = EmitF32BitsOrderedLessThan(state, src_u32, old_u32);
state.builder.AddFunction(
{OpSelect, state.uint_type, replace_old, less, src_u32, old_u32});
return replace_old;
});
});
EmitStoreU32(state, inst.dst, old);
}
void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst) {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
@@ -30,7 +30,8 @@ bool IsAtomic(Opcode op) {
case Opcode::AtomicUMaxU32:
case Opcode::AtomicAndU32:
case Opcode::AtomicOrU32:
case Opcode::AtomicXorU32: return true;
case Opcode::AtomicXorU32:
case Opcode::AtomicFMinF32: return true;
default: return false;
}
}
@@ -569,6 +569,8 @@ bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& blo
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicOrU32, error);
case Decoder::Opcode::BufferAtomicXor:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicXorU32, error);
case Decoder::Opcode::BufferAtomicFMin:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicFMinF32, error);
case Decoder::Opcode::FlatLoadUbyte:
case Decoder::Opcode::FlatLoadSbyte:
case Decoder::Opcode::FlatLoadUshort:
@@ -729,6 +731,7 @@ bool IsMemoryOpcode(Decoder::Opcode opcode) {
case Decoder::Opcode::BufferAtomicAnd:
case Decoder::Opcode::BufferAtomicOr:
case Decoder::Opcode::BufferAtomicXor:
case Decoder::Opcode::BufferAtomicFMin:
case Decoder::Opcode::FlatLoadUbyte:
case Decoder::Opcode::FlatLoadSbyte:
case Decoder::Opcode::FlatLoadUshort:
@@ -352,6 +352,7 @@ constexpr LowerMap LOWER_OPS[] = {
{Decoder::Opcode::BufferAtomicAnd, Opcode::AtomicAndU32},
{Decoder::Opcode::BufferAtomicOr, Opcode::AtomicOrU32},
{Decoder::Opcode::BufferAtomicXor, Opcode::AtomicXorU32},
{Decoder::Opcode::BufferAtomicFMin, Opcode::AtomicFMinF32},
{Decoder::Opcode::FlatLoadUbyte, Opcode::FlatLoadUbyte},
{Decoder::Opcode::FlatLoadSbyte, Opcode::FlatLoadSbyte},
{Decoder::Opcode::FlatLoadSshort, Opcode::FlatLoadSshort},
@@ -307,6 +307,7 @@ IR_OPCODE(AtomicUMaxU32, General)
IR_OPCODE(AtomicAndU32, General)
IR_OPCODE(AtomicOrU32, General)
IR_OPCODE(AtomicXorU32, General)
IR_OPCODE(AtomicFMinF32, General)
IR_OPCODE(FlatLoadUbyte, General)
IR_OPCODE(FlatLoadSbyte, General)
IR_OPCODE(FlatLoadUshort, General)