graphics: implement RDNA2 buffer atomic OR x2

This commit is contained in:
nmzik
2026-09-03 05:39:05 +02:00
parent cd74809086
commit 858c18468b
14 changed files with 195 additions and 26 deletions
@@ -601,12 +601,14 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
EXIT_NOT_IMPLEMENTED(supported_features2.features.vertexPipelineStoresAndAtomics != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features12.shaderOutputLayer != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features12.bufferDeviceAddress != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features12.shaderBufferInt64Atomics != VK_TRUE);
#if !defined(__APPLE__)
EXIT_NOT_IMPLEMENTED(supported_fragment_barycentric.fragmentShaderBarycentric != VK_TRUE);
#endif
features12.timelineSemaphore = VK_TRUE;
features12.shaderOutputLayer = VK_TRUE;
features12.bufferDeviceAddress = VK_TRUE;
features12.shaderBufferInt64Atomics = VK_TRUE;
vk::PhysicalDeviceFeatures device_features {};
device_features.fragmentStoresAndAtomics = VK_TRUE;
@@ -189,6 +189,9 @@ void AnalyzeProgramRequirements(IR::Program& program) {
const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() == IR::ValueOpcode::BufferAtomicOr64) {
requirements.buffer_int64_atomics = true;
}
const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access;
if (address_access != IR::AddressAccess::None) {
const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
@@ -39,6 +39,7 @@ enum : uint32_t {
MemoryModelGLSL450 = 1,
CapabilityShader = 1,
CapabilityInt64 = 11,
CapabilityInt64Atomics = 12,
CapabilityImageGatherExtended = 25,
CapabilityClipDistance = 32,
CapabilityCullDistance = 33,
@@ -73,6 +74,7 @@ enum : uint32_t {
DecorationBuiltIn = 11,
DecorationNoPerspective = 13,
DecorationFlat = 14,
DecorationAliased = 20,
DecorationLocation = 30,
DecorationArrayStride = 6,
DecorationBinding = 33,
@@ -352,6 +354,7 @@ struct EmitterState {
ShaderType stage = ShaderType::Unknown;
uint32_t wave_size = 64;
uint32_t storage_buffer_variable = 0;
uint32_t storage_buffer_u64_variable = 0;
std::array<uint32_t, IR::ShaderInfo::MaxBuffers> memory_byte_offsets {};
uint32_t bda_pagetable_variable = 0;
uint32_t fault_buffer_variable = 0;
@@ -390,6 +393,7 @@ uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
uint32_t TypeU32(EmitterState& state);
uint32_t TypeU64(EmitterState& state);
uint32_t TypeDeviceAddress(EmitterState& state);
uint32_t TypeScalarU64(EmitterState& state);
uint32_t TypeU32Pair(EmitterState& state);
uint32_t TypeI32(EmitterState& state);
uint32_t TypeI32Pair(EmitterState& state);
@@ -403,6 +407,8 @@ uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t point
uint32_t TypeFunction(EmitterState& state);
uint32_t TypeStorageBufferPointer(EmitterState& state);
uint32_t TypeStorageBufferElementPointer(EmitterState& state);
uint32_t TypeStorageBufferU64Pointer(EmitterState& state);
uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state);
uint32_t TypeDeviceAddressStoragePointer(EmitterState& state);
uint32_t TypePhysicalU32Pointer(EmitterState& state);
uint32_t TypePushConstantElementPointer(EmitterState& state);
@@ -618,11 +624,17 @@ struct MemoryResourceAccess {
uint32_t object_pointer = 0;
uint32_t length = 0;
uint32_t index_offset = 0;
uint32_t byte_offset = 0;
bool add_index_offset = false;
};
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
const IR::MemoryInfo& mem,
uint32_t variable,
uint32_t pointer_type);
uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
uint32_t raw_index);
@@ -632,6 +644,10 @@ uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAcce
uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index);
uint32_t EmitStorageBufferElementPointer(EmitterState& state,
const MemoryResourceAccess& access, uint32_t index,
uint32_t pointer_type);
uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value);
uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value);
@@ -672,6 +672,35 @@ uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor
return return_value ? result : ConstantU32(ctx.state, 0);
}
uint32_t EmitBufferAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem) {
auto& state = ctx.state;
return EmitValueOrDefaultIfCondition(
state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() {
const auto resource = PrepareStorageBufferResourceAccess(
state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state));
const auto byte_address = Binary(state, OpIAdd, TypeU32(state),
ByteAddress(ctx, inst, mem), resource.byte_offset);
const auto index = Binary(state, OpShiftRightLogical, TypeU32(state), byte_address,
ConstantU32(state, 3u));
return EmitValueOrDefaultIfCondition(
state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state),
ConstantU64(state, 0), [&]() {
const auto value = Unary(state, OpBitcast, TypeScalarU64(state),
ctx.Arg(inst, inst.NumArgs() - 2));
const auto old = state.builder.AllocateId();
state.builder.AddFunction(
{OpAtomicOr, TypeScalarU64(state), old,
EmitStorageBufferElementPointer(
state, resource, index, TypeStorageBufferU64ElementPointer(state)),
ConstantU32(state, ScopeDevice),
ConstantU32(state, MemorySemanticsNone), value});
EmitDeviceAtomicMemoryBarrier(state);
return Unary(state, OpBitcast, TypeU64(state), old);
});
});
}
uint32_t FloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
bool max_value) {
struct OrderedBits {
@@ -1080,12 +1109,15 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto op = inst.GetOpcode();
const auto buffer_components = IR::BufferComponentCount(op);
if (buffer_components > 1u) {
if (IR::BufferAccessOf(op) == IR::BufferAccess::Read) {
const auto access = IR::BufferAccessOf(op);
if (access == IR::BufferAccess::Read) {
ctx.Define(inst, LoadWideBuffer(ctx, inst, buffer_components));
} else {
StoreWideBuffer(ctx, inst, buffer_components);
return true;
}
if (access == IR::BufferAccess::Write) {
StoreWideBuffer(ctx, inst, buffer_components);
return true;
}
return true;
}
const auto shared_components = IR::SharedComponentCount(op);
if (shared_components > 1u) {
@@ -1181,6 +1213,10 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define(inst, EmitAtomic(ctx, inst, ctx.Memory(inst), true));
return true;
}
if (op == IR::ValueOpcode::BufferAtomicOr64) {
ctx.Define(inst, EmitBufferAtomicOr64(ctx, inst, ctx.Memory(inst)));
return true;
}
if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) {
ctx.Define(inst, FloatAtomic(ctx, inst, ctx.Memory(inst),
op == IR::ValueOpcode::BufferAtomicFMax32));
@@ -81,6 +81,27 @@ static void EnsureLdsStorage(EmitterState& state) {
state.builder.AddName(state.lds_variable, "lds_dwords");
}
MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
const IR::MemoryInfo& mem,
uint32_t variable,
uint32_t pointer_type) {
if (variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
"storage buffer descriptor array was not emitted");
}
const auto array_index =
ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
MemoryResourceAccess access {.kind = mem.kind};
access.object_pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, pointer_type, access.object_pointer, variable,
ConstantU32(state, array_index)});
access.byte_offset = state.memory_byte_offsets[array_index];
access.length = state.builder.AllocateId();
state.builder.AddFunction(
{OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
return access;
}
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem) {
MemoryResourceAccess access {.kind = mem.kind};
switch (mem.kind) {
@@ -113,22 +134,13 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
EXIT("physical address memory must use the BDA emitter\n");
case IR::ResourceKind::ScalarBuffer:
case IR::ResourceKind::Buffer: {
if (state.storage_buffer_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers,
mem.resource,
"storage buffer descriptor array was not emitted");
}
const auto array_index =
ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
access.object_pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state),
access.object_pointer, state.storage_buffer_variable,
ConstantU32(state, array_index)});
access = PrepareStorageBufferResourceAccess(
state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state));
access.index_offset =
EmitBinaryU32(state, OpShiftRightLogical, state.memory_byte_offsets[array_index],
EmitBinaryU32(state, OpShiftRightLogical, access.byte_offset,
ConstantU32(state, 2u));
access.add_index_offset = true;
break;
return access;
}
default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
}
@@ -152,17 +164,25 @@ uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAcce
uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index) {
const auto pointer = state.builder.AllocateId();
if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
const auto pointer = state.builder.AllocateId();
const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction
: state.stage == ShaderType::Compute ? StorageClassWorkgroup
: StorageClassFunction;
state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class),
pointer, access.object_pointer, index});
} else {
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
access.object_pointer, ConstantU32(state, 0), index});
return pointer;
}
return EmitStorageBufferElementPointer(state, access, index,
TypeStorageBufferElementPointer(state));
}
uint32_t EmitStorageBufferElementPointer(EmitterState& state,
const MemoryResourceAccess& access, uint32_t index,
uint32_t pointer_type) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, pointer_type, pointer, access.object_pointer,
ConstantU32(state, 0), index});
return pointer;
}
@@ -29,6 +29,10 @@ uint32_t TypeU64(EmitterState& state) {
}
uint32_t TypeDeviceAddress(EmitterState& state) {
return TypeScalarU64(state);
}
uint32_t TypeScalarU64(EmitterState& state) {
return state.builder.Type(OpTypeInt, {64, 0});
}
@@ -94,6 +98,25 @@ uint32_t TypeStorageBufferElementPointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, TypeU32(state));
}
uint32_t StorageU64RuntimeArrayType(EmitterState& state) {
return state.builder.DecoratedType(OpTypeRuntimeArray, {TypeScalarU64(state)},
{{OpDecorate, {DecorationArrayStride, sizeof(uint64_t)}}});
}
uint32_t StorageBufferU64Type(EmitterState& state) {
return state.builder.DecoratedType(
OpTypeStruct, {StorageU64RuntimeArrayType(state)},
{{OpMemberDecorate, {0, DecorationOffset, 0}}, {OpDecorate, {DecorationBlock}}});
}
uint32_t TypeStorageBufferU64Pointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, StorageBufferU64Type(state));
}
uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, TypeScalarU64(state));
}
uint32_t TypeDeviceAddressStoragePointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, TypeDeviceAddress(state));
}
@@ -162,6 +185,13 @@ void DefineDescriptorVariables(EmitterState& state) {
const auto pointer_type = TypePointer(state, StorageClassStorageBuffer, array_type);
state.storage_buffer_variable =
state.builder.DefineGlobalVariable(pointer_type, StorageClassStorageBuffer);
if (state.requirements.buffer_int64_atomics) {
const auto u64_array_type =
state.builder.Type(OpTypeArray, {StorageBufferU64Type(state), count});
state.storage_buffer_u64_variable = state.builder.DefineGlobalVariable(
TypePointer(state, StorageClassStorageBuffer, u64_array_type),
StorageClassStorageBuffer);
}
}
if (DescriptorBinding(state, IR::DescriptorBindingKind::BdaPagetable) != nullptr) {
state.bda_pagetable_variable = state.builder.DefineGlobalVariable(
@@ -570,6 +600,14 @@ void AddDescriptorAnnotationsAndNames(EmitterState& state) {
if (state.storage_buffer_variable != 0) {
Decorate(state.storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers);
}
if (state.storage_buffer_u64_variable != 0) {
Decorate(state.storage_buffer_u64_variable, "buffers_u64",
IR::DescriptorBindingKind::Buffers);
state.builder.AddAnnotation(
{OpDecorate, state.storage_buffer_variable, DecorationAliased});
state.builder.AddAnnotation(
{OpDecorate, state.storage_buffer_u64_variable, DecorationAliased});
}
if (state.bda_pagetable_variable != 0) {
Decorate(state.bda_pagetable_variable, "bda_pagetable",
IR::DescriptorBindingKind::BdaPagetable);
@@ -627,6 +665,10 @@ void DefineModule(EmitterState& state) {
state.builder.RequireCapability(CapabilityPhysicalStorageBufferAddresses);
state.builder.RequireExtension("SPV_KHR_physical_storage_buffer");
}
if (state.requirements.buffer_int64_atomics) {
state.builder.RequireCapability(CapabilityInt64);
state.builder.RequireCapability(CapabilityInt64Atomics);
}
if (state.clip_distance_variable != 0) {
state.builder.RequireCapability(CapabilityClipDistance);
}
@@ -58,6 +58,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
{0x3bu, Opcode::BUFFER_ATOMIC_XOR, 1, 32},
{0x3fu, Opcode::BUFFER_ATOMIC_FMIN, 1, 32},
{0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32},
{0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32},
};
constexpr MemoryOpcodeInfo MTBUF_OPCODE_LIST[] = {
@@ -600,6 +600,7 @@ std::string InstructionToString(const Instruction& inst) {
case Opcode::BUFFER_ATOMIC_UMAX:
case Opcode::BUFFER_ATOMIC_AND:
case Opcode::BUFFER_ATOMIC_OR:
case Opcode::BUFFER_ATOMIC_OR_X2:
case Opcode::BUFFER_ATOMIC_XOR:
case Opcode::BUFFER_ATOMIC_FMIN:
case Opcode::BUFFER_ATOMIC_FMAX:
@@ -463,6 +463,7 @@ enum class Opcode {
BUFFER_ATOMIC_UMAX,
BUFFER_ATOMIC_AND,
BUFFER_ATOMIC_OR,
BUFFER_ATOMIC_OR_X2,
BUFFER_ATOMIC_XOR,
BUFFER_ATOMIC_FMIN,
BUFFER_ATOMIC_FMAX,
@@ -522,12 +522,21 @@ bool Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
const auto memory = MemoryInfoFromDecoded(inst);
const auto resource = GetBufferResource(memory);
const auto address = ReadBufferAddress(inst, 1);
const auto result = ir.Emit(opcode,
{resource, address.index, address.offset, address.soffset,
ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()},
AddMemoryInfo(memory, inst.pc));
const auto data_src = MemorySourceAt(inst, 0);
IR::Value data = ReadU32(data_src);
if (memory.data_dwords == 2u) {
data = ir.ConstructU64(ReadU32(data_src), ReadU32(OffsetOperand(data_src, 1u)));
}
const auto result = ir.Emit(opcode,
{resource, address.index, address.offset, address.soffset, data,
ir.GetExec()},
AddMemoryInfo(memory, inst.pc));
if (inst.glc) {
WriteOperand(inst.dst, result);
WriteOperand(inst.dst,
memory.data_dwords == 1u ? result : ir.CompositeExtract(result, 0u));
if (memory.data_dwords == 2u) {
WriteOperand(OffsetOperand(inst.dst, 1u), ir.CompositeExtract(result, 1u));
}
}
return true;
}
@@ -935,6 +944,8 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst) {
return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicAnd32);
case Decoder::Opcode::BUFFER_ATOMIC_OR:
return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicOr32);
case Decoder::Opcode::BUFFER_ATOMIC_OR_X2:
return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicOr64);
case Decoder::Opcode::BUFFER_ATOMIC_XOR:
return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicXor32);
case Decoder::Opcode::BUFFER_ATOMIC_FMIN:
@@ -433,6 +433,7 @@ struct SpirvRequirements {
bool function_lds = false;
bool function_scratch = false;
bool pixel_valid_mask = false;
bool buffer_int64_atomics = false;
};
struct BlockInfo {
@@ -119,6 +119,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
case ValueOpcode::BufferAtomicUMax32:
case ValueOpcode::BufferAtomicAnd32:
case ValueOpcode::BufferAtomicOr32:
case ValueOpcode::BufferAtomicOr64:
case ValueOpcode::BufferAtomicXor32:
case ValueOpcode::BufferAtomicFMin32:
case ValueOpcode::BufferAtomicFMax32: return BufferAccess::Atomic;
@@ -128,6 +129,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
uint32_t BufferComponentCount(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::BufferAtomicOr64:
case ValueOpcode::LoadBufferU32x2:
case ValueOpcode::StoreBufferU32x2: return 2u;
case ValueOpcode::LoadBufferU32x3:
@@ -230,6 +230,7 @@ VALUE_OPCODE(BufferAtomicSMax32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicUMax32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicAnd32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicOr32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicOr64, U64, BufferResource, U32, U32, U32, U64, U1)
VALUE_OPCODE(BufferAtomicXor32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicFMin32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicFMax32, U32, BufferResource, U32, U32, U32, U32, U1)
+32
View File
@@ -8593,6 +8593,37 @@ void TestNewShaderRecompilerBufferAtomicsGuardedByBounds() {
"buffer atomic memory barrier was emitted before atomic");
}
void TestCapturedBufferAtomicOrX2() {
const uint32_t shader[] = {
0xe1680018u, 0x80000000u, // buffer_atomic_or_x2 v[0:1], s[0:3], 0 offset:24
EncodeSopp(0x01),
};
auto options = MakeCompileOptions(ShaderType::Compute);
options.dump_ir = true;
auto result = RecompileForTest(shader, options);
Check(Common::ContainsStr(result.decoded_dump, "BUFFER_ATOMIC_OR_X2"),
"captured MUBUF instruction did not decode as buffer_atomic_or_x2");
Check(Common::ContainsStr(result.ir_dump, "BufferAtomicOr64"),
"buffer_atomic_or_x2 did not lower to a 64-bit buffer atomic");
Check(!Common::ContainsStr(result.ir_dump, "SetVectorRegister"),
"GLC=0 buffer_atomic_or_x2 unexpectedly wrote an old value to v0:v1");
CheckSpirvBinaryValidates(result.spirv);
const auto source = DisassembleSpirvBinary(result.spirv);
Check(Common::ContainsStr(source, "OpCapability Int64Atomics"),
"64-bit buffer atomic SPIR-V lacks Int64Atomics capability");
Check(Common::ContainsStr(source, "ArrayStride 8"),
"64-bit buffer atomic storage view does not use eight-byte elements");
Check(CountSourceOccurrences(source, "Aliased") == 2u,
"both storage-buffer views must declare that they alias");
Check(SpirvInstructionOpcodeCount(result.spirv, 241u) == 1u,
"buffer_atomic_or_x2 must lower to exactly one OpAtomicOr");
Check(Common::ContainsStr(source, "OpMemoryBarrier"),
"64-bit buffer atomic SPIR-V lacks its device-memory barrier");
}
void TestNewShaderRecompilerBranchConditionForms() {
struct Case {
uint32_t opcode;
@@ -11935,6 +11966,7 @@ int main() {
TestComputeDispatchWaveSize();
TestNewShaderRecompilerBufferLoadsGuardedByExec();
TestNewShaderRecompilerBufferAtomicsGuardedByBounds();
TestCapturedBufferAtomicOrX2();
TestNewShaderRecompilerPixelImageSampleLodSelection();
TestNewShaderRecompilerBranchConditionForms();
TestNewShaderRecompilerSetpcBranch();