#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { uint32_t EmitDppWriteActiveBool(EmitterState& state, const IR::Operand& dst) { const auto exec_active = EmitExecActiveBool(state); if (!dst.dpp) { return exec_active; } const auto subid = EmitSubgroupLocalInvocationId(state); const auto bank_shift = state.builder.AllocateId(); const auto row_shift = state.builder.AllocateId(); const auto bank = state.builder.AllocateId(); const auto row = state.builder.AllocateId(); const auto bank_bit = state.builder.AllocateId(); const auto row_bit = state.builder.AllocateId(); const auto bank_hit = state.builder.AllocateId(); const auto row_hit = state.builder.AllocateId(); const auto bank_active = state.builder.AllocateId(); const auto row_active = state.builder.AllocateId(); const auto dpp_active = state.builder.AllocateId(); const auto ret = state.builder.AllocateId(); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, bank_shift, subid, ConstantU32(state, 2)}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, bank, bank_shift, ConstantU32(state, 3)}); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, row_shift, subid, ConstantU32(state, 4)}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, row, row_shift, ConstantU32(state, 3)}); state.builder.AddFunction( {OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank}); state.builder.AddFunction( {OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, dst.dpp_bank_mask), bank_bit}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, dst.dpp_row_mask), row_bit}); state.builder.AddFunction( {OpINotEqual, state.bool_type, bank_active, bank_hit, ConstantU32(state, 0)}); state.builder.AddFunction( {OpINotEqual, state.bool_type, row_active, row_hit, ConstantU32(state, 0)}); state.builder.AddFunction({OpLogicalAnd, state.bool_type, dpp_active, bank_active, row_active}); uint32_t write_active = dpp_active; if (!dst.dpp_bound_ctrl) { const auto target = EmitDppTargetLane(state, dst.dpp_ctrl); const auto bounded = state.builder.AllocateId(); state.builder.AddFunction( {OpLogicalAnd, state.bool_type, bounded, write_active, target.valid}); write_active = bounded; } state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, exec_active, write_active}); return ret; } uint32_t EmitSdwaDestinationMerge(EmitterState& state, uint32_t pointer, const IR::Operand& dst, uint32_t value) { if (dst.sdwa_sel == 6u) { return value; } uint32_t offset = 0; uint32_t width = 0; if (!SdwaSelectorOffsetWidth(dst.sdwa_sel, offset, width) || width == 32u) { return value; } const auto value_mask = width == 16u ? 0x0000ffffu : 0x000000ffu; const auto field_mask = value_mask << offset; const auto part = state.builder.AllocateId(); const auto shifted = offset != 0u ? state.builder.AllocateId() : part; const auto old_value = state.builder.AllocateId(); const auto kept = state.builder.AllocateId(); const auto merged = state.builder.AllocateId(); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, part, value, ConstantU32(state, value_mask)}); if (offset != 0u) { state.builder.AddFunction( {OpShiftLeftLogical, state.uint_type, shifted, part, ConstantU32(state, offset)}); } state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, kept, old_value, ConstantU32(state, ~field_mask)}); state.builder.AddFunction({OpBitwiseOr, state.uint_type, merged, kept, shifted}); return merged; } void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value) { if (dst.kind != IR::OperandKind::Register) { return; } const auto pointer = PointerForRegister(state, dst.reg); if (pointer != 0) { const auto store_value = EmitSdwaDestinationMerge(state, pointer, dst, value); const auto wave_value = IsInactiveWave32ExecHigh(state, dst.reg) ? ConstantU32(state, 0) : store_value; if (dst.reg.file == IR::RegisterFile::Vector) { const auto old_value = state.builder.AllocateId(); const auto selected = state.builder.AllocateId(); state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer}); state.builder.AddFunction({OpSelect, state.uint_type, selected, EmitDppWriteActiveBool(state, dst), wave_value, old_value}); state.builder.AddFunction({OpStore, pointer, selected}); return; } state.builder.AddFunction({OpStore, pointer, wave_value}); } } uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction({OpIAdd, state.uint_type, ret, lhs, rhs}); return ret; } uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs}); return ret; } uint32_t EmitNotEqualZeroBool(EmitterState& state, uint32_t value) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction({OpINotEqual, state.bool_type, ret, value, ConstantU32(state, 0)}); return ret; } uint32_t EmitSelectU32Value(EmitterState& state, uint32_t condition, uint32_t true_value, uint32_t false_value) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction({OpSelect, state.uint_type, ret, condition, true_value, false_value}); return ret; } uint32_t EmitByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, uint32_t src_count) { uint32_t address = ConstantU32(state, inst.memory.offset); for (uint32_t i = 0; i < src_count; i++) { address = EmitAddU32(state, address, EmitValueLoad(state, inst.src[first_src + i])); } return address; } uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) { (void)use_pc; if (mem.resource >= state.program.info.buffers.size()) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, "buffer specialization is missing"); } return state.program.info.buffers[mem.resource].packed_stride; } uint32_t StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) { (void)use_pc; if (mem.resource >= state.program.info.buffers.size()) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, "buffer specialization is missing"); } return state.program.info.buffers[mem.resource].descriptor_format; } bool ShouldApplyBufferAddTid(const IR::Instruction& inst) { return inst.memory.kind == IR::ResourceKind::Buffer && inst.memory.formatted && !inst.memory.typed && inst.op == IR::Opcode::BufferStoreDword; } uint32_t EmitBufferIndexWithAddTid(EmitterState& state, const IR::Instruction& inst, uint32_t index, uint32_t packed) { if (!ShouldApplyBufferAddTid(inst)) { return index; } const auto add_tid_bit = EmitBinaryU32(state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 20)), ConstantU32(state, 1)); const auto add_tid = EmitNotEqualZeroBool(state, add_tid_bit); const auto lane = EmitBinaryU32(state, OpBitwiseAnd, EmitLocalInvocationIndex(state), ConstantU32(state, 63)); const auto tid_index = EmitAddU32(state, index, lane); return EmitSelectU32Value(state, add_tid, tid_index, index); } uint32_t EmitOptionalLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { if (lhs == 0) { return rhs; } if (rhs == 0) { return lhs; } const auto ret = state.builder.AllocateId(); state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, lhs, rhs}); return ret; } bool IsStorageBufferMemoryKind(IR::ResourceKind kind) { switch (kind) { case IR::ResourceKind::ScalarBuffer: case IR::ResourceKind::Buffer: case IR::ResourceKind::Flat: case IR::ResourceKind::Global: case IR::ResourceKind::Scratch: return true; default: return false; } } void EmitStorageBufferOffsets(EmitterState& state) { for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) { const auto word = EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u); const auto shift = ConstantU32(state, (i % 4u) * 8u + 2u); state.storage_buffer_offsets[i] = EmitBinaryU32( state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift), ConstantU32(state, 0x3fu)); } } uint32_t EmitBufferAddressFromParts(EmitterState& state, const IR::Instruction& inst, uint32_t index, uint32_t offset, uint32_t soffset) { const auto& mem = inst.memory; const auto packed = ConstantU32(state, StorageBufferPackedStride(state, mem, inst.pc)); const auto address_index = EmitBufferIndexWithAddTid(state, inst, index, packed); const auto stride = EmitBinaryU32(state, OpBitwiseAnd, packed, ConstantU32(state, 0x3fffu)); const auto swizzle_bit = EmitBinaryU32(state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 14)), ConstantU32(state, 1)); const auto stride_nonzero = EmitNotEqualZeroBool(state, stride); const auto swizzle_nonzero = EmitNotEqualZeroBool(state, swizzle_bit); const auto use_swizzle = state.builder.AllocateId(); state.builder.AddFunction( {OpLogicalAnd, state.bool_type, use_swizzle, swizzle_nonzero, stride_nonzero}); const auto index_stride_enum = EmitBinaryU32(state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 16)), ConstantU32(state, 3)); const auto index_shift = EmitAddU32(state, index_stride_enum, ConstantU32(state, 3)); const auto index_stride = EmitBinaryU32(state, OpShiftLeftLogical, ConstantU32(state, 8), index_stride_enum); const auto index_stride_mask = EmitBinaryU32(state, OpISub, index_stride, ConstantU32(state, 1)); const auto linear_index = EmitBinaryU32(state, OpIMul, address_index, stride); const auto linear = EmitAddU32(state, linear_index, offset); const auto index_msb = EmitBinaryU32(state, OpShiftRightLogical, address_index, index_shift); const auto index_lsb = EmitBinaryU32(state, OpBitwiseAnd, address_index, index_stride_mask); const auto offset_msb_bytes = EmitBinaryU32(state, OpBitwiseAnd, offset, ConstantU32(state, ~3u)); const auto offset_lsb = EmitBinaryU32(state, OpBitwiseAnd, offset, ConstantU32(state, 3)); const auto index_part = EmitBinaryU32(state, OpIMul, index_msb, stride); const auto msb_sum = EmitAddU32(state, index_part, offset_msb_bytes); const auto msb = EmitBinaryU32(state, OpIMul, msb_sum, index_stride); const auto lsb_index = EmitBinaryU32(state, OpShiftLeftLogical, index_lsb, ConstantU32(state, 2)); const auto lsb = EmitAddU32(state, lsb_index, offset_lsb); const auto swizzled = EmitAddU32(state, msb, lsb); const auto buffer_offset = EmitSelectU32Value(state, use_swizzle, swizzled, linear); return EmitAddU32(state, buffer_offset, soffset); } uint32_t EmitBufferByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, uint32_t src_count) { const auto end = first_src + src_count; auto cursor = first_src; auto LoadAddressSource = [&]() { if (cursor >= end) { return ConstantU32(state, 0); } return EmitValueLoad(state, inst.src[cursor++]); }; uint32_t index = ConstantU32(state, 0); if (inst.memory.idxen) { index = LoadAddressSource(); } uint32_t offset = ConstantU32(state, inst.memory.offset); if (inst.memory.offen) { offset = EmitAddU32(state, offset, LoadAddressSource()); } const auto soffset = LoadAddressSource(); return EmitBufferAddressFromParts(state, inst, index, offset, soffset); } bool IsFlatMemoryKind(IR::ResourceKind kind) { return kind == IR::ResourceKind::Flat || kind == IR::ResourceKind::Global || kind == IR::ResourceKind::Scratch; } uint32_t EmitRelativeAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, uint32_t src_count, bool vector_only = true, bool align_components = false) { const auto delta = state.program.info.addresses[inst.memory.resource].specialized_base; auto immediate = static_cast(inst.memory.offset); if (align_components) { immediate = static_cast(static_cast(immediate) & ~3u); } const auto initial = delta + static_cast(static_cast(immediate)); uint32_t low = ConstantU32(state, static_cast(initial)); uint32_t high = ConstantU32(state, static_cast(initial >> 32u)); for (uint32_t i = 0; i < src_count; i++) { const auto& src = inst.src[first_src + i]; if (!vector_only || (src.kind == IR::OperandKind::Register && src.reg.file == IR::RegisterFile::Vector)) { auto value = EmitValueLoad(state, src); if (align_components) { value = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~3u)); } const auto next = EmitAddU32(state, low, value); const auto carry = state.builder.AllocateId(); state.builder.AddFunction({OpULessThan, state.bool_type, carry, next, low}); high = EmitAddU32( state, high, EmitSelectU32Value(state, carry, ConstantU32(state, 1), ConstantU32(state, 0))); low = next; } } const auto valid = state.builder.AllocateId(); state.builder.AddFunction({OpIEqual, state.bool_type, valid, high, ConstantU32(state, 0)}); return EmitSelectU32Value(state, valid, low, ConstantU32(state, UINT32_MAX)); } uint32_t EmitFlatVirtualAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, uint32_t src_count) { if (inst.memory.resource >= state.resources.addresses.size() || src_count < 2) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, inst.memory.resource, "flat address snapshot is missing"); } const auto low0 = EmitValueLoad(state, inst.src[first_src]); const auto high0 = EmitValueLoad(state, inst.src[first_src + 1u]); const auto immediate = ConstantU32(state, inst.memory.offset); const auto low = EmitAddU32(state, low0, immediate); const auto carry_bool = state.builder.AllocateId(); state.builder.AddFunction({OpULessThan, state.bool_type, carry_bool, low, low0}); const auto carry = EmitSelectU32Value(state, carry_bool, ConstantU32(state, 1), ConstantU32(state, 0)); const auto immediate_high = ConstantU32(state, (inst.memory.offset & 0x80000000u) != 0 ? UINT32_MAX : 0u); const auto high = EmitAddU32(state, EmitAddU32(state, high0, immediate_high), carry); const auto base = state.program.info.addresses[inst.memory.resource].specialized_base; const auto base_low = static_cast(base); const auto base_high = static_cast(base >> 32u); const auto relative_low = EmitBinaryU32(state, OpISub, low, ConstantU32(state, base_low)); const auto borrow_bool = state.builder.AllocateId(); state.builder.AddFunction( {OpULessThan, state.bool_type, borrow_bool, low, ConstantU32(state, base_low)}); const auto borrow = EmitSelectU32Value(state, borrow_bool, ConstantU32(state, 1), ConstantU32(state, 0)); const auto relative_high = EmitBinaryU32( state, OpISub, EmitBinaryU32(state, OpISub, high, ConstantU32(state, base_high)), borrow); const auto valid = state.builder.AllocateId(); state.builder.AddFunction( {OpIEqual, state.bool_type, valid, relative_high, ConstantU32(state, 0)}); return EmitSelectU32Value(state, valid, relative_low, ConstantU32(state, UINT32_MAX)); } uint32_t EmitMemoryByteAddress(EmitterState& state, const IR::Instruction& inst, const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count) { if (mem.kind == IR::ResourceKind::Buffer) { return EmitBufferByteAddress(state, inst, first_src, src_count); } if (mem.kind == IR::ResourceKind::Flat || ((mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) && state.program.info.addresses[mem.resource].source == IR::ScalarProvenance::Unknown)) { return EmitFlatVirtualAddress(state, inst, first_src, src_count); } if (mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) { return EmitRelativeAddress(state, inst, first_src, src_count); } return EmitByteAddress(state, inst, first_src, src_count); } uint32_t EmitDwordIndex(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, uint32_t src_count) { const auto address = EmitByteAddress(state, inst, first_src, src_count); const auto index = state.builder.AllocateId(); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); return index; } uint32_t EmitMemoryDwordIndex(EmitterState& state, const IR::Instruction& inst, const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count) { const auto address = EmitMemoryByteAddress(state, inst, mem, first_src, src_count); const auto index = state.builder.AllocateId(); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); return index; } DescriptorResourceBinding StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) { (void)use_pc; const auto binding = ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource); if (state.storage_buffer_variable == 0) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, "storage buffer descriptor array was not emitted"); } return binding; } uint32_t EmitStorageBufferIndex(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, uint32_t use_pc) { if (IsFlatMemoryKind(mem.kind)) { return index; } const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); return EmitAddU32(state, index, state.storage_buffer_offsets[binding.array_index]); } uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) { if (IsFlatMemoryKind(mem.kind)) { if (state.address_memory_variable == 0) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, mem.resource, "address memory binding was not emitted"); } const auto binding = ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource); const auto pointer = state.builder.AllocateId(); state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer, state.address_memory_variable, ConstantU32(state, binding.array_index)}); return pointer; } const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); const auto pointer = state.builder.AllocateId(); state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer, state.storage_buffer_variable, ConstantU32(state, binding.array_index)}); return pointer; } uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, uint32_t use_pc) { index = EmitStorageBufferIndex(state, mem, index, use_pc); const auto object = EmitStorageBufferObjectPointer(state, mem, use_pc); const auto length = state.builder.AllocateId(); const auto in_bounds = state.builder.AllocateId(); state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0}); state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length}); return in_bounds; } uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, uint32_t use_pc) { index = EmitStorageBufferIndex(state, mem, index, use_pc); if (IsFlatMemoryKind(mem.kind)) { if (state.address_memory_variable == 0) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, mem.resource, "address memory binding was not emitted"); } const auto binding = ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource); const auto pointer = state.builder.AllocateId(); state.builder.AddFunction( {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable, ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); return pointer; } const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); const auto pointer = state.builder.AllocateId(); state.builder.AddFunction( {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.storage_buffer_variable, ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); return pointer; } uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) { const auto pointer = state.builder.AllocateId(); state.builder.AddFunction( {OpAccessChain, state.ptr_workgroup_uint, pointer, state.lds_variable, index}); return pointer; } uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) { const auto in_bounds = state.builder.AllocateId(); const auto dwords = state.needs_function_lds ? 8192u : 1024u; state.builder.AddFunction( {OpULessThan, state.bool_type, in_bounds, index, ConstantU32(state, dwords)}); return in_bounds; } uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, uint32_t use_pc) { if (mem.kind == IR::ResourceKind::Lds) { return EmitLdsElementPointer(state, index); } if (mem.kind == IR::ResourceKind::Gds) { return EmitGdsElementPointer(state, index); } return EmitStorageBufferElementPointer(state, mem, index, use_pc); } uint32_t EmitMemoryLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, uint32_t first_src, uint32_t src_count) { auto mem = inst.memory; mem.kind = kind; const auto index = EmitMemoryDwordIndex(state, inst, mem, first_src, src_count); auto LoadFromPointer = [&](uint32_t pointer) { const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); return value; }; if (mem.kind == IR::ResourceKind::Lds) { return LoadFromPointer(EmitLdsElementPointer(state, index)); } if (mem.kind == IR::ResourceKind::Gds) { const auto in_bounds = EmitGdsElementInBounds(state, index); return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { return LoadFromPointer(EmitGdsElementPointer(state, index)); }); } if (IsStorageBufferMemoryKind(mem.kind)) { const auto in_bounds = EmitStorageBufferElementInBounds(state, mem, index, inst.pc); return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { return LoadFromPointer(EmitStorageBufferElementPointer(state, mem, index, inst.pc)); }); } return ConstantU32(state, 0); } void EmitMemoryLoadU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, uint32_t first_src, uint32_t src_count) { const auto value = EmitMemoryLoadDwordValueU32(state, inst, kind, first_src, src_count); EmitStoreU32(state, inst.dst, value); } void EmitMemoryLoadDwordsU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, uint32_t first_src, uint32_t src_count) { const auto count = std::max(1u, inst.memory.data_dwords); if (count == 1u) { EmitMemoryLoadU32(state, inst, kind, first_src, src_count); return; } std::vector values; values.reserve(count); for (uint32_t i = 0; i < count; i++) { auto load_inst = inst; load_inst.memory.offset += i * 4u; load_inst.memory.data_dwords = 1u; load_inst.memory.component_index = i; load_inst.memory.component_count = count; values.push_back(EmitMemoryLoadDwordValueU32(state, load_inst, kind, first_src, src_count)); } for (uint32_t i = 0; i < count; i++) { EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), values[i]); } } uint32_t EmitMemoryLoadSubDwordValueU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, uint32_t data_bits, bool sign_extend) { auto mem = inst.memory; mem.kind = kind; const auto address = EmitMemoryByteAddress(state, inst, mem, first_src, src_count); const auto index = state.builder.AllocateId(); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); auto LoadFromPointer = [&](uint32_t pointer) { const auto word = state.builder.AllocateId(); const auto byte_in_word = state.builder.AllocateId(); const auto shift = state.builder.AllocateId(); const auto shifted = state.builder.AllocateId(); const auto masked = state.builder.AllocateId(); const auto value = sign_extend ? state.builder.AllocateId() : masked; const auto mask = data_bits == 8u ? 0xffu : 0xffffu; state.builder.AddFunction({OpLoad, state.uint_type, word, pointer}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, byte_in_word, address, ConstantU32(state, 3)}); state.builder.AddFunction( {OpShiftLeftLogical, state.uint_type, shift, byte_in_word, ConstantU32(state, 3)}); state.builder.AddFunction({OpShiftRightLogical, state.uint_type, shifted, word, shift}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, masked, shifted, ConstantU32(state, mask)}); if (sign_extend) { const auto left = state.builder.AllocateId(); const auto sign_shift = 32u - data_bits; state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, left, masked, ConstantU32(state, sign_shift)}); state.builder.AddFunction({OpShiftRightArithmetic, state.uint_type, value, left, ConstantU32(state, sign_shift)}); } return value; }; if (mem.kind == IR::ResourceKind::Lds) { return LoadFromPointer(EmitLdsElementPointer(state, index)); } if (mem.kind == IR::ResourceKind::Gds) { const auto in_bounds = EmitGdsElementInBounds(state, index); return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { return LoadFromPointer(EmitGdsElementPointer(state, index)); }); } if (IsStorageBufferMemoryKind(mem.kind)) { const auto in_bounds = EmitStorageBufferElementInBounds(state, mem, index, inst.pc); return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { return LoadFromPointer(EmitStorageBufferElementPointer(state, mem, index, inst.pc)); }); } return ConstantU32(state, 0); } void EmitMemoryLoadSubDwordU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, uint32_t data_bits, bool sign_extend) { const auto value = EmitMemoryLoadSubDwordValueU32(state, inst, kind, first_src, src_count, data_bits, sign_extend); EmitStoreU32(state, inst.dst, value); } void EmitMemoryStoreU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, uint32_t first_src, uint32_t src_count) { auto store_inst = inst; store_inst.memory.kind = kind; const auto value = EmitValueLoad(state, inst.src[0]); uint32_t in_bounds = 0; const auto index = EmitMemoryDwordIndex(state, store_inst, store_inst.memory, first_src, src_count); if (IsStorageBufferMemoryKind(store_inst.memory.kind)) { in_bounds = EmitOptionalLogicalAndBool( state, in_bounds, EmitStorageBufferElementInBounds(state, store_inst.memory, index, inst.pc)); } else if (store_inst.memory.kind == IR::ResourceKind::Gds) { in_bounds = EmitGdsElementInBounds(state, index); } EmitIfCondition(state, in_bounds, [&]() { const auto pointer = EmitMemoryElementPointer(state, store_inst.memory, index, inst.pc); state.builder.AddFunction({OpStore, pointer, value}); }); } template void EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired_value) { const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; const auto memory = kind == IR::ResourceKind::Lds ? MemorySemanticsWorkgroupMemory : MemorySemanticsUniformMemory; const auto preheader = state.builder.AllocateId(); const auto header = state.builder.AllocateId(); const auto continue_label = state.builder.AllocateId(); const auto merge = state.builder.AllocateId(); const auto initial = state.builder.AllocateId(); const auto observed = state.builder.AllocateId(); const auto exchanged = state.builder.AllocateId(); state.builder.AddFunction({OpBranch, preheader}); state.builder.AddFunction({OpLabel, preheader}); state.builder.AddFunction({OpAtomicLoad, state.uint_type, initial, pointer, ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)}); state.builder.AddFunction({OpBranch, header}); state.builder.AddFunction({OpLabel, header}); state.builder.AddFunction( {OpPhi, state.uint_type, observed, initial, preheader, exchanged, continue_label}); const auto desired = desired_value(observed); state.builder.AddFunction({OpAtomicCompareExchange, state.uint_type, exchanged, pointer, ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone), ConstantU32(state, MemorySemanticsNone), desired, observed}); const auto success = state.builder.AllocateId(); state.builder.AddFunction({OpIEqual, state.bool_type, success, exchanged, observed}); state.builder.AddFunction({OpLoopMerge, merge, continue_label, LoopControlNone}); state.builder.AddFunction({OpBranchConditional, success, merge, continue_label}); state.builder.AddFunction({OpLabel, continue_label}); state.builder.AddFunction({OpBranch, header}); state.builder.AddFunction({OpLabel, merge}); const auto semantics = MemorySemanticsAcquireRelease | memory; state.builder.AddFunction( {OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)}); } void EmitMemoryStoreSubDwordU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, uint32_t data_bits) { auto store_inst = inst; store_inst.memory.kind = kind; uint32_t in_bounds = 0; const auto address = EmitMemoryByteAddress(state, store_inst, store_inst.memory, first_src, src_count); const auto index = state.builder.AllocateId(); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); if (IsStorageBufferMemoryKind(store_inst.memory.kind)) { in_bounds = EmitOptionalLogicalAndBool( state, in_bounds, EmitStorageBufferElementInBounds(state, store_inst.memory, index, inst.pc)); } else if (store_inst.memory.kind == IR::ResourceKind::Gds) { in_bounds = EmitGdsElementInBounds(state, index); } EmitIfCondition(state, in_bounds, [&]() { const auto pointer = EmitMemoryElementPointer(state, store_inst.memory, index, inst.pc); const auto value = EmitValueLoad(state, inst.src[0]); const auto byte_in_word = state.builder.AllocateId(); const auto shift = state.builder.AllocateId(); const auto field_mask = state.builder.AllocateId(); const auto value_masked = state.builder.AllocateId(); const auto value_shifted = state.builder.AllocateId(); const auto mask = data_bits == 8u ? 0xffu : 0xffffu; state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, byte_in_word, address, ConstantU32(state, 3)}); state.builder.AddFunction( {OpShiftLeftLogical, state.uint_type, shift, byte_in_word, ConstantU32(state, 3)}); state.builder.AddFunction( {OpShiftLeftLogical, state.uint_type, field_mask, ConstantU32(state, mask), shift}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, value_masked, value, ConstantU32(state, mask)}); state.builder.AddFunction( {OpShiftLeftLogical, state.uint_type, value_shifted, value_masked, shift}); auto Merge = [&](uint32_t word) { const auto inverted = state.builder.AllocateId(); const auto cleared = state.builder.AllocateId(); const auto merged = state.builder.AllocateId(); state.builder.AddFunction({OpNot, state.uint_type, inverted, field_mask}); state.builder.AddFunction({OpBitwiseAnd, state.uint_type, cleared, word, inverted}); state.builder.AddFunction( {OpBitwiseOr, state.uint_type, merged, cleared, value_shifted}); return merged; }; if (store_inst.memory.kind == IR::ResourceKind::Lds || store_inst.memory.kind == IR::ResourceKind::Gds) { EmitAtomicUpdateU32(state, pointer, store_inst.memory.kind, Merge); } else { const auto word = state.builder.AllocateId(); state.builder.AddFunction({OpLoad, state.uint_type, word, pointer}); state.builder.AddFunction({OpStore, pointer, Merge(word)}); } }); } uint32_t AddressSourceCount(const IR::Instruction& inst, uint32_t first_src) { return inst.src_count > first_src ? inst.src_count - first_src : 0u; } bool IsFormattedBufferComponent(const IR::Instruction& inst) { return inst.memory.kind == IR::ResourceKind::Buffer && inst.memory.formatted && inst.memory.data_dwords == 1u; } Prospero::BufferFormat FormattedBufferFormat(const EmitterState& state, const IR::Instruction& inst) { return inst.memory.typed ? Format::DecodeTBufferFormat(inst.memory.data_format, inst.memory.number_format) : static_cast( StorageBufferFormat(state, inst.memory, inst.pc)); } IR::Instruction WithFormatComponentByteOffset(const IR::Instruction& inst, Prospero::BufferFormat format) { auto rebased = inst; if (inst.memory.typed || inst.memory.component_index == 0u) { return rebased; } const auto default_component_offset = inst.memory.component_index * 4u; if (rebased.memory.offset >= default_component_offset) { rebased.memory.offset = rebased.memory.offset - default_component_offset + Format::GetFormatComponentByteOffset(format, inst.memory.component_index); } return rebased; } uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction({OpBitcast, state.uint_type, ret, value}); return ret; } uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction({OpBitcast, state.float_type, ret, value}); return ret; } uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction({OpBitcast, state.int_type, ret, value}); return ret; } uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t constant) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)}); return ret; } uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value, uint32_t false_value) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction( {OpSelect, state.float_type, ret, condition, true_value, false_value}); return ret; } bool IsSignedFormatComponent(Format::ComponentType type) { return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm || type == Format::ComponentType::Sscaled; } uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint32_t offset, uint32_t bits, bool sign_extend) { if (bits == 32u && offset == 0u) { return raw_word; } if (sign_extend) { const auto signed_word = EmitTBufferBitcastU32ToI32(state, raw_word); const auto extracted = state.builder.AllocateId(); state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_word, ConstantU32(state, offset), ConstantU32(state, bits)}); const auto ret = state.builder.AllocateId(); state.builder.AddFunction({OpBitcast, state.uint_type, ret, extracted}); return ret; } const auto extracted = state.builder.AllocateId(); state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, raw_word, ConstantU32(state, offset), ConstantU32(state, bits)}); return extracted; } uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw) { const auto unpacked = state.builder.AllocateId(); const auto value = state.builder.AllocateId(); state.builder.AddFunction( {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, raw}); state.builder.AddFunction({OpCompositeExtract, state.float_type, value, unpacked, 0}); return EmitTBufferBitcastF32ToU32(state, value); } uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) { const auto mantissa_bits = bits == 11u ? 6u : 5u; const auto mantissa_mask = (1u << mantissa_bits) - 1u; const auto mantissa = EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask)); const auto exponent = EmitBinaryU32( state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)), ConstantU32(state, 0x1fu)); const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u)); const auto exponent_bits = EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23)); const auto mantissa_bits_32 = EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits)); const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32); const auto normal = EmitTBufferBitcastU32ToF32(state, normal_bits); const auto special_bits = EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32); const auto special = EmitTBufferBitcastU32ToF32(state, special_bits); const auto mantissa_f32 = state.builder.AllocateId(); const auto subnormal = state.builder.AllocateId(); state.builder.AddFunction({OpConvertUToF, state.float_type, mantissa_f32, mantissa}); state.builder.AddFunction( {OpFMul, state.float_type, subnormal, mantissa_f32, ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast(mantissa_bits)))}); const auto zero_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 0); const auto special_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 31); const auto finite = EmitTBufferSelectF32(state, zero_exp, subnormal, normal); const auto result = EmitTBufferSelectF32(state, special_exp, special, finite); return EmitTBufferBitcastF32ToU32(state, result); } uint32_t EmitFormatRawComponent(EmitterState& state, const IR::Instruction& inst, const Format::BufferFormatInfo& info) { const auto component = inst.memory.component_index; if (component >= info.component_count) { return ConstantU32(state, 0); } auto load_inst = WithFormatComponentByteOffset(inst, info.format); const auto bits = info.component_bits[component]; const auto sign_extend = IsSignedFormatComponent(info.type); if (info.packed_bitfield) { const auto raw_word = EmitMemoryLoadDwordValueU32( state, load_inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(load_inst, 0)); return EmitExtractFormatFieldU32(state, raw_word, info.component_bit_offset[component], bits, sign_extend); } if (bits == 32u) { return EmitMemoryLoadDwordValueU32(state, load_inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(load_inst, 0)); } return EmitMemoryLoadSubDwordValueU32(state, load_inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(load_inst, 0), bits, sign_extend); } uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info, uint32_t component, uint32_t raw) { const auto bits = info.component_bits[component]; switch (info.type) { case Format::ComponentType::Uint: case Format::ComponentType::Sint: return raw; case Format::ComponentType::Uscaled: { const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw}); return EmitTBufferBitcastF32ToU32(state, value); } case Format::ComponentType::Sscaled: { const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw); const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw}); return EmitTBufferBitcastF32ToU32(state, value); } case Format::ComponentType::Unorm: { const auto value = state.builder.AllocateId(); const auto normalized = state.builder.AllocateId(); const auto max_value = static_cast((1u << bits) - 1u); state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw}); state.builder.AddFunction( {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)}); return EmitTBufferBitcastF32ToU32(state, normalized); } case Format::ComponentType::Snorm: { const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw); const auto value = state.builder.AllocateId(); const auto normalized = state.builder.AllocateId(); const auto clamped = state.builder.AllocateId(); const auto max_value = static_cast((1u << (bits - 1u)) - 1u); state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw}); state.builder.AddFunction( {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)}); state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450, GlslFMax, normalized, ConstantF32Value(state, -1.0f)}); return EmitTBufferBitcastF32ToU32(state, clamped); } case Format::ComponentType::Float: if (bits == 32u) { return raw; } if (bits == 16u) { return EmitHalfToF32Bits(state, raw); } return EmitUFloatToF32Bits(state, raw, bits); default: return raw; } } uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst, const Format::BufferFormatInfo& info) { const auto raw = EmitFormatRawComponent(state, inst, info); return NormalizeFormatComponent(state, info, inst.memory.component_index, raw); } bool EmitFormattedBufferLoadValueU32(EmitterState& state, const IR::Instruction& inst, uint32_t& value) { if (!IsFormattedBufferComponent(inst)) { return false; } const auto format = FormattedBufferFormat(state, inst); if (!Format::IsKnownFormat(format)) { return false; } const auto info = Format::GetFormatInfo(format); if (inst.memory.component_index >= info.component_count) { value = ConstantU32(state, 0); return true; } if (Format::CanUseTypedBufferLoad(info.format)) { value = EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(inst, 0)); return true; } value = UnpackTBufferFormat(state, inst, info); return true; } uint32_t EmitBufferLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst) { uint32_t value = 0; if (EmitFormattedBufferLoadValueU32(state, inst, value)) { return value; } return EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(inst, 0)); } uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format, uint32_t opcode_components) { switch (format) { case Prospero::BufferFormat::k32UInt: case Prospero::BufferFormat::k32SInt: case Prospero::BufferFormat::k32Float: return opcode_components >= 1u ? 1u : 0u; case Prospero::BufferFormat::k32_32UInt: case Prospero::BufferFormat::k32_32SInt: case Prospero::BufferFormat::k32_32Float: return opcode_components >= 2u ? 2u : 0u; case Prospero::BufferFormat::k32_32_32UInt: case Prospero::BufferFormat::k32_32_32SInt: case Prospero::BufferFormat::k32_32_32Float: return opcode_components >= 3u ? 3u : 0u; case Prospero::BufferFormat::k32_32_32_32UInt: case Prospero::BufferFormat::k32_32_32_32SInt: case Prospero::BufferFormat::k32_32_32_32Float: return opcode_components >= 4u ? 4u : 0u; default: return 0u; } } bool EmitBufferIntegerFormatStore(EmitterState& state, const IR::Instruction& inst) { if (!IsFormattedBufferComponent(inst)) { return false; } const auto format = FormattedBufferFormat(state, inst); if (!inst.memory.typed) { const auto dword_components = FormattedBufferDwordStoreComponentCount(format, inst.memory.component_count); if (dword_components != 0u) { if (inst.memory.component_index >= dword_components) { return true; } EmitMemoryStoreU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1)); return true; } switch (format) { case Prospero::BufferFormat::k8UInt: case Prospero::BufferFormat::k8SInt: if (inst.memory.component_index != 0u) { return true; } EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1), 8); return true; case Prospero::BufferFormat::k16UInt: case Prospero::BufferFormat::k16SInt: if (inst.memory.component_index != 0u) { return true; } EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1), 16); return true; case Prospero::BufferFormat::k8_8UInt: case Prospero::BufferFormat::k8_8SInt: { if (inst.memory.component_index >= 2u) { return true; } const auto rebased = WithFormatComponentByteOffset(inst, format); EmitMemoryStoreSubDwordU32(state, rebased, IR::ResourceKind::Buffer, 1, AddressSourceCount(rebased, 1), 8); return true; } case Prospero::BufferFormat::k16_16UInt: case Prospero::BufferFormat::k16_16SInt: { if (inst.memory.component_index >= 2u) { return true; } const auto rebased = WithFormatComponentByteOffset(inst, format); EmitMemoryStoreSubDwordU32(state, rebased, IR::ResourceKind::Buffer, 1, AddressSourceCount(rebased, 1), 16); return true; } default: return false; } } switch (format) { case Prospero::BufferFormat::k8UInt: case Prospero::BufferFormat::k8SInt: EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1), 8); return true; case Prospero::BufferFormat::k16UInt: case Prospero::BufferFormat::k16SInt: EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1), 16); return true; case Prospero::BufferFormat::k8_8UInt: case Prospero::BufferFormat::k8_8SInt: EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1), 8); return true; case Prospero::BufferFormat::k16_16UInt: case Prospero::BufferFormat::k16_16SInt: EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1), 16); return true; default: break; } return false; } uint32_t EmitAtomicPointer(EmitterState& state, const IR::Instruction& inst) { switch (inst.memory.kind) { case IR::ResourceKind::Lds: return EmitLdsElementPointer(state, EmitDwordIndex(state, inst, 1, 1)); case IR::ResourceKind::Gds: return EmitGdsElementPointer(state, EmitDwordIndex(state, inst, 1, 1)); case IR::ResourceKind::Buffer: { const auto index = EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1)); return EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc); } case IR::ResourceKind::StorageImageUint: { const auto view = StorageImageViewKind(state, inst.memory, true, inst.pc); const auto image_pointer = StorageImageDescriptorPointer(state, inst.memory.resource, true, inst.pc, view); const auto pointer = state.builder.AllocateId(); state.builder.AddFunction({OpImageTexelPointer, state.ptr_image_uint, pointer, image_pointer, EmitImageCoordU32(state, inst, view), ConstantU32(state, 0)}); return pointer; } default: return 0; } } void EmitDeviceAtomicMemoryBarrier(EmitterState& state) { const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory; state.builder.AddFunction( {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)}); } void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { if (IsStorageBufferMemoryKind(inst.memory.kind)) { const auto index = EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1)); const auto in_bounds = EmitStorageBufferElementInBounds(state, inst.memory, index, inst.pc); const auto value = EmitValueLoad(state, inst.src[0]); const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() { const auto pointer = EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc); const auto result = state.builder.AllocateId(); state.builder.AddFunction({opcode, state.uint_type, result, pointer, ConstantU32(state, ScopeDevice), ConstantU32(state, MemorySemanticsNone), value}); EmitDeviceAtomicMemoryBarrier(state); return result; }); EmitStoreU32(state, inst.dst, old); return; } if (inst.memory.kind == IR::ResourceKind::Gds) { const auto index = EmitDwordIndex(state, inst, 1, 1); const auto in_bounds = EmitGdsElementInBounds(state, index); const auto value = EmitValueLoad(state, inst.src[0]); const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() { const auto pointer = EmitGdsElementPointer(state, index); const auto result = state.builder.AllocateId(); state.builder.AddFunction({opcode, state.uint_type, result, pointer, ConstantU32(state, ScopeDevice), ConstantU32(state, MemorySemanticsNone), value}); EmitDeviceAtomicMemoryBarrier(state); return result; }); EmitStoreU32(state, inst.dst, old); return; } const auto pointer = EmitAtomicPointer(state, inst); if (pointer == 0) { EmitStoreU32(state, inst.dst, ConstantU32(state, 0)); return; } const auto value = EmitValueLoad(state, inst.src[0]); const auto old = state.builder.AllocateId(); const auto scope = inst.memory.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; state.builder.AddFunction({opcode, state.uint_type, old, pointer, ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone), value}); if (inst.memory.kind == IR::ResourceKind::StorageImageUint || inst.memory.kind == IR::ResourceKind::Gds) { EmitDeviceAtomicMemoryBarrier(state); } EmitStoreU32(state, inst.dst, old); } void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst) { if (state.address_memory_variable == 0) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, inst.memory.resource, "scalar memory descriptor array was not emitted"); } const auto binding = ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, inst.memory.resource); const auto address = EmitRelativeAddress(state, inst, 0, 1, false, true); const auto index = state.builder.AllocateId(); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); const auto object = state.builder.AllocateId(); state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, object, state.address_memory_variable, ConstantU32(state, binding.array_index)}); const auto length = state.builder.AllocateId(); const auto in_bounds = state.builder.AllocateId(); state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0}); state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length}); const auto value = EmitValueOrZeroIfCondition(state, in_bounds, [&]() { const auto pointer = state.builder.AllocateId(); const auto loaded = state.builder.AllocateId(); state.builder.AddFunction( {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable, ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); state.builder.AddFunction({OpLoad, state.uint_type, loaded, pointer}); return loaded; }); EmitStoreU32(state, inst.dst, value); } void EmitLoadSrtDword(EmitterState& state, const IR::Instruction& inst) { if (state.flattened_srt_variable == 0 || inst.src_count != 1 || inst.src[0].kind != IR::OperandKind::ImmediateU32) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::FlattenedSrt, inst.src_count != 0 ? inst.src[0].imm : 0, "invalid flattened SRT load"); } const auto pointer = state.builder.AllocateId(); const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.flattened_srt_variable, ConstantU32(state, 0), ConstantU32(state, inst.src[0].imm)}); state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); EmitStoreU32(state, inst.dst, value); } void EmitBufferLoadUbyte(EmitterState& state, const IR::Instruction& inst) { EmitGuardedByExec(state, [&]() { EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(inst, 0), 8, false); }); } void EmitBufferLoadSbyte(EmitterState& state, const IR::Instruction& inst) { EmitGuardedByExec(state, [&]() { EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(inst, 0), 8, true); }); } void EmitBufferLoadUshort(EmitterState& state, const IR::Instruction& inst) { EmitGuardedByExec(state, [&]() { EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(inst, 0), 16, false); }); } void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst) { EmitGuardedByExec(state, [&]() { EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(inst, 0), 16, true); }); } void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst) { EmitGuardedByExec( state, [&]() { EmitStoreU32(state, inst.dst, EmitBufferLoadDwordValueU32(state, inst)); }); } void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions, uint32_t count) { if (instructions == nullptr || count == 0u) { return; } EmitGuardedByExec(state, [&]() { // RDNA VMEM captures every VADDR component before making overlapping VDATA writes // visible. Keep the split IR components instruction-atomic by deferring all stores. std::vector values; values.reserve(count); for (uint32_t i = 0; i < count; i++) { values.push_back(EmitBufferLoadDwordValueU32(state, instructions[i])); } for (uint32_t i = 0; i < count; i++) { EmitStoreU32(state, instructions[i].dst, values[i]); } }); } void EmitBufferStoreDword(EmitterState& state, const IR::Instruction& inst) { if (EmitBufferIntegerFormatStore(state, inst)) { return; } EmitMemoryStoreU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1)); } void EmitFlatLoadUbyte(EmitterState& state, const IR::Instruction& inst) { EmitGuardedByExec(state, [&]() { EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 8, false); }); } void EmitFlatLoadSbyte(EmitterState& state, const IR::Instruction& inst) { EmitGuardedByExec(state, [&]() { EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 8, true); }); } void EmitFlatLoadUshort(EmitterState& state, const IR::Instruction& inst) { EmitGuardedByExec(state, [&]() { EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 16, false); }); } void EmitFlatLoadSshort(EmitterState& state, const IR::Instruction& inst) { EmitGuardedByExec(state, [&]() { EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 16, true); }); } void EmitFlatLoadDword(EmitterState& state, const IR::Instruction& inst) { EmitGuardedByExec(state, [&]() { EmitMemoryLoadDwordsU32(state, inst, inst.memory.kind, 0, inst.src_count); }); } uint32_t EmitDsAddtidDwordIndex(EmitterState& state, const IR::Instruction& inst, uint32_t m0_src_index) { if (state.stage != ShaderType::Compute) { const auto m0 = m0_src_index < inst.src_count ? EmitValueLoad(state, inst.src[m0_src_index]) : ConstantU32(state, 0); const auto m0_lo = state.builder.AllocateId(); const auto base = state.builder.AllocateId(); const auto index = state.builder.AllocateId(); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, m0_lo, m0, ConstantU32(state, 0x0000ffffu)}); state.builder.AddFunction( {OpIAdd, state.uint_type, base, m0_lo, ConstantU32(state, inst.memory.offset)}); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, index, base, ConstantU32(state, 2)}); return index; } const auto local_index = EmitLocalInvocationIndex(state); const auto lane = state.builder.AllocateId(); const auto lane_bytes = state.builder.AllocateId(); const auto m0 = m0_src_index < inst.src_count ? EmitValueLoad(state, inst.src[m0_src_index]) : ConstantU32(state, 0); const auto m0_lo = state.builder.AllocateId(); const auto base = state.builder.AllocateId(); const auto address = state.builder.AllocateId(); const auto index = state.builder.AllocateId(); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, lane, local_index, ConstantU32(state, 63)}); state.builder.AddFunction( {OpShiftLeftLogical, state.uint_type, lane_bytes, lane, ConstantU32(state, 2)}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, m0_lo, m0, ConstantU32(state, 0x0000ffffu)}); state.builder.AddFunction( {OpIAdd, state.uint_type, base, m0_lo, ConstantU32(state, inst.memory.offset)}); state.builder.AddFunction({OpIAdd, state.uint_type, address, base, lane_bytes}); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); return index; } void EmitDsWriteAddtidB32(EmitterState& state, const IR::Instruction& inst) { const auto value = EmitValueLoad(state, inst.src[0]); const auto index = EmitDsAddtidDwordIndex(state, inst, 1); const auto pointer = EmitLdsElementPointer(state, index); state.builder.AddFunction({OpStore, pointer, value}); } void EmitDsReadAddtidB32(EmitterState& state, const IR::Instruction& inst) { const auto index = EmitDsAddtidDwordIndex(state, inst, 0); const auto pointer = EmitLdsElementPointer(state, index); const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); EmitStoreU32(state, inst.dst, value); } struct DsCounterAddress { uint32_t index = 0; uint32_t size = 0; uint32_t in_lds_bounds = 0; }; DsCounterAddress EmitAppendConsumeAddress(EmitterState& state, const IR::Instruction& inst) { const auto m0 = inst.src_count > 0 ? EmitValueLoad(state, inst.src[0]) : ConstantU32(state, 0); const auto base = state.builder.AllocateId(); const auto size = state.builder.AllocateId(); const auto address = state.builder.AllocateId(); const auto index = state.builder.AllocateId(); const auto in_bounds = state.builder.AllocateId(); // RDNA2 puts the DS append/consume byte base in M0[31:16]. state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, base, m0, ConstantU32(state, 16)}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, size, m0, ConstantU32(state, 0xffffu)}); state.builder.AddFunction( {OpIAdd, state.uint_type, address, base, ConstantU32(state, inst.memory.offset)}); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, ConstantU32(state, inst.memory.offset + 3u), size}); return {index, size, in_bounds}; } uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) { const auto length = state.builder.AllocateId(); const auto in_bounds = state.builder.AllocateId(); state.builder.AddFunction({OpArrayLength, state.uint_type, length, state.gds_variable, 0}); state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length}); return in_bounds; } uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) { const auto pointer = state.builder.AllocateId(); state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.gds_variable, ConstantU32(state, 0), index}); return pointer; } uint32_t EmitBitCountU32Value(EmitterState& state, uint32_t value) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction({OpBitCount, state.uint_type, ret, value}); return ret; } uint32_t EmitFindLsbU32Value(EmitterState& state, uint32_t value) { const auto i32 = state.builder.AllocateId(); const auto ret = state.builder.AllocateId(); state.builder.AddFunction( {OpExtInst, state.int_type, i32, state.glsl_std450, GlslFindILsb, value}); state.builder.AddFunction({OpBitcast, state.uint_type, ret, i32}); return ret; } struct ExecMaskInfo { uint32_t active_count = 0; uint32_t first_lane = 0; uint32_t any_active = 0; }; ExecMaskInfo EmitExecMaskInfo(EmitterState& state) { uint32_t exec_lo = 0; uint32_t exec_hi = 0; if (state.per_invocation_masks) { const auto ballot = state.builder.AllocateId(); state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot, ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)}); exec_lo = state.builder.AllocateId(); state.builder.AddFunction({OpCompositeExtract, state.uint_type, exec_lo, ballot, 0}); if (state.wave_size == 64u) { exec_hi = state.builder.AllocateId(); state.builder.AddFunction({OpCompositeExtract, state.uint_type, exec_hi, ballot, 1}); } else { exec_hi = ConstantU32(state, 0); } } else { exec_lo = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0}); exec_hi = WaveMaskHighLoad(state, IR::RegisterFile::Exec); } const auto lo_count = EmitBitCountU32Value(state, exec_lo); const auto hi_count = EmitBitCountU32Value(state, exec_hi); const auto active_count = state.builder.AllocateId(); state.builder.AddFunction({OpIAdd, state.uint_type, active_count, lo_count, hi_count}); const auto lo_nonzero = EmitNotEqualZeroBool(state, exec_lo); const auto hi_nonzero = EmitNotEqualZeroBool(state, exec_hi); const auto any_active = EmitLogicalOrBool(state, lo_nonzero, hi_nonzero); const auto hi_lsb = EmitFindLsbU32Value(state, exec_hi); const auto hi_lane = state.builder.AllocateId(); state.builder.AddFunction({OpIAdd, state.uint_type, hi_lane, hi_lsb, ConstantU32(state, 32)}); const auto lo_lsb = EmitFindLsbU32Value(state, exec_lo); const auto selected = state.builder.AllocateId(); state.builder.AddFunction({OpSelect, state.uint_type, selected, lo_nonzero, lo_lsb, hi_lane}); const auto first_lane = state.builder.AllocateId(); state.builder.AddFunction( {OpSelect, state.uint_type, first_lane, any_active, selected, ConstantU32(state, 0)}); return {active_count, first_lane, any_active}; } void EmitDsAppendConsume(EmitterState& state, const IR::Instruction& inst, uint32_t atomic_opcode) { const auto gds = inst.memory.kind == IR::ResourceKind::Gds; if (gds && state.gds_variable == 0) { EmitStoreU32(state, inst.dst, ConstantU32(state, 0)); return; } const auto address = EmitAppendConsumeAddress(state, inst); const auto storage_bounds = gds ? EmitGdsElementInBounds(state, address.index) : EmitLdsElementInBounds(state, address.index); const auto m0_bounds = gds ? EmitNotEqualZeroBool(state, address.size) : address.in_lds_bounds; const auto in_bounds = EmitLogicalAndBool(state, m0_bounds, storage_bounds); const auto exec = EmitExecMaskInfo(state); const auto subid = EmitSubgroupLocalInvocationId(state); const auto first_lane = state.builder.AllocateId(); const auto do_atomic = state.builder.AllocateId(); state.builder.AddFunction({OpIEqual, state.bool_type, first_lane, subid, exec.first_lane}); const auto first_active = EmitLogicalAndBool(state, first_lane, exec.any_active); state.builder.AddFunction({OpLogicalAnd, state.bool_type, do_atomic, first_active, in_bounds}); const auto atomic_value = EmitValueOrZeroIfCondition(state, do_atomic, [&]() { const auto pointer = gds ? EmitGdsElementPointer(state, address.index) : EmitLdsElementPointer(state, address.index); const auto result = state.builder.AllocateId(); state.builder.AddFunction({atomic_opcode, state.uint_type, result, pointer, ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup), ConstantU32(state, MemorySemanticsNone), exec.active_count}); if (gds) { EmitDeviceAtomicMemoryBarrier(state); } else { const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)}); } return result; }); const auto broadcast = state.builder.AllocateId(); state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, broadcast, ConstantU32(state, ScopeSubgroup), atomic_value, exec.first_lane}); const auto value = EmitSelectU32Value(state, exec.any_active, broadcast, ConstantU32(state, 0)); EmitStoreU32(state, inst.dst, value); } void EmitDsFloatMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool max_value) { const auto index = EmitDwordIndex(state, inst, 1, 1); const auto in_bounds = inst.memory.kind == IR::ResourceKind::Gds ? EmitGdsElementInBounds(state, index) : 0u; EmitIfCondition(state, in_bounds, [&]() { const auto pointer = EmitMemoryElementPointer(state, inst.memory, index, inst.pc); const auto data_f32 = EmitFloatLoad(state, inst.src[0]); const auto cmp_f32 = EmitFloatLoad(state, inst.src[2]); EmitAtomicUpdateU32(state, pointer, inst.memory.kind, [&](uint32_t old_u32) { const auto old_f32 = state.builder.AllocateId(); const auto store_src = state.builder.AllocateId(); const auto value_f32 = state.builder.AllocateId(); const auto value_u32 = state.builder.AllocateId(); state.builder.AddFunction({OpBitcast, state.float_type, old_f32, old_u32}); state.builder.AddFunction({max_value ? OpFOrdGreaterThan : OpFOrdLessThan, state.bool_type, store_src, max_value ? old_f32 : cmp_f32, max_value ? cmp_f32 : old_f32}); state.builder.AddFunction( {OpSelect, state.float_type, value_f32, store_src, data_f32, old_f32}); state.builder.AddFunction({OpBitcast, state.uint_type, value_u32, value_f32}); return value_u32; }); }); } uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) { if ((control & 0xc000u) == 0xc000u) { const uint32_t mask = control & 0x1fu; const uint32_t rotate = (control >> 5u) & 0x1fu; const uint32_t rotate_delta = (control & 0x400u) != 0u ? ((32u - rotate) & 0x1fu) : rotate; const auto lane = EmitAndConstant(state, subid, 31); const auto rotated_sum = EmitAddU32(state, lane, ConstantU32(state, rotate_delta)); const auto rotated = EmitAndConstant(state, rotated_sum, 31); const auto kept = EmitAndConstant(state, lane, mask); const auto moved = EmitAndConstant(state, rotated, (~mask) & 31u); const auto combined = EmitOrU32(state, kept, moved); const auto base = EmitAndConstant(state, subid, 0xffffffe0u); return EmitOrU32(state, base, combined); } if ((control & 0x8000u) != 0) { const auto lane2 = state.builder.AllocateId(); const auto shift = state.builder.AllocateId(); const auto perm0 = state.builder.AllocateId(); const auto perm = state.builder.AllocateId(); const auto base = state.builder.AllocateId(); const auto target = state.builder.AllocateId(); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, lane2, subid, ConstantU32(state, 3)}); state.builder.AddFunction( {OpShiftLeftLogical, state.uint_type, shift, lane2, ConstantU32(state, 1)}); state.builder.AddFunction( {OpShiftRightLogical, state.uint_type, perm0, ConstantU32(state, control), shift}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, perm, perm0, ConstantU32(state, 3)}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xfffffffcu)}); state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, perm}); return target; } const auto lane = state.builder.AllocateId(); const auto masked = state.builder.AllocateId(); const auto ored = state.builder.AllocateId(); const auto xored = state.builder.AllocateId(); const auto base = state.builder.AllocateId(); const auto target = state.builder.AllocateId(); state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)}); state.builder.AddFunction( {OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)}); state.builder.AddFunction( {OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)}); state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored}); return target; } void EmitDsSwizzleB32(EmitterState& state, const IR::Instruction& inst) { const auto source = EmitValueLoad(state, inst.src[0]); const auto control = inst.src_count > 1u ? inst.src[1].imm & 0xffffu : 0u; const auto subid = EmitSubgroupLocalInvocationId(state); const auto target = EmitDsSwizzleTargetLane(state, subid, control); const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value, ConstantU32(state, ScopeSubgroup), source, target}); const auto exec_active = EmitLaneIndexActiveBool(state, target); const auto subgroup_active = EmitSubgroupLaneActiveBool(state, target); const auto source_active = state.builder.AllocateId(); const auto selected = state.builder.AllocateId(); // RDNA2 DS_SWIZZLE returns zero when the selected source thread is invalid. state.builder.AddFunction( {OpLogicalAnd, state.bool_type, source_active, exec_active, subgroup_active}); state.builder.AddFunction( {OpSelect, state.uint_type, selected, source_active, value, ConstantU32(state, 0)}); EmitStoreU32(state, inst.dst, selected); } } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter