From 5970c4ef95c62a10c87a551dc5ae6d2fbe8a0c08 Mon Sep 17 00:00:00 2001 From: nmzik Date: Mon, 17 Aug 2026 14:35:28 +0200 Subject: [PATCH] shader_recompiler: remove legacy SPIR-V emitter --- .../shader/recompiler/ShaderRecompiler.cpp | 3 +- .../recompiler/backend/spirv/SpirvEmitter.cpp | 5 +- .../recompiler/backend/spirv/SpirvEmitter.h | 3 +- .../backend/spirv/spirvEmitterAluOps.cpp | 2550 ----------------- .../backend/spirv/spirvEmitterAnalysis.cpp | 27 - .../backend/spirv/spirvEmitterExport.cpp | 226 -- .../backend/spirv/spirvEmitterFlowOps.cpp | 379 --- .../spirv/spirvEmitterImageHelpers.cpp | 333 --- .../backend/spirv/spirvEmitterImageOps.cpp | 342 --- .../backend/spirv/spirvEmitterInternal.h | 671 ----- .../backend/spirv/spirvEmitterMemory.cpp | 1713 ----------- .../backend/spirv/spirvEmitterValue.cpp | 26 + .../spirv/spirvEmitterValueAluHelpers.cpp | 423 +++ .../backend/spirv/spirvEmitterValueFlow.cpp | 35 +- .../spirv/spirvEmitterValueHelpers.cpp | 247 ++ .../backend/spirv/spirvEmitterValueImage.cpp | 17 + .../spirv/spirvEmitterValueMemoryHelpers.cpp | 371 +++ .../backend/spirv/spirvEmitterValues.cpp | 839 ------ tests/shaderCfgTests.cpp | 73 - 19 files changed, 1112 insertions(+), 7171 deletions(-) delete mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluOps.cpp delete mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterExport.cpp delete mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlowOps.cpp delete mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageHelpers.cpp delete mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageOps.cpp delete mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueAluHelpers.cpp create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueHelpers.cpp create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemoryHelpers.cpp delete mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterValues.cpp diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index ecac2ce54..75e912204 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -953,8 +953,7 @@ bool TryRecompile(std::span code, const CompileOptions& options, std::string emit_error; LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash); - if (!Spirv::EmitProgram(ir, resources, options.input_info, spirv, &emit_error, - options.dump_ir)) { + if (!Spirv::EmitProgram(ir, resources, options.input_info, spirv, &emit_error)) { LOGF("%s typed SPIR-V emit failed: %s\n", GetDumpLabel(options), emit_error.c_str()); if (dispatcher_fallback && error != nullptr) { *error = fmt::format("dispatcher fallback failed after {}: {}", diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 8a1588db2..5abac1d1f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -312,8 +312,8 @@ bool ProgramRequiresExactSubgroupSize(const IR::Program& program) { } bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources, - ShaderStageInputInfo input_info, std::vector& spirv, std::string* error, - bool preserve_debug_values) { + ShaderStageInputInfo input_info, std::vector& spirv, + std::string* error) { using namespace Emitter; if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex && @@ -339,7 +339,6 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc SetError(error, "SPIR-V emitter requires planned typed SSA"); return false; } - (void)preserve_debug_values; const auto& value_program = *program.values; if (!IR::ValidateValueProgram(value_program, true, error)) { return false; diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h index 8e478c9a3..476bce9fa 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h @@ -12,8 +12,7 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv { bool ProgramRequiresExactSubgroupSize(const IR::Program& program); bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources, - ShaderStageInputInfo input_info, std::vector& spirv, std::string* error, - bool preserve_debug_values = false); + ShaderStageInputInfo input_info, std::vector& spirv, std::string* error); } // namespace Libs::Graphics::ShaderRecompiler::Spirv diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluOps.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluOps.cpp deleted file mode 100644 index 81216d32d..000000000 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluOps.cpp +++ /dev/null @@ -1,2550 +0,0 @@ -#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" - -namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { - -void EmitPerInvocationMask(EmitterState& state, const IR::Operand& dst, uint32_t value_bool) { - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.uint_type, value, value_bool, ConstantU32(state, 1), - ConstantU32(state, 0)}); - EmitStoreU32(state, dst, value); - EmitStoreU32(state, OffsetRegisterOperand(dst, 1), ConstantU32(state, 0)); -} - -uint32_t EmitLogicalBinary(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) { - uint32_t logical_opcode = OpLogicalNotEqual; - switch (opcode) { - case OpBitwiseAnd: logical_opcode = OpLogicalAnd; break; - case OpBitwiseOr: logical_opcode = OpLogicalOr; break; - default: break; - } - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({logical_opcode, state.bool_type, value, lhs, rhs}); - return value; -} - -void EmitAbsI32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto neg = state.builder.AllocateId(); - const auto cond = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpSNegate, state.uint_type, neg, src}); - state.builder.AddFunction({OpSLessThan, state.bool_type, cond, src, ConstantU32(state, 0)}); - state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, neg, src}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitUnaryU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, ret, src}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitUnaryU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - if (state.per_invocation_masks) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalNot, state.bool_type, ret, - EmitLaneMaskOperandActiveBool(state, inst.src[0])}); - EmitPerInvocationMask(state, inst.dst, ret); - return; - } - const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto ret_low = state.builder.AllocateId(); - const auto ret_high = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, ret_low, low}); - state.builder.AddFunction({opcode, state.uint_type, ret_high, high}); - EmitStoreU32(state, inst.dst, ret_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); -} - -void EmitFindLsbU32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto i32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.int_type, i32, state.glsl_std450, GlslFindILsb, src}); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, i32}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitFindMsbFromHighU32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto i32 = state.builder.AllocateId(); - const auto msb = state.builder.AllocateId(); - const auto pos = state.builder.AllocateId(); - const auto non_zero = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.int_type, i32, state.glsl_std450, GlslFindUMsb, src}); - state.builder.AddFunction({OpBitcast, state.uint_type, msb, i32}); - state.builder.AddFunction({OpISub, state.uint_type, pos, ConstantU32(state, 31), msb}); - state.builder.AddFunction({OpINotEqual, state.bool_type, non_zero, src, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, ret, non_zero, pos, ConstantU32(state, 0xffffffffu)}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitFindMsbFromHighU64(EmitterState& state, const IR::Instruction& inst) { - const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto high_i32 = state.builder.AllocateId(); - const auto high_msb = state.builder.AllocateId(); - const auto high_pos = state.builder.AllocateId(); - const auto high_non_zero = state.builder.AllocateId(); - const auto low_i32 = state.builder.AllocateId(); - const auto low_msb = state.builder.AllocateId(); - const auto low_pos = state.builder.AllocateId(); - const auto low_total = state.builder.AllocateId(); - const auto low_non_zero = state.builder.AllocateId(); - const auto low_ret = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.int_type, high_i32, state.glsl_std450, GlslFindUMsb, high}); - state.builder.AddFunction({OpBitcast, state.uint_type, high_msb, high_i32}); - state.builder.AddFunction( - {OpISub, state.uint_type, high_pos, ConstantU32(state, 31), high_msb}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, high_non_zero, high, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpExtInst, state.int_type, low_i32, state.glsl_std450, GlslFindUMsb, low}); - state.builder.AddFunction({OpBitcast, state.uint_type, low_msb, low_i32}); - state.builder.AddFunction({OpISub, state.uint_type, low_pos, ConstantU32(state, 31), low_msb}); - state.builder.AddFunction( - {OpIAdd, state.uint_type, low_total, low_pos, ConstantU32(state, 32)}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, low_non_zero, low, ConstantU32(state, 0)}); - state.builder.AddFunction({OpSelect, state.uint_type, low_ret, low_non_zero, low_total, - ConstantU32(state, 0xffffffffu)}); - state.builder.AddFunction({OpSelect, state.uint_type, ret, high_non_zero, high_pos, low_ret}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBitCountU64(EmitterState& state, const IR::Instruction& inst) { - const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto low_count = state.builder.AllocateId(); - const auto high_count = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitCount, state.uint_type, low_count, low}); - state.builder.AddFunction({OpBitCount, state.uint_type, high_count, high}); - state.builder.AddFunction({OpIAdd, state.uint_type, ret, low_count, high_count}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBitCountU32(EmitterState& state, const IR::Instruction& inst) { - const auto bits = EmitValueLoad(state, inst.src[0]); - const auto count = state.builder.AllocateId(); - state.builder.AddFunction({OpBitCount, state.uint_type, count, bits}); - EmitStoreU32(state, inst.dst, count); -} - -void EmitBitCountAddU32(EmitterState& state, const IR::Instruction& inst) { - const auto bits = EmitValueLoad(state, inst.src[0]); - const auto add = EmitValueLoad(state, inst.src[1]); - const auto count = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitCount, state.uint_type, count, bits}); - state.builder.AddFunction({OpIAdd, state.uint_type, ret, count, add}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBinaryU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - auto rhs = EmitValueLoad(state, inst.src[1]); - if (opcode == OpShiftLeftLogical || opcode == OpShiftRightLogical || - opcode == OpShiftRightArithmetic) { - rhs = EmitAndConstant(state, rhs, 31u); - } - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBinaryNotRhsU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto not_rhs = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpNot, state.uint_type, not_rhs, rhs}); - state.builder.AddFunction({opcode, state.uint_type, ret, lhs, not_rhs}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBinaryThenNotU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto tmp = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, tmp, lhs, rhs}); - state.builder.AddFunction({OpNot, state.uint_type, ret, tmp}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitMaskedBitCountU32(EmitterState& state, const IR::Instruction& inst, uint32_t exec_index) { - const auto bits = EmitValueLoad(state, inst.src[0]); - const auto add = EmitValueLoad(state, inst.src[1]); - const auto thread_mask = EmitThreadMaskBelowPartU32(state, exec_index); - const auto active_bits = state.builder.AllocateId(); - const auto count = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, active_bits, bits, thread_mask}); - state.builder.AddFunction({OpBitCount, state.uint_type, count, active_bits}); - state.builder.AddFunction({OpIAdd, state.uint_type, ret, count, add}); - EmitStoreU32(state, inst.dst, ret); -} - -uint32_t EmitMask5U32(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret, value, ConstantU32(state, 31)}); - return ret; -} - -void EmitChainedBinaryU32(EmitterState& state, const IR::Instruction& inst, uint32_t first_op, - uint32_t second_op, bool mask_first_rhs, bool mask_second_rhs) { - const auto src0 = EmitValueLoad(state, inst.src[0]); - const auto src1_raw = EmitValueLoad(state, inst.src[1]); - const auto src2_raw = EmitValueLoad(state, inst.src[2]); - const auto src1 = mask_first_rhs ? EmitMask5U32(state, src1_raw) : src1_raw; - const auto src2 = mask_second_rhs ? EmitMask5U32(state, src2_raw) : src2_raw; - const auto tmp = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({first_op, state.uint_type, tmp, src0, src1}); - state.builder.AddFunction({second_op, state.uint_type, ret, tmp, src2}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBinaryU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - if (state.per_invocation_masks) { - const auto ret = - EmitLogicalBinary(state, opcode, EmitLaneMaskOperandActiveBool(state, inst.src[0]), - EmitLaneMaskOperandActiveBool(state, inst.src[1])); - EmitPerInvocationMask(state, inst.dst, ret); - return; - } - const auto lhs_low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto lhs_high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto rhs_low = EmitSequentialValueLoad(state, inst.src[1], 0); - const auto rhs_high = EmitSequentialValueLoad(state, inst.src[1], 1); - const auto ret_low = state.builder.AllocateId(); - const auto ret_high = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, ret_low, lhs_low, rhs_low}); - state.builder.AddFunction({opcode, state.uint_type, ret_high, lhs_high, rhs_high}); - EmitStoreU32(state, inst.dst, ret_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); -} - -void EmitBinaryNotRhsU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - if (state.per_invocation_masks) { - const auto not_rhs = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalNot, state.bool_type, not_rhs, - EmitLaneMaskOperandActiveBool(state, inst.src[1])}); - const auto ret = EmitLogicalBinary( - state, opcode, EmitLaneMaskOperandActiveBool(state, inst.src[0]), not_rhs); - EmitPerInvocationMask(state, inst.dst, ret); - return; - } - const auto lhs_low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto lhs_high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto rhs_low = EmitSequentialValueLoad(state, inst.src[1], 0); - const auto rhs_high = EmitSequentialValueLoad(state, inst.src[1], 1); - const auto not_low = state.builder.AllocateId(); - const auto not_high = state.builder.AllocateId(); - const auto ret_low = state.builder.AllocateId(); - const auto ret_high = state.builder.AllocateId(); - state.builder.AddFunction({OpNot, state.uint_type, not_low, rhs_low}); - state.builder.AddFunction({OpNot, state.uint_type, not_high, rhs_high}); - state.builder.AddFunction({opcode, state.uint_type, ret_low, lhs_low, not_low}); - state.builder.AddFunction({opcode, state.uint_type, ret_high, lhs_high, not_high}); - EmitStoreU32(state, inst.dst, ret_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); -} - -void EmitBinaryThenNotU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - if (state.per_invocation_masks) { - const auto value = - EmitLogicalBinary(state, opcode, EmitLaneMaskOperandActiveBool(state, inst.src[0]), - EmitLaneMaskOperandActiveBool(state, inst.src[1])); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalNot, state.bool_type, ret, value}); - EmitPerInvocationMask(state, inst.dst, ret); - return; - } - const auto lhs_low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto lhs_high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto rhs_low = EmitSequentialValueLoad(state, inst.src[1], 0); - const auto rhs_high = EmitSequentialValueLoad(state, inst.src[1], 1); - const auto tmp_low = state.builder.AllocateId(); - const auto tmp_high = state.builder.AllocateId(); - const auto ret_low = state.builder.AllocateId(); - const auto ret_high = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, tmp_low, lhs_low, rhs_low}); - state.builder.AddFunction({opcode, state.uint_type, tmp_high, lhs_high, rhs_high}); - state.builder.AddFunction({OpNot, state.uint_type, ret_low, tmp_low}); - state.builder.AddFunction({OpNot, state.uint_type, ret_high, tmp_high}); - EmitStoreU32(state, inst.dst, ret_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); -} - -void EmitSelectU64(EmitterState& state, const IR::Instruction& inst) { - const auto cond = EmitConditionBool(state, inst.src[0]); - const auto true_low = EmitSequentialValueLoad(state, inst.src[1], 0); - const auto true_high = EmitSequentialValueLoad(state, inst.src[1], 1); - const auto false_low = EmitSequentialValueLoad(state, inst.src[2], 0); - const auto false_high = EmitSequentialValueLoad(state, inst.src[2], 1); - const auto ret_low = state.builder.AllocateId(); - const auto ret_high = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.uint_type, ret_low, cond, true_low, false_low}); - state.builder.AddFunction({OpSelect, state.uint_type, ret_high, cond, true_high, false_high}); - EmitStoreU32(state, inst.dst, ret_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); -} - -uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value, - uint32_t false_value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, true_value, false_value}); - return ret; -} - -void EmitStoreSccBool(EmitterState& state, uint32_t cond) { - const auto value = - EmitSelectValueU32(state, cond, ConstantU32(state, 1), ConstantU32(state, 0)); - EmitStoreU32(state, SccOperand(), value); -} - -uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask); - -void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift, - uint32_t& out_low, uint32_t& out_high) { - const auto amount = EmitAndConstant(state, shift, 63u); - const auto dword_shift = EmitAndConstant(state, amount, 31u); - const auto less32 = state.builder.AllocateId(); - const auto non_zero = state.builder.AllocateId(); - const auto carry_count_base = state.builder.AllocateId(); - const auto carry_count = state.builder.AllocateId(); - const auto low_part = state.builder.AllocateId(); - const auto high_part = state.builder.AllocateId(); - const auto carry_part = state.builder.AllocateId(); - const auto carry = state.builder.AllocateId(); - const auto high_less = state.builder.AllocateId(); - const auto high_ge = state.builder.AllocateId(); - state.builder.AddFunction( - {OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)}); - state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, low_part, low, dword_shift}); - state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_part, high, dword_shift}); - state.builder.AddFunction({OpShiftRightLogical, state.uint_type, carry_part, low, carry_count}); - state.builder.AddFunction( - {OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, high_less, high_part, carry}); - state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_ge, low, dword_shift}); - out_low = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0)); - out_high = EmitSelectValueU32(state, less32, high_less, high_ge); -} - -void EmitShiftLeftLogicalU64(EmitterState& state, const IR::Instruction& inst) { - const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto shift = EmitValueLoad(state, inst.src[1]); - uint32_t ret_low = 0; - uint32_t ret_high = 0; - EmitShiftLeftLogicalU64Values(state, low, high, shift, ret_low, ret_high); - EmitStoreU32(state, inst.dst, ret_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); -} - -void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, - uint32_t shift, uint32_t& out_low, uint32_t& out_high) { - const auto amount = EmitAndConstant(state, shift, 63u); - const auto dword_shift = EmitAndConstant(state, amount, 31u); - const auto less32 = state.builder.AllocateId(); - const auto non_zero = state.builder.AllocateId(); - const auto carry_count_base = state.builder.AllocateId(); - const auto carry_count = state.builder.AllocateId(); - const auto low_part = state.builder.AllocateId(); - const auto high_part = state.builder.AllocateId(); - const auto carry_part = state.builder.AllocateId(); - const auto carry = state.builder.AllocateId(); - const auto low_less = state.builder.AllocateId(); - const auto low_ge = state.builder.AllocateId(); - state.builder.AddFunction( - {OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)}); - state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_part, low, dword_shift}); - state.builder.AddFunction({OpShiftRightLogical, state.uint_type, high_part, high, dword_shift}); - state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, carry_part, high, carry_count}); - state.builder.AddFunction( - {OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, low_less, low_part, carry}); - state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_ge, high, dword_shift}); - out_low = EmitSelectValueU32(state, less32, low_less, low_ge); - out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0)); -} - -void EmitShiftRightLogicalU64(EmitterState& state, const IR::Instruction& inst) { - const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto shift = EmitValueLoad(state, inst.src[1]); - uint32_t ret_low = 0; - uint32_t ret_high = 0; - EmitShiftRightLogicalU64Values(state, low, high, shift, ret_low, ret_high); - EmitStoreU32(state, inst.dst, ret_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); -} - -void EmitAdd3U32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto add = EmitValueLoad(state, inst.src[2]); - const auto tmp = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpIAdd, state.uint_type, tmp, lhs, rhs}); - state.builder.AddFunction({OpIAdd, state.uint_type, ret, tmp, add}); - EmitStoreU32(state, inst.dst, ret); -} - -uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, ret, value, ConstantU32(state, mask)}); - return ret; -} - -void EmitShiftU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 31u); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBitClearU32(EmitterState& state, const IR::Instruction& inst) { - const auto value = EmitValueLoad(state, inst.src[0]); - const auto bit = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 31u); - const auto mask = state.builder.AllocateId(); - const auto inv = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, mask, ConstantU32(state, 1), bit}); - state.builder.AddFunction({OpNot, state.uint_type, inv, mask}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret, value, inv}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBitSetU32(EmitterState& state, const IR::Instruction& inst) { - const auto value = EmitValueLoad(state, inst.src[0]); - const auto bit = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 31u); - const auto mask = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, mask, ConstantU32(state, 1), bit}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, ret, value, mask}); - EmitStoreU32(state, inst.dst, ret); -} - -uint32_t EmitNegateU32(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpSNegate, state.uint_type, ret, value}); - return ret; -} - -uint32_t EmitSignExtendLow16U32(EmitterState& state, uint32_t value) { - const auto signed_value = state.builder.AllocateId(); - const auto extracted = state.builder.AllocateId(); - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, signed_value, value}); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_value, - ConstantU32(state, 0), ConstantU32(state, 16)}); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, extracted}); - return bits; -} - -uint32_t EmitU16LaneBits(EmitterState& state, const IR::Operand& operand, bool high_lane, - bool sign_extend) { - const auto raw = EmitValueLoad(state, operand); - const auto selected_lane = high_lane ? operand.op_sel_hi : operand.op_sel; - const auto lane = selected_lane ? 1u : 0u; - auto bits = lane != 0u ? EmitShiftRightConstant(state, raw, 16) : raw; - bits = EmitAndConstant(state, bits, 0xffffu); - if (high_lane ? operand.negate_hi : operand.negate) { - bits = EmitAndConstant(state, EmitNegateU32(state, bits), 0xffffu); - } - return sign_extend ? EmitSignExtendLow16U32(state, bits) : bits; -} - -void EmitShiftU16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode, - bool arithmetic) { - const auto lhs = EmitU16LaneBits(state, inst.src[0], false, arithmetic); - const auto rhs = EmitAndConstant(state, EmitU16LaneBits(state, inst.src[1], false), 0xfu); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs}); - EmitStoreU32(state, inst.dst, EmitAndConstant(state, ret, 0xffffu)); -} - -void EmitBinaryU16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitU16LaneBits(state, inst.src[0], false); - const auto rhs = EmitU16LaneBits(state, inst.src[1], false); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs}); - EmitStoreU32(state, inst.dst, EmitAndConstant(state, ret, 0xffffu)); -} - -void EmitMinMaxU16(EmitterState& state, const IR::Instruction& inst, bool signed_value, - bool max_value) { - const auto lhs = EmitU16LaneBits(state, inst.src[0], false, signed_value); - const auto rhs = EmitU16LaneBits(state, inst.src[1], false, signed_value); - const auto cond = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - uint32_t op = 0; - if (signed_value) { - op = max_value ? OpSGreaterThan : OpSLessThan; - } else { - op = max_value ? OpUGreaterThan : OpULessThan; - } - state.builder.AddFunction({op, state.bool_type, cond, lhs, rhs}); - state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs}); - EmitStoreU32(state, inst.dst, EmitAndConstant(state, ret, 0xffffu)); -} - -uint32_t EmitXorConstant(EmitterState& state, uint32_t value, uint32_t mask) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseXor, state.uint_type, ret, value, ConstantU32(state, mask)}); - return ret; -} - -AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs, - uint32_t carry_in) { - const auto pair0 = state.builder.AllocateId(); - const auto sum0 = state.builder.AllocateId(); - const auto carry0 = state.builder.AllocateId(); - const auto pair1 = state.builder.AllocateId(); - const auto sum1 = state.builder.AllocateId(); - const auto carry1 = state.builder.AllocateId(); - const auto carry = state.builder.AllocateId(); - state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair0, lhs, rhs}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum0, pair0, 0}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry0, pair0, 1}); - state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair1, sum0, carry_in}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum1, pair1, 0}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry1, pair1, 1}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, carry, carry0, carry1}); - return {sum1, carry}; -} - -void EmitLaneMaskPairFromBool(EmitterState& state, const IR::Operand& dst, uint32_t value_bool) { - const auto write_mask = state.builder.AllocateId(); - state.builder.AddFunction( - {OpLogicalAnd, state.bool_type, write_mask, value_bool, EmitExecActiveBool(state)}); - if (state.per_invocation_masks) { - EmitPerInvocationMask(state, dst, write_mask); - return; - } - const auto ballot = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot, - ConstantU32(state, ScopeSubgroup), write_mask}); - for (uint32_t part = 0; part < 2u; part++) { - uint32_t result = ConstantU32(state, 0); - if (state.wave_size != 32u || part == 0u) { - result = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, result, ballot, part}); - } - EmitStoreU32(state, OffsetRegisterOperand(dst, part), result); - } -} - -void EmitIAddCarryU32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto carry_in_active = EmitLaneMaskOperandActiveBool(state, inst.src[2]); - const auto carry_in = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.uint_type, carry_in, carry_in_active, - ConstantU32(state, 1), ConstantU32(state, 0)}); - const auto result = EmitAddCarryValues(state, lhs, rhs, carry_in); - const auto carry_bool = state.builder.AllocateId(); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, carry_bool, result.carry, ConstantU32(state, 0)}); - EmitStoreU32(state, inst.dst, result.sum); - EmitLaneMaskPairFromBool(state, inst.dst2, carry_bool); -} - -void EmitISubBorrowU32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto result = state.builder.AllocateId(); - const auto borrow = state.builder.AllocateId(); - state.builder.AddFunction({OpISub, state.uint_type, result, lhs, rhs}); - state.builder.AddFunction({OpUGreaterThan, state.bool_type, borrow, rhs, lhs}); - EmitStoreU32(state, inst.dst, result); - EmitLaneMaskPairFromBool(state, inst.dst2, borrow); -} - -void EmitScalarAddCarryU32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto carry_in = EmitAndConstant(state, EmitValueLoad(state, inst.src[2]), 1u); - const auto result = EmitAddCarryValues(state, lhs, rhs, carry_in); - EmitStoreU32(state, inst.dst, result.sum); - EmitStoreU32(state, SccOperand(), result.carry); -} - -void EmitScalarSubBorrowU32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto result = state.builder.AllocateId(); - const auto borrow = state.builder.AllocateId(); - state.builder.AddFunction({OpISub, state.uint_type, result, lhs, rhs}); - state.builder.AddFunction({OpUGreaterThan, state.bool_type, borrow, rhs, lhs}); - EmitStoreU32(state, inst.dst, result); - EmitStoreSccBool(state, borrow); -} - -void EmitScalarSubBorrowCarryU32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto borrow_in = EmitAndConstant(state, EmitValueLoad(state, inst.src[2]), 1u); - const auto partial = state.builder.AllocateId(); - const auto result = state.builder.AllocateId(); - const auto borrow0 = state.builder.AllocateId(); - const auto borrow1 = state.builder.AllocateId(); - state.builder.AddFunction({OpISub, state.uint_type, partial, lhs, rhs}); - state.builder.AddFunction({OpUGreaterThan, state.bool_type, borrow0, rhs, lhs}); - state.builder.AddFunction({OpISub, state.uint_type, result, partial, borrow_in}); - state.builder.AddFunction({OpUGreaterThan, state.bool_type, borrow1, borrow_in, partial}); - EmitStoreU32(state, inst.dst, result); - EmitStoreSccBool(state, EmitLogicalOrBool(state, borrow0, borrow1)); -} - -void EmitUMadU64U32(EmitterState& state, const IR::Instruction& inst) { - const auto src0 = EmitValueLoad(state, inst.src[0]); - const auto src1 = EmitValueLoad(state, inst.src[1]); - const auto add_low = EmitSequentialValueLoad(state, inst.src[2], 0); - const auto add_high = EmitSequentialValueLoad(state, inst.src[2], 1); - const auto mul_pair = state.builder.AllocateId(); - const auto mul_low = state.builder.AllocateId(); - const auto mul_high = state.builder.AllocateId(); - const auto low_pair = state.builder.AllocateId(); - const auto result_low = state.builder.AllocateId(); - const auto carry_low = state.builder.AllocateId(); - const auto high_pair = state.builder.AllocateId(); - const auto high_sum = state.builder.AllocateId(); - const auto carry_high0 = state.builder.AllocateId(); - const auto carry_pair = state.builder.AllocateId(); - const auto result_high = state.builder.AllocateId(); - const auto carry_high1 = state.builder.AllocateId(); - const auto carry = state.builder.AllocateId(); - const auto carry_bool = state.builder.AllocateId(); - - state.builder.AddFunction({OpUMulExtended, state.uint_pair_type, mul_pair, src0, src1}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, mul_low, mul_pair, 0}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, mul_high, mul_pair, 1}); - state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, low_pair, mul_low, add_low}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, result_low, low_pair, 0}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry_low, low_pair, 1}); - state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, high_pair, mul_high, add_high}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, high_sum, high_pair, 0}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry_high0, high_pair, 1}); - state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, carry_pair, high_sum, carry_low}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, result_high, carry_pair, 0}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry_high1, carry_pair, 1}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, carry, carry_high0, carry_high1}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, carry_bool, carry, ConstantU32(state, 0)}); - - EmitStoreU32(state, inst.dst, result_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), result_high); - EmitLaneMaskPairFromBool(state, inst.dst2, carry_bool); -} - -void EmitScalarSignedOverflowI32(EmitterState& state, const IR::Instruction& inst, bool subtract) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto result = state.builder.AllocateId(); - const auto lhs_sign = state.builder.AllocateId(); - const auto rhs_sign = state.builder.AllocateId(); - const auto result_sign = state.builder.AllocateId(); - const auto sign_match = state.builder.AllocateId(); - const auto sign_change = state.builder.AllocateId(); - const auto overflow = state.builder.AllocateId(); - state.builder.AddFunction({subtract ? OpISub : OpIAdd, state.uint_type, result, lhs, rhs}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, lhs_sign, lhs, ConstantU32(state, 31)}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, rhs_sign, rhs, ConstantU32(state, 31)}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, result_sign, result, ConstantU32(state, 31)}); - state.builder.AddFunction( - {subtract ? OpINotEqual : OpIEqual, state.bool_type, sign_match, lhs_sign, rhs_sign}); - state.builder.AddFunction({OpINotEqual, state.bool_type, sign_change, lhs_sign, result_sign}); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, overflow, sign_match, sign_change}); - EmitStoreU32(state, inst.dst, result); - EmitStoreSccBool(state, overflow); -} - -void EmitScalarShiftLeftAddCarryU32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[2]); - const auto shift = inst.src[1].kind == IR::OperandKind::ImmediateU32 ? inst.src[1].imm : 0u; - const auto shifted = state.builder.AllocateId(); - const auto pair = state.builder.AllocateId(); - const auto result = state.builder.AllocateId(); - const auto add_carry = state.builder.AllocateId(); - const auto shifted_out = state.builder.AllocateId(); - const auto shift_carry = state.builder.AllocateId(); - const auto add_carry_bool = state.builder.AllocateId(); - const auto carry = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, shifted, lhs, ConstantU32(state, shift)}); - state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair, shifted, rhs}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, result, pair, 0}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, add_carry, pair, 1}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, shifted_out, lhs, ConstantU32(state, 32u - shift)}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, shift_carry, shifted_out, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, add_carry_bool, add_carry, ConstantU32(state, 0)}); - state.builder.AddFunction({OpLogicalOr, state.bool_type, carry, shift_carry, add_carry_bool}); - EmitStoreU32(state, inst.dst, result); - EmitStoreSccBool(state, carry); -} - -void EmitMulU24U32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[0]), 0x00ffffffu); - const auto rhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 0x00ffffffu); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpIMul, state.uint_type, ret, lhs, rhs}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitMulI24U32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs_bits = EmitValueLoad(state, inst.src[0]); - const auto rhs_bits = EmitValueLoad(state, inst.src[1]); - const auto lhs_i32 = state.builder.AllocateId(); - const auto rhs_i32 = state.builder.AllocateId(); - const auto lhs_i24 = state.builder.AllocateId(); - const auto rhs_i24 = state.builder.AllocateId(); - const auto mul = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, lhs_i32, lhs_bits}); - state.builder.AddFunction({OpBitcast, state.int_type, rhs_i32, rhs_bits}); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, lhs_i24, lhs_i32, - ConstantU32(state, 0), ConstantU32(state, 24)}); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, rhs_i24, rhs_i32, - ConstantU32(state, 0), ConstantU32(state, 24)}); - state.builder.AddFunction({OpIMul, state.int_type, mul, lhs_i24, rhs_i24}); - state.builder.AddFunction({OpBitcast, state.uint_type, ret, mul}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitMulHighU32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto full = state.builder.AllocateId(); - const auto high = state.builder.AllocateId(); - state.builder.AddFunction({OpUMulExtended, state.uint_pair_type, full, lhs, rhs}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, full, 1}); - EmitStoreU32(state, inst.dst, high); -} - -void EmitMulHighI32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs_bits = EmitValueLoad(state, inst.src[0]); - const auto rhs_bits = EmitValueLoad(state, inst.src[1]); - const auto lhs = state.builder.AllocateId(); - const auto rhs = state.builder.AllocateId(); - const auto full = state.builder.AllocateId(); - const auto high_i32 = state.builder.AllocateId(); - const auto high_u32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, lhs, lhs_bits}); - state.builder.AddFunction({OpBitcast, state.int_type, rhs, rhs_bits}); - state.builder.AddFunction({OpSMulExtended, state.int_pair_type, full, lhs, rhs}); - state.builder.AddFunction({OpCompositeExtract, state.int_type, high_i32, full, 1}); - state.builder.AddFunction({OpBitcast, state.uint_type, high_u32, high_i32}); - EmitStoreU32(state, inst.dst, high_u32); -} - -void EmitMadU32U24(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[0]), 0x00ffffffu); - const auto rhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 0x00ffffffu); - const auto add = EmitValueLoad(state, inst.src[2]); - const auto mul = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpIMul, state.uint_type, mul, lhs, rhs}); - state.builder.AddFunction({OpIAdd, state.uint_type, ret, mul, add}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitMadI32I24(EmitterState& state, const IR::Instruction& inst) { - const auto lhs_bits = EmitValueLoad(state, inst.src[0]); - const auto rhs_bits = EmitValueLoad(state, inst.src[1]); - const auto add_bits = EmitValueLoad(state, inst.src[2]); - const auto lhs_i32 = state.builder.AllocateId(); - const auto rhs_i32 = state.builder.AllocateId(); - const auto add_i32 = state.builder.AllocateId(); - const auto lhs_i24 = state.builder.AllocateId(); - const auto rhs_i24 = state.builder.AllocateId(); - const auto mul = state.builder.AllocateId(); - const auto add = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, lhs_i32, lhs_bits}); - state.builder.AddFunction({OpBitcast, state.int_type, rhs_i32, rhs_bits}); - state.builder.AddFunction({OpBitcast, state.int_type, add_i32, add_bits}); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, lhs_i24, lhs_i32, - ConstantU32(state, 0), ConstantU32(state, 24)}); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, rhs_i24, rhs_i32, - ConstantU32(state, 0), ConstantU32(state, 24)}); - state.builder.AddFunction({OpIMul, state.int_type, mul, lhs_i24, rhs_i24}); - state.builder.AddFunction({OpIAdd, state.int_type, add, mul, add_i32}); - state.builder.AddFunction({OpBitcast, state.uint_type, ret, add}); - EmitStoreU32(state, inst.dst, ret); -} - -uint32_t EmitShiftLeftConstant(EmitterState& state, uint32_t value, uint32_t shift) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, ret, value, ConstantU32(state, shift)}); - return ret; -} - -uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, ret, value, ConstantU32(state, shift)}); - return ret; -} - -uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, ret, lhs, rhs}); - return ret; -} - -uint32_t EmitBitReplicateHalfU32(EmitterState& state, uint32_t value) { - auto ret = EmitOrU32(state, value, EmitShiftLeftConstant(state, value, 8)); - ret = EmitAndConstant(state, ret, 0x00ff00ffu); - ret = EmitOrU32(state, ret, EmitShiftLeftConstant(state, ret, 4)); - ret = EmitAndConstant(state, ret, 0x0f0f0f0fu); - ret = EmitOrU32(state, ret, EmitShiftLeftConstant(state, ret, 2)); - ret = EmitAndConstant(state, ret, 0x33333333u); - ret = EmitOrU32(state, ret, EmitShiftLeftConstant(state, ret, 1)); - ret = EmitAndConstant(state, ret, 0x55555555u); - return EmitOrU32(state, ret, EmitShiftLeftConstant(state, ret, 1)); -} - -void EmitBitReplicateB64B32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto low = EmitBitReplicateHalfU32(state, EmitAndConstant(state, src, 0x0000ffffu)); - const auto high = EmitBitReplicateHalfU32(state, EmitShiftRightConstant(state, src, 16)); - EmitStoreU32(state, inst.dst, low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high); -} - -uint32_t EmitBitFieldExtractConstant(EmitterState& state, uint32_t value, uint32_t offset, - uint32_t count) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, ret, value, - ConstantU32(state, offset), ConstantU32(state, count)}); - return ret; -} - -uint32_t EmitRightAlignedMaskU32(EmitterState& state, uint32_t count) { - const auto shifted_one = state.builder.AllocateId(); - const auto mask = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, shifted_one, ConstantU32(state, 1), count}); - state.builder.AddFunction({OpISub, state.uint_type, mask, shifted_one, ConstantU32(state, 1)}); - return mask; -} - -void EmitBitFieldMaskU32(EmitterState& state, const IR::Instruction& inst) { - const auto count = EmitAndConstant(state, EmitValueLoad(state, inst.src[0]), 31u); - const auto offset = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 31u); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitFieldInsert, state.uint_type, ret, ConstantU32(state, 0), - ConstantU32(state, 0xffffffffu), offset, count}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitRightAlignedMaskU64(EmitterState& state, uint32_t count, uint32_t& out_low, - uint32_t& out_high) { - const auto count_lt32 = state.builder.AllocateId(); - const auto count_gt32 = state.builder.AllocateId(); - const auto low_count = state.builder.AllocateId(); - const auto high_base = state.builder.AllocateId(); - const auto high_count = state.builder.AllocateId(); - state.builder.AddFunction( - {OpULessThan, state.bool_type, count_lt32, count, ConstantU32(state, 32)}); - state.builder.AddFunction( - {OpUGreaterThan, state.bool_type, count_gt32, count, ConstantU32(state, 32)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, low_count, count_lt32, count, ConstantU32(state, 32)}); - state.builder.AddFunction({OpISub, state.uint_type, high_base, count, ConstantU32(state, 32)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, high_count, count_gt32, high_base, ConstantU32(state, 0)}); - out_low = state.builder.AllocateId(); - out_high = state.builder.AllocateId(); - state.builder.AddFunction({OpBitFieldInsert, state.uint_type, out_low, ConstantU32(state, 0), - ConstantU32(state, 0xffffffffu), ConstantU32(state, 0), low_count}); - state.builder.AddFunction({OpBitFieldInsert, state.uint_type, out_high, ConstantU32(state, 0), - ConstantU32(state, 0xffffffffu), ConstantU32(state, 0), high_count}); -} - -void EmitBitFieldMaskU64(EmitterState& state, const IR::Instruction& inst) { - const auto count = EmitAndConstant(state, EmitValueLoad(state, inst.src[0]), 63u); - if (state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register && - IsMaskRegisterFile(inst.dst.reg.file)) { - const auto active = state.builder.AllocateId(); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, active, count, ConstantU32(state, 0)}); - EmitPerInvocationMask(state, inst.dst, active); - return; - } - const auto offset = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 63u); - uint32_t low_mask = 0; - uint32_t high_mask = 0; - uint32_t shifted_low = 0; - uint32_t shifted_high = 0; - EmitRightAlignedMaskU64(state, count, low_mask, high_mask); - EmitShiftLeftLogicalU64Values(state, low_mask, high_mask, offset, shifted_low, shifted_high); - EmitStoreU32(state, inst.dst, shifted_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), shifted_high); -} - -void EmitBitFieldExtractU64(EmitterState& state, const IR::Instruction& inst) { - const auto src_low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto src_high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto field = EmitValueLoad(state, inst.src[1]); - const auto offset = EmitBitFieldExtractConstant(state, field, 0, 6); - const auto raw_count = EmitBitFieldExtractConstant(state, field, 16, 7); - const auto available = state.builder.AllocateId(); - const auto use_raw = state.builder.AllocateId(); - const auto count = state.builder.AllocateId(); - uint32_t shifted_low = 0; - uint32_t shifted_high = 0; - uint32_t mask_low = 0; - uint32_t mask_high = 0; - const auto ret_low = state.builder.AllocateId(); - const auto ret_high = state.builder.AllocateId(); - state.builder.AddFunction({OpISub, state.uint_type, available, ConstantU32(state, 64), offset}); - state.builder.AddFunction({OpULessThan, state.bool_type, use_raw, raw_count, available}); - state.builder.AddFunction({OpSelect, state.uint_type, count, use_raw, raw_count, available}); - EmitShiftRightLogicalU64Values(state, src_low, src_high, offset, shifted_low, shifted_high); - EmitRightAlignedMaskU64(state, count, mask_low, mask_high); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret_low, shifted_low, mask_low}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret_high, shifted_high, mask_high}); - EmitStoreU32(state, inst.dst, ret_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); -} - -void EmitBitFieldExtractU32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto field = EmitValueLoad(state, inst.src[1]); - const auto offset = EmitBitFieldExtractConstant(state, field, 0, 5); - const auto count = EmitBitFieldExtractConstant(state, field, 16, 7); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, ret, src, offset, count}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBitFieldExtract3U32(EmitterState& state, const IR::Instruction& inst, bool signed_value) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto offset = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 31u); - const auto count = EmitAndConstant(state, EmitValueLoad(state, inst.src[2]), 31u); - const auto ret = state.builder.AllocateId(); - if (signed_value) { - const auto shifted = state.builder.AllocateId(); - const auto mask = EmitRightAlignedMaskU32(state, count); - state.builder.AddFunction({OpShiftRightArithmetic, state.uint_type, shifted, src, offset}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret, shifted, mask}); - EmitStoreU32(state, inst.dst, ret); - return; - } - state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, ret, src, offset, count}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBitFieldInsertSelectU32(EmitterState& state, const IR::Instruction& inst) { - const auto mask = EmitValueLoad(state, inst.src[0]); - const auto insert = EmitValueLoad(state, inst.src[1]); - const auto base = EmitValueLoad(state, inst.src[2]); - const auto not_mask = state.builder.AllocateId(); - const auto selected = state.builder.AllocateId(); - const auto preserved = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpNot, state.uint_type, not_mask, mask}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, selected, mask, insert}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, preserved, not_mask, base}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, ret, selected, preserved}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitAlignBitU32(EmitterState& state, const IR::Instruction& inst) { - const auto hi = EmitValueLoad(state, inst.src[0]); - const auto lo = EmitValueLoad(state, inst.src[1]); - const auto shift = EmitAndConstant(state, EmitValueLoad(state, inst.src[2]), 31u); - const auto lo_part = state.builder.AllocateId(); - const auto hi_shift_raw = state.builder.AllocateId(); - const auto hi_shift = state.builder.AllocateId(); - const auto hi_part_raw = state.builder.AllocateId(); - const auto non_zero = state.builder.AllocateId(); - const auto hi_part = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpShiftRightLogical, state.uint_type, lo_part, lo, shift}); - state.builder.AddFunction( - {OpISub, state.uint_type, hi_shift_raw, ConstantU32(state, 32), shift}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, hi_shift, hi_shift_raw, ConstantU32(state, 31)}); - state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, hi_part_raw, hi, hi_shift}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, non_zero, shift, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, hi_part, non_zero, hi_part_raw, ConstantU32(state, 0)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, ret, lo_part, hi_part}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitSelectU32(EmitterState& state, const IR::Instruction& inst) { - const auto cond = EmitConditionBool(state, inst.src[0]); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, - EmitValueLoad(state, inst.src[1]), - EmitValueLoad(state, inst.src[2])}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitSelectMaskU32(EmitterState& state, const IR::Instruction& inst) { - const auto cond = EmitLaneMaskOperandActiveBool(state, inst.src[0]); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, - EmitValueLoad(state, inst.src[1]), - EmitValueLoad(state, inst.src[2])}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitSelectF32Bits(EmitterState& state, const IR::Instruction& inst) { - const auto cond = EmitConditionBool(state, inst.src[0]); - const auto selected = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.float_type, selected, cond, - EmitFloatLoad(state, inst.src[1]), - EmitFloatLoad(state, inst.src[2])}); - state.builder.AddFunction({OpBitcast, state.uint_type, ret, selected}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitSelectMaskF32Bits(EmitterState& state, const IR::Instruction& inst) { - const auto cond = EmitLaneMaskOperandActiveBool(state, inst.src[0]); - const auto selected = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.float_type, selected, cond, - EmitFloatLoad(state, inst.src[1]), - EmitFloatLoad(state, inst.src[2])}); - state.builder.AddFunction({OpBitcast, state.uint_type, ret, selected}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitPackU16(EmitterState& state, const IR::Instruction& inst, bool high_src0, bool high_src1) { - const auto src0 = EmitValueLoad(state, inst.src[0]); - const auto src1 = EmitValueLoad(state, inst.src[1]); - const auto lo = high_src0 ? EmitBitFieldExtractConstant(state, src0, 16, 16) - : EmitAndConstant(state, src0, 0xffffu); - const auto hi = high_src1 ? EmitBitFieldExtractConstant(state, src1, 16, 16) - : EmitAndConstant(state, src1, 0xffffu); - const auto shifted = EmitShiftLeftConstant(state, hi, 16); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, ret, lo, shifted}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitPackU8F32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitFloatLoad(state, inst.src[0]); - const auto byte_index = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 3u); - const auto base = EmitValueLoad(state, inst.src[2]); - - const auto truncated = EmitTruncF32Value(state, src); - const auto below_zero = state.builder.AllocateId(); - const auto above_max = state.builder.AllocateId(); - const auto safe_low = state.builder.AllocateId(); - const auto safe_value = state.builder.AllocateId(); - const auto converted = state.builder.AllocateId(); - state.builder.AddFunction( - {OpFOrdLessThanEqual, state.bool_type, below_zero, src, ConstantF32(state, 0)}); - state.builder.AddFunction( - {OpFOrdGreaterThanEqual, state.bool_type, above_max, src, ConstantF32(state, 0x437f0000u)}); - const auto zero_or_nan = EmitLogicalOrBool(state, EmitClassifyF32(state, src).nan, below_zero); - state.builder.AddFunction( - {OpSelect, state.float_type, safe_low, zero_or_nan, ConstantF32(state, 0), truncated}); - state.builder.AddFunction({OpSelect, state.float_type, safe_value, above_max, - ConstantF32(state, 0x437f0000u), safe_low}); - state.builder.AddFunction({OpConvertFToU, state.uint_type, converted, safe_value}); - const auto byte_value = EmitAndConstant(state, converted, 0xffu); - - const auto shift = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, shift, byte_index, ConstantU32(state, 3)}); - const auto byte_mask = state.builder.AllocateId(); - const auto inv_mask = state.builder.AllocateId(); - const auto cleared = state.builder.AllocateId(); - const auto shifted = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, byte_mask, ConstantU32(state, 0xffu), shift}); - state.builder.AddFunction({OpNot, state.uint_type, inv_mask, byte_mask}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, cleared, base, inv_mask}); - state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, shifted, byte_value, shift}); - EmitStoreU32(state, inst.dst, EmitOrU32(state, cleared, shifted)); -} - -void EmitBinaryF32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitFloatLoad(state, inst.src[0]); - const auto rhs = EmitFloatLoad(state, inst.src[1]); - const auto f32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.float_type, f32, lhs, rhs}); - const auto result = ApplyResultModifiersF32(state, f32, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, result}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitLdexpF32(EmitterState& state, const IR::Instruction& inst) { - const auto value = EmitFloatLoad(state, inst.src[0]); - const auto exp_bits = EmitValueLoad(state, inst.src[1]); - const auto exp_i32 = state.builder.AllocateId(); - const auto ret_f32 = state.builder.AllocateId(); - const auto ret_u32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, exp_i32, exp_bits}); - state.builder.AddFunction( - {OpExtInst, state.float_type, ret_f32, state.glsl_std450, GlslLdexp, value, exp_i32}); - const auto result = ApplyResultModifiersF32(state, ret_f32, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, ret_u32, result}); - EmitStoreU32(state, inst.dst, ret_u32); -} - -uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value, - uint32_t constant) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)}); - return ret; -} - -uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpISub, state.uint_type, ret, ConstantU32(state, constant), value}); - return ret; -} - -uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) { - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, f32}); - - const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u); - const auto exp = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu); - const auto mant = EmitAndConstant(state, bits, 0x007fffffu); - - const auto half_exp = state.builder.AllocateId(); - const auto normal_exp = state.builder.AllocateId(); - const auto normal_mant = EmitShiftRightConstant(state, mant, 13); - const auto normal_payload = state.builder.AllocateId(); - const auto normal = state.builder.AllocateId(); - state.builder.AddFunction({OpISub, state.uint_type, half_exp, exp, ConstantU32(state, 112)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, normal_exp, half_exp, ConstantU32(state, 10)}); - state.builder.AddFunction( - {OpBitwiseOr, state.uint_type, normal_payload, normal_exp, normal_mant}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, normal, sign, normal_payload}); - - const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u)); - const auto raw_sub_shift = EmitSubConstantMinusU32(state, 126, exp); - const auto exp_lt_103 = EmitCompareU32Constant(state, OpULessThan, exp, 103); - const auto exp_gt_112 = EmitCompareU32Constant(state, OpUGreaterThan, exp, 112); - const auto sub_shift_low = - EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift); - const auto sub_shift = - EmitSelectValueU32(state, exp_gt_112, ConstantU32(state, 14), sub_shift_low); - const auto sub_mant = state.builder.AllocateId(); - const auto subnormal = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, sub_mant, mant_with_hidden, sub_shift}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, subnormal, sign, sub_mant}); - - const auto nan_payload = - EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u)); - const auto nan = - EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload)); - const auto inf = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u)); - const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu)); - const auto mant_zero = EmitCompareU32Constant(state, OpIEqual, mant, 0); - const auto special = EmitSelectValueU32(state, mant_zero, inf, nan); - - const auto exp_le_112 = EmitCompareU32Constant(state, OpULessThanEqual, exp, 112); - const auto exp_ge_143 = EmitCompareU32Constant(state, OpUGreaterThanEqual, exp, 143); - const auto exp_eq_255 = EmitCompareU32Constant(state, OpIEqual, exp, 255); - const auto finite0 = EmitSelectValueU32(state, exp_le_112, subnormal, normal); - const auto finite1 = EmitSelectValueU32(state, exp_lt_103, sign, finite0); - const auto finite2 = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1); - return EmitAndConstant(state, EmitSelectValueU32(state, exp_eq_255, special, finite2), 0xffffu); -} - -void EmitPackF32ToF16Rtz(EmitterState& state, const IR::Instruction& inst) { - const auto lo = EmitF32ToF16RtzBits(state, EmitFloatLoad(state, inst.src[0])); - const auto hi = EmitF32ToF16RtzBits(state, EmitFloatLoad(state, inst.src[1])); - const auto shifted = EmitShiftLeftConstant(state, hi, 16); - const auto ret = EmitOrU32(state, lo, shifted); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitPackNormalizedF32(EmitterState& state, const IR::Instruction& inst, uint32_t ext_inst) { - const auto src0 = EmitFloatLoad(state, inst.src[0]); - const auto src1 = EmitFloatLoad(state, inst.src[1]); - const auto pair = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, pair, src0, src1}); - state.builder.AddFunction({OpExtInst, state.uint_type, ret, state.glsl_std450, ext_inst, pair}); - EmitStoreU32(state, inst.dst, ret); -} - -uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) { - const auto cond = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {max_value ? OpUGreaterThan : OpULessThan, state.bool_type, cond, lhs, rhs}); - state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs}); - return ret; -} - -void EmitMinMaxU32(EmitterState& state, const IR::Instruction& inst, bool max_value) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto ret = EmitMinMaxU32Value(state, lhs, rhs, max_value); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitSadU32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto add = EmitValueLoad(state, inst.src[2]); - const auto high = EmitMinMaxU32Value(state, lhs, rhs, true); - const auto low = EmitMinMaxU32Value(state, lhs, rhs, false); - const auto diff = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpISub, state.uint_type, diff, high, low}); - state.builder.AddFunction({OpIAdd, state.uint_type, ret, diff, add}); - EmitStoreU32(state, inst.dst, ret); -} - -uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) { - const auto cond = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {max_value ? OpSGreaterThan : OpSLessThan, state.bool_type, cond, lhs, rhs}); - state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs}); - return ret; -} - -void EmitMinMaxI32(EmitterState& state, const IR::Instruction& inst, bool max_value) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto ret = EmitMinMaxI32Value(state, lhs, rhs, max_value); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitMinMax3U32(EmitterState& state, const IR::Instruction& inst, bool signed_value, - bool max_value) { - const auto src0 = EmitValueLoad(state, inst.src[0]); - const auto src1 = EmitValueLoad(state, inst.src[1]); - const auto src2 = EmitValueLoad(state, inst.src[2]); - const auto tmp = signed_value ? EmitMinMaxI32Value(state, src0, src1, max_value) - : EmitMinMaxU32Value(state, src0, src1, max_value); - const auto ret = signed_value ? EmitMinMaxI32Value(state, tmp, src2, max_value) - : EmitMinMaxU32Value(state, tmp, src2, max_value); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitMed3U32(EmitterState& state, const IR::Instruction& inst, bool signed_value) { - const auto src0 = EmitValueLoad(state, inst.src[0]); - const auto src1 = EmitValueLoad(state, inst.src[1]); - const auto src2 = EmitValueLoad(state, inst.src[2]); - const auto low_pair = signed_value ? EmitMinMaxI32Value(state, src0, src1, false) - : EmitMinMaxU32Value(state, src0, src1, false); - const auto high_pair = signed_value ? EmitMinMaxI32Value(state, src0, src1, true) - : EmitMinMaxU32Value(state, src0, src1, true); - const auto high_min = signed_value ? EmitMinMaxI32Value(state, high_pair, src2, false) - : EmitMinMaxU32Value(state, high_pair, src2, false); - const auto ret = signed_value ? EmitMinMaxI32Value(state, low_pair, high_min, true) - : EmitMinMaxU32Value(state, low_pair, high_min, true); - EmitStoreU32(state, inst.dst, ret); -} - -uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, ret, value}); - return ret; -} - -uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.float_type, ret, value}); - return ret; -} - -uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret, lhs, rhs}); - return ret; -} - -uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, lhs, rhs}); - return ret; -} - -uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalOr, state.bool_type, ret, lhs, rhs}); - return ret; -} - -uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalNot, state.bool_type, ret, value}); - return ret; -} - -F32Class EmitClassifyF32(EmitterState& state, uint32_t value) { - F32Class cls; - cls.bits = EmitBitcastF32ToU32(state, value); - const auto abs_bits = EmitAndConstant(state, cls.bits, 0x7fffffffu); - const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u); - const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu); - const auto exponent_max = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u); - const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0); - const auto quiet_bit_clear = - EmitCompareU32Constant(state, OpIEqual, EmitAndConstant(state, cls.bits, 0x00400000u), 0); - cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero); - cls.snan = EmitLogicalAndBool(state, cls.nan, quiet_bit_clear); - cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); - cls.quiet_bits = EmitOrU32(state, cls.bits, ConstantU32(state, 0x00400000u)); - return cls; -} - -uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit, - uint32_t class_match) { - const auto selected = - EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0); - return EmitLogicalAndBool(state, selected, class_match); -} - -uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) { - const auto bits = EmitBitcastF32ToU32(state, value); - const auto sign_bits = EmitAndConstant(state, bits, 0x80000000u); - const auto abs_bits = EmitAndConstant(state, bits, 0x7fffffffu); - const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u); - const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu); - const auto quiet_bits = EmitAndConstant(state, mantissa_bits, 0x00400000u); - - const auto sign = EmitCompareU32Constant(state, OpINotEqual, sign_bits, 0); - const auto positive = EmitLogicalNotBool(state, sign); - const auto exponent_zero = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0); - const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero); - const auto exponent_inf = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u); - const auto finite_exponent = EmitLogicalNotBool(state, exponent_inf); - const auto mantissa_zero = EmitCompareU32Constant(state, OpIEqual, mantissa_bits, 0); - const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero); - const auto quiet = EmitCompareU32Constant(state, OpINotEqual, quiet_bits, 0); - - const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero); - const auto snan = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet)); - const auto qnan = EmitLogicalAndBool(state, nan_common, quiet); - const auto inf = EmitLogicalAndBool(state, exponent_inf, mantissa_zero); - const auto normal = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent); - const auto denorm = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero); - const auto zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); - - uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan); - match = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan)); - match = EmitLogicalOrBool( - state, match, EmitClassMaskBitMatch(state, mask, 2, EmitLogicalAndBool(state, inf, sign))); - match = EmitLogicalOrBool( - state, match, - EmitClassMaskBitMatch(state, mask, 3, EmitLogicalAndBool(state, normal, sign))); - match = EmitLogicalOrBool( - state, match, - EmitClassMaskBitMatch(state, mask, 4, EmitLogicalAndBool(state, denorm, sign))); - match = EmitLogicalOrBool( - state, match, EmitClassMaskBitMatch(state, mask, 5, EmitLogicalAndBool(state, zero, sign))); - match = EmitLogicalOrBool( - state, match, - EmitClassMaskBitMatch(state, mask, 6, EmitLogicalAndBool(state, zero, positive))); - match = EmitLogicalOrBool( - state, match, - EmitClassMaskBitMatch(state, mask, 7, EmitLogicalAndBool(state, denorm, positive))); - match = EmitLogicalOrBool( - state, match, - EmitClassMaskBitMatch(state, mask, 8, EmitLogicalAndBool(state, normal, positive))); - return EmitLogicalOrBool( - state, match, - EmitClassMaskBitMatch(state, mask, 9, EmitLogicalAndBool(state, inf, positive))); -} - -uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) { - const auto lhs_class = EmitClassifyF32(state, lhs); - const auto rhs_class = EmitClassifyF32(state, rhs); - - const auto numeric_cond = state.builder.AllocateId(); - state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, state.bool_type, - numeric_cond, lhs, rhs}); - const auto ordered_bits = - EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits); - - const auto both_zero = EmitLogicalAndBool(state, lhs_class.zero, rhs_class.zero); - const auto zero_bits = max_value ? EmitAndU32(state, lhs_class.bits, rhs_class.bits) - : EmitOrU32(state, lhs_class.bits, rhs_class.bits); - const auto numeric_bits = EmitSelectValueU32(state, both_zero, zero_bits, ordered_bits); - - const auto rhs_nan_bits = - EmitSelectValueU32(state, rhs_class.nan, lhs_class.bits, numeric_bits); - const auto non_snan_bits = - EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits); - const auto rhs_snan_bits = - EmitSelectValueU32(state, rhs_class.snan, rhs_class.quiet_bits, non_snan_bits); - const auto result_bits = - EmitSelectValueU32(state, lhs_class.snan, lhs_class.quiet_bits, rhs_snan_bits); - return EmitBitcastU32ToF32(state, result_bits); -} - -void EmitMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool max_value) { - const auto lhs = EmitFloatLoad(state, inst.src[0]); - const auto rhs = EmitFloatLoad(state, inst.src[1]); - const auto u32 = state.builder.AllocateId(); - const auto f32 = EmitMinMaxF32Value(state, lhs, rhs, max_value); - const auto result = ApplyResultModifiersF32(state, f32, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, result}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitCompareResult(EmitterState& state, const IR::Operand& dst, uint32_t cond) { - if (IsSccOperand(dst)) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpSelect, state.uint_type, ret, cond, ConstantU32(state, 1), ConstantU32(state, 0)}); - EmitStoreU32(state, dst, ret); - return; - } - - EmitLaneMaskPairFromBool(state, dst, cond); -} - -void EmitCompareU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.bool_type, cond, lhs, rhs}); - EmitCompareResult(state, inst.dst, cond); -} - -void EmitCompareU16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[0]), 0xffffu); - const auto rhs = EmitAndConstant(state, EmitValueLoad(state, inst.src[1]), 0xffffu); - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.bool_type, cond, lhs, rhs}); - EmitCompareResult(state, inst.dst, cond); -} - -void EmitCompareI16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitSignExtendLow16U32(state, EmitValueLoad(state, inst.src[0])); - const auto rhs = EmitSignExtendLow16U32(state, EmitValueLoad(state, inst.src[1])); - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.bool_type, cond, lhs, rhs}); - EmitCompareResult(state, inst.dst, cond); -} - -void EmitBitCompareB32(EmitterState& state, const IR::Instruction& inst, bool bit_set) { - const auto value = EmitValueLoad(state, inst.src[0]); - const auto bit_index = state.builder.AllocateId(); - const auto bit = state.builder.AllocateId(); - const auto cond = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, bit_index, - EmitValueLoad(state, inst.src[1]), ConstantU32(state, 31)}); - state.builder.AddFunction( - {OpBitFieldUExtract, state.uint_type, bit, value, bit_index, ConstantU32(state, 1)}); - state.builder.AddFunction( - {OpIEqual, state.bool_type, cond, bit, ConstantU32(state, bit_set ? 1u : 0u)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, ret, cond, ConstantU32(state, 1), ConstantU32(state, 0)}); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitCompareU64(EmitterState& state, const IR::Instruction& inst) { - const bool equal = inst.op == IR::Opcode::CompareEqU64; - const auto lhs_low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto lhs_high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto rhs_low = EmitSequentialValueLoad(state, inst.src[1], 0); - const auto rhs_high = EmitSequentialValueLoad(state, inst.src[1], 1); - const auto low = state.builder.AllocateId(); - const auto high = state.builder.AllocateId(); - const auto cond = state.builder.AllocateId(); - const auto compare_op = equal ? OpIEqual : OpINotEqual; - state.builder.AddFunction({compare_op, state.bool_type, low, lhs_low, rhs_low}); - state.builder.AddFunction({compare_op, state.bool_type, high, lhs_high, rhs_high}); - state.builder.AddFunction( - {equal ? OpLogicalAnd : OpLogicalOr, state.bool_type, cond, low, high}); - EmitCompareResult(state, inst.dst, cond); -} - -void EmitCompareConstant(EmitterState& state, const IR::Instruction& inst, bool value) { - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction({value ? OpIEqual : OpINotEqual, state.bool_type, cond, - ConstantU32(state, 0), ConstantU32(state, 0)}); - EmitCompareResult(state, inst.dst, cond); -} - -void EmitCompareMaskU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitValueLoad(state, inst.src[0]); - const auto rhs = EmitValueLoad(state, inst.src[1]); - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.bool_type, cond, lhs, rhs}); - EmitCompareResult(state, inst.dst, cond); -} - -void EmitCompareF32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitFloatLoad(state, inst.src[0]); - const auto rhs = EmitFloatLoad(state, inst.src[1]); - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.bool_type, cond, lhs, rhs}); - EmitCompareResult(state, inst.dst, cond); -} - -void EmitCompareOrderedF32(EmitterState& state, const IR::Instruction& inst, bool ordered) { - const auto lhs = EmitFloatLoad(state, inst.src[0]); - const auto rhs = EmitFloatLoad(state, inst.src[1]); - const auto lhs_check = state.builder.AllocateId(); - const auto rhs_check = state.builder.AllocateId(); - const auto cond = state.builder.AllocateId(); - if (ordered) { - state.builder.AddFunction({OpFOrdEqual, state.bool_type, lhs_check, lhs, lhs}); - state.builder.AddFunction({OpFOrdEqual, state.bool_type, rhs_check, rhs, rhs}); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, cond, lhs_check, rhs_check}); - } else { - state.builder.AddFunction({OpFUnordNotEqual, state.bool_type, lhs_check, lhs, lhs}); - state.builder.AddFunction({OpFUnordNotEqual, state.bool_type, rhs_check, rhs, rhs}); - state.builder.AddFunction({OpLogicalOr, state.bool_type, cond, lhs_check, rhs_check}); - } - EmitCompareResult(state, inst.dst, cond); -} - -void EmitCompareClassF32(EmitterState& state, const IR::Instruction& inst) { - const auto value = EmitFloatLoad(state, inst.src[0]); - const auto mask = EmitValueLoad(state, inst.src[1]); - EmitCompareResult(state, inst.dst, EmitClassMaskF32(state, value, mask)); -} - -void EmitCompareMaskF32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitFloatLoad(state, inst.src[0]); - const auto rhs = EmitFloatLoad(state, inst.src[1]); - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.bool_type, cond, lhs, rhs}); - EmitCompareResult(state, inst.dst, cond); -} - -void EmitConvertU32ToF32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto f32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction({OpConvertUToF, state.float_type, f32, src}); - const auto result = ApplyResultModifiersF32(state, f32, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, result}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitConvertI32ToF32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto i32 = state.builder.AllocateId(); - const auto f32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, i32, src}); - state.builder.AddFunction({OpConvertSToF, state.float_type, f32, i32}); - const auto result = ApplyResultModifiersF32(state, f32, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, result}); - EmitStoreU32(state, inst.dst, u32); -} - -uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, ret, state.glsl_std450, GlslTrunc, value}); - return ret; -} - -void EmitConvertF32ToU32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitFloatLoad(state, inst.src[0]); - const auto trunc = EmitTruncF32Value(state, src); - const auto converted = state.builder.AllocateId(); - const auto below_zero = state.builder.AllocateId(); - const auto above_max = state.builder.AllocateId(); - state.builder.AddFunction({OpConvertFToU, state.uint_type, converted, trunc}); - state.builder.AddFunction( - {OpFOrdLessThanEqual, state.bool_type, below_zero, src, ConstantF32(state, 0)}); - state.builder.AddFunction( - {OpFOrdGreaterThanEqual, state.bool_type, above_max, src, ConstantF32(state, 0x4f800000u)}); - const auto zero_or_nan = EmitLogicalOrBool(state, EmitClassifyF32(state, src).nan, below_zero); - const auto saturated = - EmitSelectValueU32(state, above_max, ConstantU32(state, 0xffffffffu), converted); - const auto ret = EmitSelectValueU32(state, zero_or_nan, ConstantU32(state, 0), saturated); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitConvertF32ToI32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitFloatLoad(state, inst.src[0]); - const auto trunc = EmitTruncF32Value(state, src); - const auto i32 = state.builder.AllocateId(); - const auto converted = state.builder.AllocateId(); - const auto below_min = state.builder.AllocateId(); - const auto above_max = state.builder.AllocateId(); - state.builder.AddFunction({OpConvertFToS, state.int_type, i32, trunc}); - state.builder.AddFunction({OpBitcast, state.uint_type, converted, i32}); - state.builder.AddFunction( - {OpFOrdLessThanEqual, state.bool_type, below_min, src, ConstantF32(state, 0xcf000000u)}); - state.builder.AddFunction( - {OpFOrdGreaterThanEqual, state.bool_type, above_max, src, ConstantF32(state, 0x4f000000u)}); - const auto high_sat = - EmitSelectValueU32(state, above_max, ConstantU32(state, 0x7fffffffu), converted); - const auto low_sat = - EmitSelectValueU32(state, below_min, ConstantU32(state, 0x80000000u), high_sat); - const auto ret = - EmitSelectValueU32(state, EmitClassifyF32(state, src).nan, ConstantU32(state, 0), low_sat); - EmitStoreU32(state, inst.dst, ret); -} - -void EmitConvertF32ToF16(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitFloatLoad(state, inst.src[0]); - const auto pair = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpCompositeConstruct, state.vec2_float_type, pair, src, ConstantF32(state, 0)}); - state.builder.AddFunction( - {OpExtInst, state.uint_type, u32, state.glsl_std450, GlslPackHalf2x16, pair}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitConvertF16ToF32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto unpacked = state.builder.AllocateId(); - auto f32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, src}); - state.builder.AddFunction({OpCompositeExtract, state.float_type, f32, unpacked, 0}); - if (inst.src[0].absolute) { - const auto abs_value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, f32}); - f32 = abs_value; - } - if (inst.src[0].negate) { - const auto neg_value = state.builder.AllocateId(); - state.builder.AddFunction({OpFNegate, state.float_type, neg_value, f32}); - f32 = neg_value; - } - const auto result = ApplyResultModifiersF32(state, f32, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, result}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitConvertF16ToU16(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto unpacked = state.builder.AllocateId(); - auto f32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, src}); - state.builder.AddFunction({OpCompositeExtract, state.float_type, f32, unpacked, 0}); - if (inst.src[0].absolute) { - const auto abs_value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, f32}); - f32 = abs_value; - } - if (inst.src[0].negate) { - const auto neg_value = state.builder.AllocateId(); - state.builder.AddFunction({OpFNegate, state.float_type, neg_value, f32}); - f32 = neg_value; - } - - const auto truncated = EmitTruncF32Value(state, f32); - const auto below_zero = state.builder.AllocateId(); - const auto above_max = state.builder.AllocateId(); - const auto safe_low = state.builder.AllocateId(); - const auto safe_value = state.builder.AllocateId(); - const auto converted = state.builder.AllocateId(); - state.builder.AddFunction( - {OpFOrdLessThanEqual, state.bool_type, below_zero, f32, ConstantF32(state, 0)}); - state.builder.AddFunction( - {OpFOrdGreaterThanEqual, state.bool_type, above_max, f32, ConstantF32(state, 0x477fff00u)}); - const auto zero_or_nan = EmitLogicalOrBool(state, EmitClassifyF32(state, f32).nan, below_zero); - state.builder.AddFunction( - {OpSelect, state.float_type, safe_low, zero_or_nan, ConstantF32(state, 0), truncated}); - state.builder.AddFunction({OpSelect, state.float_type, safe_value, above_max, - ConstantF32(state, 0x477fff00u), safe_low}); - state.builder.AddFunction({OpConvertFToU, state.uint_type, converted, safe_value}); - EmitStoreU32(state, inst.dst, EmitAndConstant(state, converted, 0xffffu)); -} - -void EmitConvertU16ToF16(EmitterState& state, const IR::Instruction& inst) { - const auto src_bits = EmitU16LaneBits(state, inst.src[0], false, false); - const auto f32 = state.builder.AllocateId(); - state.builder.AddFunction({OpConvertUToF, state.float_type, f32, src_bits}); - EmitStoreU32(state, inst.dst, EmitPackLowF32ToF16Bits(state, f32)); -} - -void EmitConvertI16ToF16(EmitterState& state, const IR::Instruction& inst) { - const auto src_bits = EmitU16LaneBits(state, inst.src[0], false, true); - const auto i32 = state.builder.AllocateId(); - const auto f32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, i32, src_bits}); - state.builder.AddFunction({OpConvertSToF, state.float_type, f32, i32}); - EmitStoreU32(state, inst.dst, EmitPackLowF32ToF16Bits(state, f32)); -} - -void EmitConvertF16ToI16(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto unpacked = state.builder.AllocateId(); - auto f32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, src}); - state.builder.AddFunction({OpCompositeExtract, state.float_type, f32, unpacked, 0}); - if (inst.src[0].absolute) { - const auto abs_value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, f32}); - f32 = abs_value; - } - if (inst.src[0].negate) { - const auto neg_value = state.builder.AllocateId(); - state.builder.AddFunction({OpFNegate, state.float_type, neg_value, f32}); - f32 = neg_value; - } - - const auto truncated = EmitTruncF32Value(state, f32); - const auto below_min = state.builder.AllocateId(); - const auto above_max = state.builder.AllocateId(); - const auto safe_low = state.builder.AllocateId(); - const auto safe_value = state.builder.AllocateId(); - const auto nan_safe = state.builder.AllocateId(); - const auto i32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpFOrdLessThanEqual, state.bool_type, below_min, f32, ConstantF32(state, 0xc7000000u)}); - state.builder.AddFunction( - {OpFOrdGreaterThanEqual, state.bool_type, above_max, f32, ConstantF32(state, 0x46fffe00u)}); - state.builder.AddFunction({OpSelect, state.float_type, safe_low, below_min, - ConstantF32(state, 0xc7000000u), truncated}); - state.builder.AddFunction({OpSelect, state.float_type, safe_value, above_max, - ConstantF32(state, 0x46fffe00u), safe_low}); - state.builder.AddFunction({OpSelect, state.float_type, nan_safe, - EmitClassifyF32(state, f32).nan, ConstantF32(state, 0), safe_value}); - state.builder.AddFunction({OpConvertFToS, state.int_type, i32, nan_safe}); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, i32}); - EmitStoreU32(state, inst.dst, EmitAndConstant(state, u32, 0xffffu)); -} - -void EmitConvertRoundPlusInfF32ToI32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitFloatLoad(state, inst.src[0]); - const auto biased = state.builder.AllocateId(); - const auto rounded = state.builder.AllocateId(); - const auto i32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpFAdd, state.float_type, biased, src, ConstantF32(state, 0x3f000000u)}); - state.builder.AddFunction( - {OpExtInst, state.float_type, rounded, state.glsl_std450, GlslFloor, biased}); - state.builder.AddFunction({OpConvertFToS, state.int_type, i32, rounded}); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, i32}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitConvertFloorF32ToI32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitFloatLoad(state, inst.src[0]); - const auto floor = state.builder.AllocateId(); - const auto i32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, floor, state.glsl_std450, GlslFloor, src}); - state.builder.AddFunction({OpConvertFToS, state.int_type, i32, floor}); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, i32}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitConvertI4ToOffsetF32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto src_i32 = state.builder.AllocateId(); - const auto nibble = state.builder.AllocateId(); - const auto f32 = state.builder.AllocateId(); - const auto scaled = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, src_i32, src}); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, nibble, src_i32, - ConstantU32(state, 0), ConstantU32(state, 4)}); - state.builder.AddFunction({OpConvertSToF, state.float_type, f32, nibble}); - state.builder.AddFunction( - {OpFDiv, state.float_type, scaled, f32, ConstantF32(state, 0x41800000u)}); - const auto result = ApplyResultModifiersF32(state, scaled, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, result}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitConvertByteU32ToF32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto index = inst.src[1].kind == IR::OperandKind::ImmediateU32 ? inst.src[1].imm : 0u; - const auto byte = EmitBitFieldExtractConstant(state, src, (index & 3u) * 8u, 8u); - const auto f32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction({OpConvertUToF, state.float_type, f32, byte}); - const auto result = ApplyResultModifiersF32(state, f32, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, result}); - EmitStoreU32(state, inst.dst, u32); -} - -uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) { - const auto bits = state.builder.AllocateId(); - const auto abs_bits = state.builder.AllocateId(); - const auto sign_bits = state.builder.AllocateId(); - const auto non_zero = state.builder.AllocateId(); - const auto subnormal = state.builder.AllocateId(); - const auto flush = state.builder.AllocateId(); - const auto selected = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, value}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, sign_bits, bits, ConstantU32(state, 0x80000000u)}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, non_zero, abs_bits, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpULessThan, state.bool_type, subnormal, abs_bits, ConstantU32(state, 0x00800000u)}); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, flush, non_zero, subnormal}); - state.builder.AddFunction({OpSelect, state.uint_type, selected, flush, sign_bits, bits}); - state.builder.AddFunction({OpBitcast, state.float_type, ret, selected}); - return ret; -} - -void EmitRcpF32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitFlushF32DenormToSignedZero(state, EmitFloatLoad(state, inst.src[0])); - const auto f32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpFDiv, state.float_type, f32, ConstantF32(state, 0x3f800000u), src}); - const auto result = ApplyResultModifiersF32(state, f32, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, result}); - EmitStoreU32(state, inst.dst, u32); -} - -uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero) { - const auto fract = state.builder.AllocateId(); - const auto bits = state.builder.AllocateId(); - const auto abs_bits = state.builder.AllocateId(); - const auto large = state.builder.AllocateId(); - const auto finite = state.builder.AllocateId(); - const auto large_finite = state.builder.AllocateId(); - const auto reduced = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, fract, state.glsl_std450, GlslFract, src}); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, src}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)}); - state.builder.AddFunction( - {OpUGreaterThanEqual, state.bool_type, large, abs_bits, ConstantU32(state, 0x4b000000u)}); - state.builder.AddFunction( - {OpULessThan, state.bool_type, finite, abs_bits, ConstantU32(state, 0x7f800000u)}); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, large_finite, large, finite}); - state.builder.AddFunction( - {OpSelect, state.float_type, reduced, large_finite, ConstantF32(state, 0), fract}); - if (!preserve_signed_zero) { - return reduced; - } - const auto zero = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpIEqual, state.bool_type, zero, abs_bits, ConstantU32(state, 0)}); - state.builder.AddFunction({OpSelect, state.float_type, ret, zero, src, reduced}); - return ret; -} - -void EmitFloatExtInst(EmitterState& state, const IR::Instruction& inst, uint32_t ext_inst, - bool scale_by_two_pi, bool flush_denorm) { - auto src = EmitFloatLoad(state, inst.src[0]); - if (flush_denorm) { - src = EmitFlushF32DenormToSignedZero(state, src); - } - if (scale_by_two_pi) { - src = EmitTrigCycleF32(state, src, ext_inst == GlslSin); - const auto scaled = state.builder.AllocateId(); - state.builder.AddFunction( - {OpFMul, state.float_type, scaled, src, ConstantF32(state, 0x40c90fdbu)}); - src = scaled; - } - const auto f32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction({OpExtInst, state.float_type, f32, state.glsl_std450, ext_inst, src}); - const auto result = ApplyResultModifiersF32(state, f32, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, result}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitMadF32(EmitterState& state, const IR::Instruction& inst) { - const auto lhs = EmitMixF32Load(state, inst.src[0]); - const auto rhs = EmitMixF32Load(state, inst.src[1]); - const auto add = EmitMixF32Load(state, inst.src[2]); - const auto f32 = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, f32, state.glsl_std450, GlslFma, lhs, rhs, add}); - const auto result = ApplyResultModifiersF32(state, f32, inst.dst); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, result}); - EmitStoreU32(state, inst.dst, u32); -} - -uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpFNegate, state.float_type, ret, value}); - return ret; -} - -uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, ret, state.glsl_std450, GlslFAbs, value}); - return ret; -} - -uint32_t EmitFCompareValue(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.bool_type, ret, lhs, rhs}); - return ret; -} - -uint32_t EmitLogicalAndValue(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, lhs, rhs}); - return ret; -} - -uint32_t EmitFSelectValue(EmitterState& state, uint32_t cond, uint32_t true_value, - uint32_t false_value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.float_type, ret, cond, true_value, false_value}); - return ret; -} - -uint32_t EmitFMulConstant(EmitterState& state, uint32_t value, uint32_t bits) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpFMul, state.float_type, ret, value, ConstantF32(state, bits)}); - return ret; -} - -CubeF32Values EmitCubeF32Values(EmitterState& state, const IR::Instruction& inst) { - CubeF32Values cube; - cube.x = EmitFloatLoad(state, inst.src[0]); - cube.y = EmitFloatLoad(state, inst.src[1]); - cube.z = EmitFloatLoad(state, inst.src[2]); - cube.nx = EmitFNegateValue(state, cube.x); - cube.ny = EmitFNegateValue(state, cube.y); - cube.nz = EmitFNegateValue(state, cube.z); - const auto ax = EmitFAbsValue(state, cube.x); - const auto ay = EmitFAbsValue(state, cube.y); - const auto az = EmitFAbsValue(state, cube.z); - const auto z_ge_x = EmitFCompareValue(state, OpFOrdGreaterThanEqual, az, ax); - const auto z_ge_y = EmitFCompareValue(state, OpFOrdGreaterThanEqual, az, ay); - cube.z_face = EmitLogicalAndValue(state, z_ge_x, z_ge_y); - cube.y_face = EmitFCompareValue(state, OpFOrdGreaterThanEqual, ay, ax); - cube.x_neg = EmitFCompareValue(state, OpFOrdLessThan, cube.x, ConstantF32(state, 0)); - cube.y_neg = EmitFCompareValue(state, OpFOrdLessThan, cube.y, ConstantF32(state, 0)); - cube.z_neg = EmitFCompareValue(state, OpFOrdLessThan, cube.z, ConstantF32(state, 0)); - return cube; -} - -uint32_t EmitCubeIdF32(EmitterState& state, const CubeF32Values& cube) { - const auto x_face = - EmitFSelectValue(state, cube.x_neg, ConstantF32(state, 0x3f800000u), ConstantF32(state, 0)); - const auto y_face = EmitFSelectValue(state, cube.y_neg, ConstantF32(state, 0x40400000u), - ConstantF32(state, 0x40000000u)); - const auto z_face = EmitFSelectValue(state, cube.z_neg, ConstantF32(state, 0x40a00000u), - ConstantF32(state, 0x40800000u)); - const auto xy = EmitFSelectValue(state, cube.y_face, y_face, x_face); - return EmitFSelectValue(state, cube.z_face, z_face, xy); -} - -uint32_t EmitCubeScF32(EmitterState& state, const CubeF32Values& cube) { - const auto x_sc = EmitFSelectValue(state, cube.x_neg, cube.z, cube.nz); - const auto z_sc = EmitFSelectValue(state, cube.z_neg, cube.nx, cube.x); - const auto xy = EmitFSelectValue(state, cube.y_face, cube.x, x_sc); - return EmitFSelectValue(state, cube.z_face, z_sc, xy); -} - -uint32_t EmitCubeTcF32(EmitterState& state, const CubeF32Values& cube) { - const auto y_tc = EmitFSelectValue(state, cube.y_neg, cube.nz, cube.z); - const auto xy = EmitFSelectValue(state, cube.y_face, y_tc, cube.ny); - return EmitFSelectValue(state, cube.z_face, cube.ny, xy); -} - -uint32_t EmitCubeMaF32(EmitterState& state, const CubeF32Values& cube) { - const auto x_ma = EmitFMulConstant(state, cube.x, 0x40000000u); - const auto y_ma = EmitFMulConstant(state, cube.y, 0x40000000u); - const auto z_ma = EmitFMulConstant(state, cube.z, 0x40000000u); - const auto xy = EmitFSelectValue(state, cube.y_face, y_ma, x_ma); - return EmitFSelectValue(state, cube.z_face, z_ma, xy); -} - -void EmitCubeF32(EmitterState& state, const IR::Instruction& inst) { - const auto cube = EmitCubeF32Values(state, inst); - uint32_t ret = 0; - switch (inst.op) { - case IR::Opcode::CubeIdF32: ret = EmitCubeIdF32(state, cube); break; - case IR::Opcode::CubeScF32: ret = EmitCubeScF32(state, cube); break; - case IR::Opcode::CubeTcF32: ret = EmitCubeTcF32(state, cube); break; - case IR::Opcode::CubeMaF32: ret = EmitCubeMaF32(state, cube); break; - default: ret = ConstantF32(state, 0); break; - } - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, ret}); - EmitStoreU32(state, inst.dst, bits); -} - -void EmitDot2AccF32F16(EmitterState& state, const IR::Instruction& inst) { - const auto src0 = EmitValueLoad(state, inst.src[0]); - const auto src1 = EmitValueLoad(state, inst.src[1]); - const auto vec0 = state.builder.AllocateId(); - const auto vec1 = state.builder.AllocateId(); - const auto a_lo = state.builder.AllocateId(); - const auto a_hi = state.builder.AllocateId(); - const auto b_lo = state.builder.AllocateId(); - const auto b_hi = state.builder.AllocateId(); - const auto acc = EmitFloatLoad(state, inst.src[2]); - const auto lo = state.builder.AllocateId(); - const auto dot = state.builder.AllocateId(); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.vec2_float_type, vec0, state.glsl_std450, GlslUnpackHalf2x16, src0}); - state.builder.AddFunction( - {OpExtInst, state.vec2_float_type, vec1, state.glsl_std450, GlslUnpackHalf2x16, src1}); - state.builder.AddFunction({OpCompositeExtract, state.float_type, a_lo, vec0, 0}); - state.builder.AddFunction({OpCompositeExtract, state.float_type, a_hi, vec0, 1}); - state.builder.AddFunction({OpCompositeExtract, state.float_type, b_lo, vec1, 0}); - state.builder.AddFunction({OpCompositeExtract, state.float_type, b_hi, vec1, 1}); - state.builder.AddFunction( - {OpExtInst, state.float_type, lo, state.glsl_std450, GlslFma, a_lo, b_lo, acc}); - state.builder.AddFunction( - {OpExtInst, state.float_type, dot, state.glsl_std450, GlslFma, a_hi, b_hi, lo}); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, dot}); - EmitStoreU32(state, inst.dst, u32); -} - -uint32_t EmitPackedF16Lane(EmitterState& state, const IR::Operand& operand, bool high_lane) { - const auto raw = EmitValueLoad(state, operand); - const auto unpacked = state.builder.AllocateId(); - auto value = state.builder.AllocateId(); - const auto selected_lane = high_lane ? operand.op_sel_hi : operand.op_sel; - const auto lane = selected_lane ? 1u : 0u; - state.builder.AddFunction( - {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, raw}); - state.builder.AddFunction({OpCompositeExtract, state.float_type, value, unpacked, lane}); - if (operand.absolute) { - const auto abs_value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, value}); - value = abs_value; - } - if (high_lane ? operand.negate_hi : operand.negate) { - const auto neg_value = state.builder.AllocateId(); - state.builder.AddFunction({OpFNegate, state.float_type, neg_value, value}); - value = neg_value; - } - return value; -} - -void EmitCompareF16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitPackedF16Lane(state, inst.src[0], false); - const auto rhs = EmitPackedF16Lane(state, inst.src[1], false); - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.bool_type, cond, lhs, rhs}); - EmitCompareResult(state, inst.dst, cond); -} - -uint32_t EmitPackedF16LaneBits(EmitterState& state, const IR::Operand& operand, bool high_lane) { - const auto raw = EmitValueLoad(state, operand); - const auto selected_lane = high_lane ? operand.op_sel_hi : operand.op_sel; - const auto lane = selected_lane ? 1u : 0u; - auto bits = lane != 0u ? EmitShiftRightConstant(state, raw, 16) : raw; - bits = EmitAndConstant(state, bits, 0xffffu); - if (operand.absolute) { - bits = EmitAndConstant(state, bits, 0x7fffu); - } - if (high_lane ? operand.negate_hi : operand.negate) { - bits = EmitXorConstant(state, bits, 0x8000u); - } - return bits; -} - -uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) { - const auto unpacked = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, bits}); - state.builder.AddFunction({OpCompositeExtract, state.float_type, ret, unpacked, 0}); - return ret; -} - -uint32_t EmitPackLowF32ToF16Bits(EmitterState& state, uint32_t value) { - const auto pair = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpCompositeConstruct, state.vec2_float_type, pair, value, ConstantF32(state, 0)}); - state.builder.AddFunction( - {OpExtInst, state.uint_type, ret, state.glsl_std450, GlslPackHalf2x16, pair}); - return EmitAndConstant(state, ret, 0xffffu); -} - -void EmitPackB32F16(EmitterState& state, const IR::Instruction& inst) { - const auto lo = EmitPackedF16LaneBits(state, inst.src[0], false); - const auto hi = - EmitShiftLeftConstant(state, EmitPackedF16LaneBits(state, inst.src[1], false), 16); - EmitStoreU32(state, inst.dst, EmitOrU32(state, lo, hi)); -} - -F16Class EmitClassifyF16Bits(EmitterState& state, uint32_t value) { - F16Class cls; - cls.bits = EmitAndConstant(state, value, 0xffffu); - const auto abs_bits = EmitAndConstant(state, cls.bits, 0x7fffu); - const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7c00u); - const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x03ffu); - const auto exponent_max = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7c00u); - const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0); - const auto quiet_bit_clear = - EmitCompareU32Constant(state, OpIEqual, EmitAndConstant(state, cls.bits, 0x0200u), 0); - cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero); - cls.snan = EmitLogicalAndBool(state, cls.nan, quiet_bit_clear); - cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); - cls.quiet_bits = EmitOrU32(state, cls.bits, ConstantU32(state, 0x0200u)); - return cls; -} - -uint32_t EmitNegativeNumericF16Bool(EmitterState& state, const F16Class& cls) { - const auto sign = - EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, cls.bits, 0x8000u), 0); - const auto nonzero = EmitLogicalNotBool(state, cls.zero); - return EmitLogicalAndBool(state, EmitLogicalAndBool(state, sign, nonzero), - EmitLogicalNotBool(state, cls.nan)); -} - -uint32_t EmitRcpF16Bits(EmitterState& state, uint32_t bits) { - const auto f32 = EmitF16BitsToF32(state, bits); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpFDiv, state.float_type, ret, ConstantF32(state, 0x3f800000u), f32}); - return EmitPackLowF32ToF16Bits(state, ret); -} - -uint32_t EmitSqrtF16Bits(EmitterState& state, uint32_t bits) { - const auto cls = EmitClassifyF16Bits(state, bits); - const auto f32 = EmitF16BitsToF32(state, cls.bits); - const auto sqrt = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, sqrt, state.glsl_std450, GlslSqrt, f32}); - const auto packed = EmitPackLowF32ToF16Bits(state, sqrt); - const auto numeric = EmitSelectValueU32(state, EmitNegativeNumericF16Bool(state, cls), - ConstantU32(state, 0xfe00u), packed); - return EmitSelectValueU32(state, cls.zero, cls.bits, numeric); -} - -uint32_t EmitRsqF16Bits(EmitterState& state, uint32_t bits) { - const auto cls = EmitClassifyF16Bits(state, bits); - const auto f32 = EmitF16BitsToF32(state, cls.bits); - const auto rsq = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, rsq, state.glsl_std450, GlslInverseSqrt, f32}); - const auto packed = EmitPackLowF32ToF16Bits(state, rsq); - const auto sign = - EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, cls.bits, 0x8000u), 0); - const auto zero_result = - EmitSelectValueU32(state, sign, ConstantU32(state, 0xfc00u), ConstantU32(state, 0x7c00u)); - const auto numeric = EmitSelectValueU32(state, EmitNegativeNumericF16Bool(state, cls), - ConstantU32(state, 0xfe00u), packed); - return EmitSelectValueU32(state, cls.zero, zero_result, numeric); -} - -uint32_t EmitLog2F16Bits(EmitterState& state, uint32_t bits) { - const auto cls = EmitClassifyF16Bits(state, bits); - const auto f32 = EmitF16BitsToF32(state, cls.bits); - const auto log2 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, log2, state.glsl_std450, GlslLog2, f32}); - const auto packed = EmitPackLowF32ToF16Bits(state, log2); - return EmitSelectValueU32(state, EmitNegativeNumericF16Bool(state, cls), - ConstantU32(state, 0xfe00u), packed); -} - -uint32_t EmitExp2F16Bits(EmitterState& state, uint32_t bits) { - const auto f32 = EmitF16BitsToF32(state, bits); - const auto exp2 = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, exp2, state.glsl_std450, GlslExp2, f32}); - return EmitPackLowF32ToF16Bits(state, exp2); -} - -uint32_t EmitF16MathExtInstBits(EmitterState& state, uint32_t bits, uint32_t ext_inst) { - const auto f32 = EmitF16BitsToF32(state, bits); - const auto result = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, result, state.glsl_std450, ext_inst, f32}); - return EmitPackLowF32ToF16Bits(state, result); -} - -void EmitSpecialF16(EmitterState& state, const IR::Instruction& inst) { - const auto bits = EmitPackedF16LaneBits(state, inst.src[0], false); - uint32_t ret = ConstantU32(state, 0); - switch (inst.op) { - case IR::Opcode::RcpF16: ret = EmitRcpF16Bits(state, bits); break; - case IR::Opcode::SqrtF16: ret = EmitSqrtF16Bits(state, bits); break; - case IR::Opcode::InverseSqrtF16: ret = EmitRsqF16Bits(state, bits); break; - case IR::Opcode::Log2F16: ret = EmitLog2F16Bits(state, bits); break; - case IR::Opcode::Exp2F16: ret = EmitExp2F16Bits(state, bits); break; - case IR::Opcode::FloorF16: ret = EmitF16MathExtInstBits(state, bits, GlslFloor); break; - case IR::Opcode::CeilF16: ret = EmitF16MathExtInstBits(state, bits, GlslCeil); break; - case IR::Opcode::TruncF16: ret = EmitF16MathExtInstBits(state, bits, GlslTrunc); break; - case IR::Opcode::RoundEvenF16: - ret = EmitF16MathExtInstBits(state, bits, GlslRoundEven); - break; - default: break; - } - EmitStoreU32(state, inst.dst, ret); -} - -void EmitBinaryF16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - const auto lhs = EmitPackedF16Lane(state, inst.src[0], false); - const auto rhs = EmitPackedF16Lane(state, inst.src[1], false); - auto value = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.float_type, value, lhs, rhs}); - value = ApplyResultModifiersF32(state, value, inst.dst); - EmitStoreU32(state, inst.dst, EmitPackLowF32ToF16Bits(state, value)); -} - -void EmitFmaF16(EmitterState& state, const IR::Instruction& inst) { - const auto fma = - ApplyResultModifiersF32(state, EmitPackedF16FmaLane(state, inst, false), inst.dst); - EmitStoreU32(state, inst.dst, EmitPackLowF32ToF16Bits(state, fma)); -} - -uint32_t EmitMinMaxF16Bits(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) { - const auto lhs_class = EmitClassifyF16Bits(state, lhs); - const auto rhs_class = EmitClassifyF16Bits(state, rhs); - const auto lhs_f32 = EmitF16BitsToF32(state, lhs_class.bits); - const auto rhs_f32 = EmitF16BitsToF32(state, rhs_class.bits); - - const auto numeric_cond = state.builder.AllocateId(); - state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, state.bool_type, - numeric_cond, lhs_f32, rhs_f32}); - const auto ordered_bits = - EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits); - - const auto both_zero = EmitLogicalAndBool(state, lhs_class.zero, rhs_class.zero); - const auto zero_bits = max_value ? EmitAndU32(state, lhs_class.bits, rhs_class.bits) - : EmitOrU32(state, lhs_class.bits, rhs_class.bits); - const auto numeric_bits = EmitSelectValueU32(state, both_zero, zero_bits, ordered_bits); - - const auto rhs_nan_bits = - EmitSelectValueU32(state, rhs_class.nan, lhs_class.bits, numeric_bits); - const auto non_snan_bits = - EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits); - const auto rhs_snan_bits = - EmitSelectValueU32(state, rhs_class.snan, rhs_class.quiet_bits, non_snan_bits); - const auto result_bits = - EmitSelectValueU32(state, lhs_class.snan, lhs_class.quiet_bits, rhs_snan_bits); - return EmitAndConstant(state, result_bits, 0xffffu); -} - -void EmitMinMaxF16(EmitterState& state, const IR::Instruction& inst, bool max_value) { - const auto bits = - EmitMinMaxF16Bits(state, EmitPackedF16LaneBits(state, inst.src[0], false), - EmitPackedF16LaneBits(state, inst.src[1], false), max_value); - EmitStoreU32(state, inst.dst, bits); -} - -uint32_t EmitMinMax3F16Bits(EmitterState& state, uint32_t src0, uint32_t src1, uint32_t src2, - bool max_value) { - return EmitMinMaxF16Bits(state, EmitMinMaxF16Bits(state, src0, src1, max_value), src2, - max_value); -} - -void EmitMinMax3F16(EmitterState& state, const IR::Instruction& inst, bool max_value) { - const auto bits = - EmitMinMax3F16Bits(state, EmitPackedF16LaneBits(state, inst.src[0], false), - EmitPackedF16LaneBits(state, inst.src[1], false), - EmitPackedF16LaneBits(state, inst.src[2], false), max_value); - EmitStoreU32(state, inst.dst, bits); -} - -uint32_t EmitF16BitsEqualBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto lhs_f32 = EmitF16BitsToF32(state, lhs); - const auto rhs_f32 = EmitF16BitsToF32(state, rhs); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpFOrdEqual, state.bool_type, ret, lhs_f32, rhs_f32}); - return ret; -} - -uint32_t EmitAnyNanF16Bool(EmitterState& state, uint32_t src0, uint32_t src1, uint32_t src2) { - const auto nan01 = EmitLogicalOrBool(state, EmitClassifyF16Bits(state, src0).nan, - EmitClassifyF16Bits(state, src1).nan); - return EmitLogicalOrBool(state, nan01, EmitClassifyF16Bits(state, src2).nan); -} - -void EmitMed3F16(EmitterState& state, const IR::Instruction& inst) { - const auto src0 = EmitPackedF16LaneBits(state, inst.src[0], false); - const auto src1 = EmitPackedF16LaneBits(state, inst.src[1], false); - const auto src2 = EmitPackedF16LaneBits(state, inst.src[2], false); - - const auto min3 = EmitMinMax3F16Bits(state, src0, src1, src2, false); - const auto max3 = EmitMinMax3F16Bits(state, src0, src1, src2, true); - const auto med_src0 = EmitMinMaxF16Bits(state, src1, src2, true); - const auto med_src1 = EmitMinMaxF16Bits(state, src0, src2, true); - const auto med_src2 = EmitMinMaxF16Bits(state, src0, src1, true); - const auto med01 = - EmitSelectValueU32(state, EmitF16BitsEqualBool(state, max3, src1), med_src1, med_src2); - const auto median = - EmitSelectValueU32(state, EmitF16BitsEqualBool(state, max3, src0), med_src0, med01); - const auto bits = - EmitSelectValueU32(state, EmitAnyNanF16Bool(state, src0, src1, src2), min3, median); - EmitStoreU32(state, inst.dst, bits); -} - -uint32_t EmitPackedMinMaxF16(EmitterState& state, const IR::Instruction& inst, bool max_value) { - const auto lo = EmitMinMaxF16Bits(state, EmitPackedF16LaneBits(state, inst.src[0], false), - EmitPackedF16LaneBits(state, inst.src[1], false), max_value); - const auto hi = EmitMinMaxF16Bits(state, EmitPackedF16LaneBits(state, inst.src[0], true), - EmitPackedF16LaneBits(state, inst.src[1], true), max_value); - return EmitOrU32(state, lo, EmitShiftLeftConstant(state, hi, 16)); -} - -uint32_t EmitPackedF16BinaryLane(EmitterState& state, const IR::Instruction& inst, bool high_lane, - uint32_t opcode) { - const auto lhs = EmitPackedF16Lane(state, inst.src[0], high_lane); - const auto rhs = EmitPackedF16Lane(state, inst.src[1], high_lane); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.float_type, ret, lhs, rhs}); - return ret; -} - -uint32_t EmitPackedF16FmaLane(EmitterState& state, const IR::Instruction& inst, bool high_lane) { - const auto lhs = EmitPackedF16Lane(state, inst.src[0], high_lane); - const auto rhs = EmitPackedF16Lane(state, inst.src[1], high_lane); - const auto add = EmitPackedF16Lane(state, inst.src[2], high_lane); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, ret, state.glsl_std450, GlslFma, lhs, rhs, add}); - return ret; -} - -uint32_t EmitPackedF16BitsWithResultModifiers(EmitterState& state, uint32_t bits, - const IR::Operand& dst) { - if (!dst.clamp && dst.omod == 0u) { - return bits; - } - auto lo = ApplyResultModifiersF32( - state, EmitF16BitsToF32(state, EmitAndConstant(state, bits, 0xffffu)), dst); - auto hi = ApplyResultModifiersF32( - state, EmitF16BitsToF32(state, EmitShiftRightConstant(state, bits, 16)), dst); - const auto pair = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, pair, lo, hi}); - state.builder.AddFunction( - {OpExtInst, state.uint_type, ret, state.glsl_std450, GlslPackHalf2x16, pair}); - return ret; -} - -void EmitPackedF16(EmitterState& state, const IR::Instruction& inst) { - uint32_t lo = 0; - uint32_t hi = 0; - switch (inst.op) { - case IR::Opcode::PackedAddF16: - lo = EmitPackedF16BinaryLane(state, inst, false, OpFAdd); - hi = EmitPackedF16BinaryLane(state, inst, true, OpFAdd); - break; - case IR::Opcode::PackedMulF16: - lo = EmitPackedF16BinaryLane(state, inst, false, OpFMul); - hi = EmitPackedF16BinaryLane(state, inst, true, OpFMul); - break; - case IR::Opcode::PackedMinF16: - EmitStoreU32(state, inst.dst, - EmitPackedF16BitsWithResultModifiers( - state, EmitPackedMinMaxF16(state, inst, false), inst.dst)); - return; - case IR::Opcode::PackedMaxF16: - EmitStoreU32(state, inst.dst, - EmitPackedF16BitsWithResultModifiers( - state, EmitPackedMinMaxF16(state, inst, true), inst.dst)); - return; - case IR::Opcode::PackedFmaF16: - lo = EmitPackedF16FmaLane(state, inst, false); - hi = EmitPackedF16FmaLane(state, inst, true); - break; - default: - lo = ConstantF32(state, 0); - hi = ConstantF32(state, 0); - break; - } - - lo = ApplyResultModifiersF32(state, lo, inst.dst); - hi = ApplyResultModifiersF32(state, hi, inst.dst); - const auto pair = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, pair, lo, hi}); - state.builder.AddFunction( - {OpExtInst, state.uint_type, ret, state.glsl_std450, GlslPackHalf2x16, pair}); - EmitStoreU32(state, inst.dst, ret); -} - -uint32_t EmitPackedInteger16Select(EmitterState& state, uint32_t lhs, uint32_t rhs, - bool signed_compare, bool max_value) { - const auto cond = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - uint32_t op = 0; - if (signed_compare) { - op = max_value ? OpSGreaterThanEqual : OpSLessThan; - } else { - op = max_value ? OpUGreaterThanEqual : OpULessThan; - } - state.builder.AddFunction({op, state.bool_type, cond, lhs, rhs}); - state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs}); - return ret; -} - -uint32_t EmitPackedInteger16Lane(EmitterState& state, const IR::Instruction& inst, bool high_lane) { - const auto opcode = inst.op; - if (opcode == IR::Opcode::PackedLshlrevB16 || opcode == IR::Opcode::PackedLshrrevB16 || - opcode == IR::Opcode::PackedAshrrevI16) { - const auto count = - EmitAndConstant(state, EmitU16LaneBits(state, inst.src[0], high_lane), 0xfu); - const auto arithmetic = opcode == IR::Opcode::PackedAshrrevI16; - const auto value = EmitU16LaneBits(state, inst.src[1], high_lane, arithmetic); - const auto ret = state.builder.AllocateId(); - uint32_t shift_op = OpShiftRightLogical; - switch (opcode) { - case IR::Opcode::PackedLshlrevB16: shift_op = OpShiftLeftLogical; break; - case IR::Opcode::PackedAshrrevI16: shift_op = OpShiftRightArithmetic; break; - default: break; - } - state.builder.AddFunction({shift_op, state.uint_type, ret, value, count}); - return ret; - } - - const auto signed_compare = - opcode == IR::Opcode::PackedMaxI16 || opcode == IR::Opcode::PackedMinI16; - const auto lhs = EmitU16LaneBits(state, inst.src[0], high_lane, signed_compare); - const auto rhs = EmitU16LaneBits(state, inst.src[1], high_lane, signed_compare); - - switch (opcode) { - case IR::Opcode::PackedMadI16: - case IR::Opcode::PackedMadU16: { - const auto addend = - EmitU16LaneBits(state, inst.src[2], high_lane, opcode == IR::Opcode::PackedMadI16); - const auto mul = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpIMul, state.uint_type, mul, lhs, rhs}); - state.builder.AddFunction({OpIAdd, state.uint_type, ret, mul, addend}); - return ret; - } - case IR::Opcode::PackedMulLoU16: { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpIMul, state.uint_type, ret, lhs, rhs}); - return ret; - } - case IR::Opcode::PackedAddI16: - case IR::Opcode::PackedAddU16: { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpIAdd, state.uint_type, ret, lhs, rhs}); - return ret; - } - case IR::Opcode::PackedSubI16: - case IR::Opcode::PackedSubU16: { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpISub, state.uint_type, ret, lhs, rhs}); - return ret; - } - case IR::Opcode::PackedMaxI16: - case IR::Opcode::PackedMaxU16: - return EmitPackedInteger16Select(state, lhs, rhs, signed_compare, true); - case IR::Opcode::PackedMinI16: - case IR::Opcode::PackedMinU16: - return EmitPackedInteger16Select(state, lhs, rhs, signed_compare, false); - default: break; - } - - return ConstantU32(state, 0); -} - -void EmitPackedInteger16(EmitterState& state, const IR::Instruction& inst) { - const auto lo = EmitPackedInteger16Lane(state, inst, false); - const auto hi = EmitPackedInteger16Lane(state, inst, true); - const auto lo_bits = EmitAndConstant(state, lo, 0xffffu); - const auto hi_bits = EmitShiftLeftConstant(state, EmitAndConstant(state, hi, 0xffffu), 16); - EmitStoreU32(state, inst.dst, EmitOrU32(state, lo_bits, hi_bits)); -} - -void EmitMadMixF16(EmitterState& state, const IR::Instruction& inst) { - const auto src0 = EmitMixF32Load(state, inst.src[0]); - const auto src1 = EmitMixF32Load(state, inst.src[1]); - const auto src2 = EmitMixF32Load(state, inst.src[2]); - const auto fma = state.builder.AllocateId(); - const auto pair = state.builder.AllocateId(); - const auto packed = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, fma, state.glsl_std450, GlslFma, src0, src1, src2}); - const auto result = ApplyResultModifiersF32(state, fma, inst.dst); - state.builder.AddFunction( - {OpCompositeConstruct, state.vec2_float_type, pair, result, ConstantF32(state, 0)}); - state.builder.AddFunction( - {OpExtInst, state.uint_type, packed, state.glsl_std450, GlslPackHalf2x16, pair}); - EmitStoreU32(state, inst.dst, packed); -} - -void EmitMinMax3F32(EmitterState& state, const IR::Instruction& inst, bool max_value) { - const auto src0 = EmitFloatLoad(state, inst.src[0]); - const auto src1 = EmitFloatLoad(state, inst.src[1]); - const auto src2 = EmitFloatLoad(state, inst.src[2]); - const auto tmp = EmitMinMaxF32Value(state, src0, src1, max_value); - const auto base = EmitMinMaxF32Value(state, tmp, src2, max_value); - const auto f32 = ApplyResultModifiersF32(state, base, inst.dst); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, f32}); - EmitStoreU32(state, inst.dst, u32); -} - -void EmitMed3F32(EmitterState& state, const IR::Instruction& inst) { - const auto src0 = EmitFloatLoad(state, inst.src[0]); - const auto src1 = EmitFloatLoad(state, inst.src[1]); - const auto src2 = EmitFloatLoad(state, inst.src[2]); - const auto low_pair = EmitMinMaxF32Value(state, src0, src1, false); - const auto min3 = EmitMinMaxF32Value(state, low_pair, src2, false); - const auto high_pair = EmitMinMaxF32Value(state, src0, src1, true); - const auto high_min = EmitMinMaxF32Value(state, high_pair, src2, false); - const auto median = EmitMinMaxF32Value(state, low_pair, high_min, true); - const auto nan01 = EmitLogicalOrBool(state, EmitClassifyF32(state, src0).nan, - EmitClassifyF32(state, src1).nan); - const auto any_nan = EmitLogicalOrBool(state, nan01, EmitClassifyF32(state, src2).nan); - const auto base = EmitFSelectValue(state, any_nan, min3, median); - const auto f32 = ApplyResultModifiersF32(state, base, inst.dst); - const auto u32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, u32, f32}); - EmitStoreU32(state, inst.dst, u32); -} - -} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index 814abdcff..08b504c5f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -46,34 +46,10 @@ void CollectRegister(std::vector& registers, IR::Register reg) registers.push_back({reg, 0}); } -IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index) { - IR::Operand operand; - operand.kind = IR::OperandKind::Register; - operand.reg = {file, index}; - return operand; -} - -IR::Register SccRegister() { - return {IR::RegisterFile::Scc, static_cast(Decoder::OperandKind::Scc)}; -} - -IR::Operand SccOperand() { - return MakeRegisterOperand(IR::RegisterFile::Scc, - static_cast(Decoder::OperandKind::Scc)); -} - bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg) { return state.wave_size == 32u && reg.file == IR::RegisterFile::Exec && reg.index == 1u; } -bool IsMaskRegisterFile(IR::RegisterFile file) { - return file == IR::RegisterFile::Exec || file == IR::RegisterFile::Vcc; -} - -bool IsSccOperand(const IR::Operand& operand) { - return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc; -} - bool HasOutput(const std::vector& outputs, IR::StageOutputKind kind, uint32_t index) { return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) { @@ -277,9 +253,6 @@ uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) { } uint32_t ConstantU32(EmitterState& state, uint32_t value); -void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value); -uint32_t EmitSubgroupLocalInvocationId(EmitterState& state); -uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane); [[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state, IR::DescriptorBindingKind kind, uint32_t resource, const char* reason) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterExport.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterExport.cpp deleted file mode 100644 index 53a127187..000000000 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterExport.cpp +++ /dev/null @@ -1,226 +0,0 @@ -#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" - -namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { - -uint32_t EmitExportComponentF32(EmitterState& state, const IR::Instruction& inst, - uint32_t component) { - const bool enabled = ((inst.export_info.en >> component) & 1u) != 0; - if (!enabled || component >= inst.src_count || component >= 4u) { - return ConstantF32(state, component == 3u ? 0x3f800000u : 0u); - } - return EmitFloatLoad(state, inst.src[component]); -} - -uint32_t EmitExportVec4F32(EmitterState& state, const IR::Instruction& inst) { - if (inst.export_info.compr) { - uint32_t components[4] = { - ConstantF32(state, 0u), - ConstantF32(state, 0u), - ConstantF32(state, 0u), - ConstantF32(state, 0x3f800000u), - }; - for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count; - pair_index++) { - const auto raw = EmitValueLoad(state, inst.src[pair_index]); - const auto unpacked = state.builder.AllocateId(); - state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked, - state.glsl_std450, GlslUnpackHalf2x16, raw}); - for (uint32_t lane = 0; lane < 2u; lane++) { - const auto component = pair_index * 2u + lane; - if (((inst.export_info.en >> component) & 1u) == 0) { - continue; - } - components[component] = state.builder.AllocateId(); - state.builder.AddFunction( - {OpCompositeExtract, state.float_type, components[component], unpacked, lane}); - } - } - const auto vec = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, components[0], - components[1], components[2], components[3]}); - return vec; - } - - const auto x = EmitExportComponentF32(state, inst, 0); - const auto y = EmitExportComponentF32(state, inst, 1); - const auto z = EmitExportComponentF32(state, inst, 2); - const auto w = EmitExportComponentF32(state, inst, 3); - const auto vec = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, x, y, z, w}); - return vec; -} - -uint32_t EmitExportComponentU32(EmitterState& state, const IR::Instruction& inst, - uint32_t component) { - const bool enabled = ((inst.export_info.en >> component) & 1u) != 0; - if (!enabled || component >= inst.src_count || component >= 4u) { - return ConstantU32(state, component == 3u ? 1u : 0u); - } - return EmitValueLoad(state, inst.src[component]); -} - -uint32_t EmitExportVec4U32(EmitterState& state, const IR::Instruction& inst) { - uint32_t components[4] = { - ConstantU32(state, 0u), - ConstantU32(state, 0u), - ConstantU32(state, 0u), - ConstantU32(state, 1u), - }; - - if (inst.export_info.compr) { - for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count; - pair_index++) { - const auto raw = EmitValueLoad(state, inst.src[pair_index]); - for (uint32_t lane = 0; lane < 2u; lane++) { - const auto component = pair_index * 2u + lane; - if (((inst.export_info.en >> component) & 1u) == 0) { - continue; - } - components[component] = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitFieldUExtract, state.uint_type, components[component], raw, - ConstantU32(state, lane * 16u), ConstantU32(state, 16u)}); - } - } - } else { - for (uint32_t component = 0; component < 4u; component++) { - components[component] = EmitExportComponentU32(state, inst, component); - } - } - - const auto vec = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, vec, components[0], - components[1], components[2], components[3]}); - return vec; -} - -static bool MrtUsesUintOutput(const EmitterState& state, const IR::Instruction& inst) { - return state.stage == ShaderType::Pixel && inst.export_info.kind == IR::ExportTargetKind::Mrt && - inst.export_info.index < std::size(state.input_info.pixel->target_output_mode) && - state.input_info.pixel->target_output_mode[inst.export_info.index] == 7u; -} - -uint32_t ApplyMrtExportMapping(EmitterState& state, const IR::Instruction& inst, uint32_t value, - uint32_t vector_type) { - if (state.stage != ShaderType::Pixel || inst.export_info.kind != IR::ExportTargetKind::Mrt || - inst.export_info.index >= state.input_info.pixel->target_export_mapping.size()) { - return value; - } - - const auto mapping = state.input_info.pixel->target_export_mapping[inst.export_info.index]; - if (mapping.IsIdentity()) { - return value; - } - - const auto mapped = state.builder.AllocateId(); - state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value, mapping.Map(0), - mapping.Map(1), mapping.Map(2), mapping.Map(3)}); - return mapped; -} - -bool ExportWritesData(const IR::Instruction& inst) { - switch (inst.export_info.kind) { - case IR::ExportTargetKind::Null: - case IR::ExportTargetKind::Primitive: return false; - case IR::ExportTargetKind::MrtZ: return (inst.export_info.en & 0x5u) != 0; - default: return inst.export_info.en != 0; - } -} - -void EmitMrtZExport(EmitterState& state, const IR::Instruction& inst) { - if ((inst.export_info.en & 0x1u) != 0 && state.depth_variable != 0) { - state.builder.AddFunction( - {OpStore, state.depth_variable, EmitExportComponentF32(state, inst, 0)}); - } - - if ((inst.export_info.en & 0x4u) != 0 && state.sample_mask_variable != 0) { - const auto raw = - inst.src_count > 2u ? EmitValueLoad(state, inst.src[2]) : ConstantU32(state, 0); - const auto mask = state.builder.AllocateId(); - const auto ptr = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, mask, raw}); - state.builder.AddFunction({OpAccessChain, state.ptr_output_int, ptr, - state.sample_mask_variable, ConstantU32(state, 0)}); - state.builder.AddFunction({OpStore, ptr, mask}); - } -} - -void EmitExport(EmitterState& state, const IR::Instruction& inst) { - if (inst.export_info.kind == IR::ExportTargetKind::Null || - inst.export_info.kind == IR::ExportTargetKind::Primitive) { - return; - } - - if (!ExportWritesData(inst)) { - return; - } - - if (inst.export_info.kind == IR::ExportTargetKind::MrtZ) { - EmitMrtZExport(state, inst); - return; - } - - const auto variable = OutputVariableForExport(state, inst.export_info); - if (variable == 0) { - return; - } - - const auto uint_output = MrtUsesUintOutput(state, inst); - const auto vector_type = uint_output ? state.vec4_uint_type : state.vec4_float_type; - const auto value = ApplyMrtExportMapping( - state, inst, uint_output ? EmitExportVec4U32(state, inst) : EmitExportVec4F32(state, inst), - vector_type); - if (inst.export_info.kind == IR::ExportTargetKind::Position) { - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction( - {OpAccessChain, state.ptr_output_vec4_float, pointer, variable, ConstantU32(state, 0)}); - state.builder.AddFunction({OpStore, pointer, value}); - return; - } - - state.builder.AddFunction({OpStore, variable, value}); -} - -bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst) { - return state.stage == ShaderType::Pixel && inst.export_info.vm && state.needs_pixel_valid_mask; -} - -void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) { - const auto kill_label = state.builder.AllocateId(); - const auto merge_label = state.builder.AllocateId(); - const auto inactive = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active}); - state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); - state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label}); - state.builder.AddFunction({OpLabel, kill_label}); - state.builder.AddFunction({OpKill}); - state.builder.AddFunction({OpLabel, merge_label}); -} - -void EmitUpdatePixelValidMask(EmitterState& state) { - if (state.pixel_valid_mask_variable == 0) { - return; - } - - const auto active = EmitExecActiveBool(state); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpSelect, state.uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)}); - state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value}); -} - -void EmitKillIfPixelValidMaskInactive(EmitterState& state) { - if (state.pixel_valid_mask_variable == 0) { - return; - } - - const auto mask_value = state.builder.AllocateId(); - const auto active = state.builder.AllocateId(); - state.builder.AddFunction( - {OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)}); - EmitKillIfBoolFalse(state, active); -} - -} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlowOps.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlowOps.cpp deleted file mode 100644 index 0d89eb8a3..000000000 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlowOps.cpp +++ /dev/null @@ -1,379 +0,0 @@ -#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" - -namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { - -void EmitMoveU32(EmitterState& state, const IR::Instruction& inst) { - const auto value = EmitValueLoad(state, inst.src[0]); - EmitStoreU32(state, inst.dst, value); -} - -void EmitMoveF32Bits(EmitterState& state, const IR::Instruction& inst) { - const auto value = EmitFloatLoad(state, inst.src[0]); - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, value}); - EmitStoreU32(state, inst.dst, bits); -} - -uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base) { - uint32_t limit = base.index + 1u; - for (const auto& binding: state.registers) { - if (binding.reg.file == IR::RegisterFile::Vector) { - limit = std::max(limit, binding.reg.index + 1u); - } - } - return std::min(limit, 256u); -} - -void EmitMoveRelSourceU32(EmitterState& state, const IR::Instruction& inst) { - if (inst.src[0].kind != IR::OperandKind::Register || - inst.src[0].reg.file != IR::RegisterFile::Vector) { - EmitMoveU32(state, inst); - return; - } - - const auto base_reg = inst.src[0].reg; - const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0); - auto selected = EmitRegisterLoad(state, base_reg); - const auto limit = MoveRelRegisterLimit(state, base_reg); - for (uint32_t reg = base_reg.index + 1u; reg < limit; reg++) { - const auto candidate = EmitRegisterLoad(state, {IR::RegisterFile::Vector, reg}); - const auto match = state.builder.AllocateId(); - const auto next = state.builder.AllocateId(); - state.builder.AddFunction( - {OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)}); - state.builder.AddFunction({OpSelect, state.uint_type, next, match, candidate, selected}); - selected = next; - } - EmitStoreU32(state, inst.dst, selected); -} - -void EmitMoveRelDestU32(EmitterState& state, const IR::Instruction& inst) { - if (inst.dst.kind != IR::OperandKind::Register || - inst.dst.reg.file != IR::RegisterFile::Vector) { - EmitMoveU32(state, inst); - return; - } - - const auto base_reg = inst.dst.reg; - const auto value = EmitValueLoad(state, inst.src[0]); - const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0); - const auto exec_active = EmitExecActiveBool(state); - const auto limit = MoveRelRegisterLimit(state, base_reg); - for (uint32_t reg = base_reg.index; reg < limit; reg++) { - const auto pointer = PointerForRegister(state, {IR::RegisterFile::Vector, reg}); - if (pointer == 0) { - continue; - } - const auto old_value = state.builder.AllocateId(); - const auto match = state.builder.AllocateId(); - const auto write = state.builder.AllocateId(); - const auto merged = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer}); - state.builder.AddFunction( - {OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)}); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, write, exec_active, match}); - state.builder.AddFunction({OpSelect, state.uint_type, merged, write, value, old_value}); - state.builder.AddFunction({OpStore, pointer, merged}); - } -} - -void EmitMoveU64(EmitterState& state, const IR::Instruction& inst) { - if (state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register && - IsMaskRegisterFile(inst.dst.reg.file)) { - EmitPerInvocationMask(state, inst.dst, EmitLaneMaskOperandActiveBool(state, inst.src[0])); - return; - } - const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); - EmitStoreU32(state, inst.dst, low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high); -} - -uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) { - uint32_t ret = ConstantU32(state, 0); - for (uint32_t i = 0; i < 8u; i++) { - const auto mask = 0x0fu << (i * 4u); - const auto masked = state.builder.AllocateId(); - const auto non_zero = state.builder.AllocateId(); - const auto expanded = state.builder.AllocateId(); - const auto combined = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)}); - state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero, - ConstantU32(state, mask), ConstantU32(state, 0)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded}); - ret = combined; - } - return ret; -} - -void EmitWqmB64(EmitterState& state, const IR::Instruction& inst) { - if (!state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register && - inst.dst.reg.file == IR::RegisterFile::Scalar) { - const auto low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto high = EmitSequentialValueLoad(state, inst.src[0], 1); - const auto ret_low = EmitWqmLaneU32(state, low); - const auto ret_high = EmitWqmLaneU32(state, high); - EmitStoreU32(state, inst.dst, ret_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high); - return; - } - const auto ballot = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot, - ConstantU32(state, ScopeSubgroup), - EmitLaneMaskOperandActiveBool(state, inst.src[0])}); - const auto low = state.builder.AllocateId(); - const auto high = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0}); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1}); - const auto wqm_low = EmitWqmLaneU32(state, low); - const auto wqm_high = EmitWqmLaneU32(state, high); - const auto lane = EmitSubgroupLocalInvocationId(state); - const auto high_half = state.builder.AllocateId(); - const auto mask = state.builder.AllocateId(); - const auto bit_index = state.builder.AllocateId(); - const auto bit = state.builder.AllocateId(); - const auto hit = state.builder.AllocateId(); - const auto active = state.builder.AllocateId(); - state.builder.AddFunction( - {OpUGreaterThanEqual, state.bool_type, high_half, lane, ConstantU32(state, 32)}); - state.builder.AddFunction({OpSelect, state.uint_type, mask, high_half, wqm_high, wqm_low}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, bit_index, lane, ConstantU32(state, 31)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_index}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit}); - state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)}); - if (state.per_invocation_masks) { - EmitPerInvocationMask(state, inst.dst, active); - } else { - const auto result = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.uint_type, result, active, ConstantU32(state, 1), - ConstantU32(state, 0)}); - EmitStoreU32(state, inst.dst, result); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ConstantU32(state, 0)); - } -} - -void EmitSaveexecPerInvocation(EmitterState& state, const IR::Instruction& inst) { - const auto old_exec = EmitExecActiveBool(state); - const auto src = EmitLaneMaskOperandActiveBool(state, inst.src[0]); - EmitPerInvocationMask(state, inst.dst, old_exec); - - uint32_t lhs = old_exec; - uint32_t rhs = src; - bool use_or = false; - switch (inst.saveexec_mode) { - case IR::SaveexecMode::And: break; - case IR::SaveexecMode::Andn1: rhs = EmitLogicalNotBool(state, src); break; - case IR::SaveexecMode::Orn2: - lhs = EmitLogicalNotBool(state, old_exec); - use_or = true; - break; - } - const auto result = - use_or ? EmitLogicalOrBool(state, lhs, rhs) : EmitLogicalAndBool(state, lhs, rhs); - EmitPerInvocationMask(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), result); - EmitStoreSccBool(state, result); -} - -void EmitSaveexecB32(EmitterState& state, const IR::Instruction& inst) { - if (state.per_invocation_masks) { - EmitSaveexecPerInvocation(state, inst); - return; - } - const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0}); - const auto src_low = EmitValueLoad(state, inst.src[0]); - - EmitStoreU32(state, inst.dst, old_low); - - uint32_t rhs_low = src_low; - if (inst.saveexec_mode == IR::SaveexecMode::Andn1) { - rhs_low = state.builder.AllocateId(); - state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low}); - } - - const auto new_low = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, new_low, rhs_low, old_low}); - EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low); - - if (state.wave_size == 32u) { - EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), ConstantU32(state, 0)); - } - - const auto cond = state.builder.AllocateId(); - const auto scc = state.builder.AllocateId(); - state.builder.AddFunction({OpINotEqual, state.bool_type, cond, new_low, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)}); - EmitStoreU32(state, SccOperand(), scc); -} - -void EmitSaveexecB64(EmitterState& state, const IR::Instruction& inst) { - if (state.per_invocation_masks) { - EmitSaveexecPerInvocation(state, inst); - return; - } - const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0}); - const auto old_high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1}); - const auto src_low = EmitSequentialValueLoad(state, inst.src[0], 0); - const auto src_high = EmitSequentialValueLoad(state, inst.src[0], 1); - - EmitStoreU32(state, inst.dst, old_low); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), old_high); - - uint32_t lhs_low = old_low; - uint32_t lhs_high = old_high; - uint32_t rhs_low = src_low; - uint32_t rhs_high = src_high; - uint32_t opcode = OpBitwiseAnd; - if (inst.saveexec_mode == IR::SaveexecMode::Andn1) { - rhs_low = state.builder.AllocateId(); - rhs_high = state.builder.AllocateId(); - state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low}); - state.builder.AddFunction({OpNot, state.uint_type, rhs_high, src_high}); - } - if (inst.saveexec_mode == IR::SaveexecMode::Orn2) { - lhs_low = state.builder.AllocateId(); - lhs_high = state.builder.AllocateId(); - state.builder.AddFunction({OpNot, state.uint_type, lhs_low, old_low}); - state.builder.AddFunction({OpNot, state.uint_type, lhs_high, old_high}); - opcode = OpBitwiseOr; - } - - const auto new_low = state.builder.AllocateId(); - const auto new_high = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, new_low, lhs_low, rhs_low}); - state.builder.AddFunction({opcode, state.uint_type, new_high, lhs_high, rhs_high}); - EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low); - EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), new_high); - - const auto active_new_high = state.wave_size == 32u ? ConstantU32(state, 0) : new_high; - const auto mask = state.builder.AllocateId(); - const auto cond = state.builder.AllocateId(); - const auto scc = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, mask, new_low, active_new_high}); - state.builder.AddFunction({OpINotEqual, state.bool_type, cond, mask, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)}); - EmitStoreU32(state, SccOperand(), scc); -} - -void EmitReadFirstLaneU32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto active = EmitExecActiveBool(state); - const auto ballot = state.builder.AllocateId(); - const auto first_lane = state.builder.AllocateId(); - const auto first_value = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot, - ConstantU32(state, ScopeSubgroup), active}); - state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first_lane, - ConstantU32(state, ScopeSubgroup), ballot}); - state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, first_value, - ConstantU32(state, ScopeSubgroup), src, first_lane}); - EmitStoreU32(state, inst.dst, first_value); -} - -uint32_t EmitLaneIndex(EmitterState& state, const IR::Operand& operand) { - const auto lane = state.builder.AllocateId(); - const auto mask = state.wave_size == 32u ? 31u : 63u; - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, EmitValueLoad(state, operand), - ConstantU32(state, mask)}); - return lane; -} - -void EmitReadLaneU32(EmitterState& state, const IR::Instruction& inst) { - const auto src = EmitValueLoad(state, inst.src[0]); - const auto lane = EmitLaneIndex(state, inst.src[1]); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value, - ConstantU32(state, ScopeSubgroup), src, lane}); - EmitStoreU32(state, inst.dst, value); -} - -void EmitWriteLaneU32(EmitterState& state, const IR::Instruction& inst) { - const auto value = EmitValueLoad(state, inst.src[0]); - const auto lane = EmitLaneIndex(state, inst.src[1]); - const auto subid = EmitSubgroupLocalInvocationId(state); - const auto hit = state.builder.AllocateId(); - const auto pointer = - inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(state, inst.dst.reg) : 0; - const auto old = pointer != 0 ? state.builder.AllocateId() : ConstantU32(state, 0); - const auto merged = state.builder.AllocateId(); - if (pointer != 0) { - state.builder.AddFunction({OpLoad, state.uint_type, old, pointer}); - } - state.builder.AddFunction({OpIEqual, state.bool_type, hit, subid, lane}); - state.builder.AddFunction({OpSelect, state.uint_type, merged, hit, value, old}); - if (pointer != 0) { - // RDNA2 V_WRITELANE_B32 ignores EXEC, unlike ordinary VGPR destination writes. - state.builder.AddFunction({OpStore, pointer, merged}); - } -} - -void EmitPermlaneB32(EmitterState& state, const IR::Instruction& inst, bool x16) { - const auto subid = EmitSubgroupLocalInvocationId(state); - const auto row = state.builder.AllocateId(); - const auto lane = state.builder.AllocateId(); - const auto lane8 = state.builder.AllocateId(); - const auto shift = state.builder.AllocateId(); - const auto upper = state.builder.AllocateId(); - const auto selected = state.builder.AllocateId(); - const auto index0 = state.builder.AllocateId(); - const auto index1 = state.builder.AllocateId(); - const auto target = state.builder.AllocateId(); - const auto shuffled = state.builder.AllocateId(); - const auto value = EmitValueLoad(state, inst.src[0]); - const auto src1 = EmitValueLoad(state, inst.src[1]); - const auto src2 = EmitValueLoad(state, inst.src[2]); - uint32_t row_value = row; - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)}); - if (x16) { - row_value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)}); - } - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)}); - state.builder.AddFunction( - {OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)}); - state.builder.AddFunction({OpSelect, state.uint_type, selected, upper, src2, src1}); - state.builder.AddFunction({OpShiftRightLogical, state.uint_type, index0, selected, shift}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, index1, index0, ConstantU32(state, 15)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index1}); - state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled, - ConstantU32(state, ScopeSubgroup), value, target}); - uint32_t ret = shuffled; - if (!inst.dst.op_sel) { - const auto source_active = EmitLaneIndexActiveBool(state, target); - ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpSelect, state.uint_type, ret, source_active, shuffled, ConstantU32(state, 0)}); - } - EmitStoreU32(state, inst.dst, ret); -} - -void EmitControlNop(EmitterState& state, const IR::Instruction& inst) { - (void)state; - (void)inst; -} - -void EmitWaitcnt(EmitterState& state, const IR::Instruction& inst) { - (void)state; - (void)inst; -} - -void EmitBarrier(EmitterState& state, const IR::Instruction& inst) { - (void)inst; - const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; - state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup), - ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)}); -} - -} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageHelpers.cpp deleted file mode 100644 index 026866500..000000000 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageHelpers.cpp +++ /dev/null @@ -1,333 +0,0 @@ -#include "common/assert.h" -#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" - -namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { -namespace { - -uint32_t EmitCubeAxisF32(EmitterState& state, uint32_t value) { - const auto normalized = state.builder.AllocateId(); - state.builder.AddFunction( - {OpFSub, state.float_type, normalized, value, ConstantF32(state, 0x3f800000u)}); - return normalized; -} - -uint32_t EmitCubeLayerF32(EmitterState& state, uint32_t face_id) { - // Sampled RDNA2 cubemaps encode face_id as slice * 8 + face. The native - // 2D-array view stores six contiguous faces per slice, so remove the two - // reserved face IDs from every preceding slice. - const auto guest_layer = state.builder.AllocateId(); - const auto slice = state.builder.AllocateId(); - const auto padding = state.builder.AllocateId(); - const auto host_layer = state.builder.AllocateId(); - const auto result = state.builder.AllocateId(); - state.builder.AddFunction({OpConvertFToU, state.uint_type, guest_layer, face_id}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, slice, guest_layer, ConstantU32(state, 3)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, padding, slice, ConstantU32(state, 1)}); - state.builder.AddFunction({OpISub, state.uint_type, host_layer, guest_layer, padding}); - state.builder.AddFunction({OpConvertUToF, state.float_type, result, host_layer}); - return result; -} - -uint32_t EmitImageCoordF32Impl(EmitterState& state, const IR::Instruction& inst, - const IR::Operand& address, uint32_t first_component, - uint32_t components) { - auto x = EmitImageAddressFloatLoad(state, inst, address, first_component); - if (components == 1u) { - return x; - } - auto y = inst.memory.image_address_components > first_component + 1u - ? EmitImageAddressFloatLoad(state, inst, address, first_component + 1u) - : EmitZeroF32(state); - if (inst.memory.image_cube) { - // RDNA2 sampled cubemap S/T coordinates are biased by +1 relative to - // normalized 2D-array coordinates. - x = EmitCubeAxisF32(state, x); - y = EmitCubeAxisF32(state, y); - } - const auto coord = state.builder.AllocateId(); - if (components == 3u) { - auto z = inst.memory.image_address_components > first_component + 2u - ? EmitImageAddressFloatLoad(state, inst, address, first_component + 2u) - : EmitZeroF32(state); - if (inst.memory.image_cube) { - z = EmitCubeLayerF32(state, z); - } - state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z}); - } else { - state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y}); - } - return coord; -} - -} // namespace - -bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag) { - return (inst.memory.image_sample_flags & flag) != 0; -} - -ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view) { - ImageSampleLayout layout; - uint32_t cursor = 0; - if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) { - layout.offset = cursor++; - } - if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) { - layout.bias = cursor++; - } - if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare)) { - layout.dref = cursor++; - } - if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) { - const auto components = ImageViewSpatialComponents(view); - layout.grad_x = cursor; - cursor += components; - layout.grad_y = cursor; - cursor += components; - } - layout.coord = cursor; - cursor += ImageViewCoordinateComponents(view); - if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod)) { - layout.lod = cursor++; - } - return layout; -} - -uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout, ImageViewKind view) { - return EmitImageCoordF32Impl(state, inst, inst.src[0], layout.coord, - ImageViewCoordinateComponents(view)); -} - -uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout) { - if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero) || - layout.lod == NoImageComponent || inst.memory.image_address_components <= layout.lod) { - return EmitZeroF32(state); - } - return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.lod); -} - -uint32_t EmitImageDrefF32(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout) { - if (layout.dref == NoImageComponent || inst.memory.image_address_components <= layout.dref) { - return EmitZeroF32(state); - } - return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.dref); -} - -uint32_t EmitImageBiasF32(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout) { - if (layout.bias == NoImageComponent || inst.memory.image_address_components <= layout.bias) { - return EmitZeroF32(state); - } - return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.bias); -} - -uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst, - uint32_t first_component, ImageViewKind view) { - const auto x = inst.memory.image_address_components > first_component - ? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component) - : EmitZeroF32(state); - const auto components = ImageViewSpatialComponents(view); - if (components == 1u) { - return x; - } - const auto y = inst.memory.image_address_components > first_component + 1u - ? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 1u) - : EmitZeroF32(state); - const auto grad = state.builder.AllocateId(); - if (components == 3u) { - const auto z = - inst.memory.image_address_components > first_component + 2u - ? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 2u) - : EmitZeroF32(state); - state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, grad, x, y, z}); - } else { - state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, grad, x, y}); - } - return grad; -} - -uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout, ImageViewKind view) { - const auto components = ImageViewSpatialComponents(view); - if (layout.offset == NoImageComponent || - inst.memory.image_address_components <= layout.offset) { - const auto zero = ConstantI32(state, 0); - if (components == 1u) { - return zero; - } - const auto ret = state.builder.AllocateId(); - if (components == 3u) { - state.builder.AddFunction( - {OpCompositeConstruct, state.vec3_int_type, ret, zero, zero, zero}); - } else { - state.builder.AddFunction({OpCompositeConstruct, state.vec2_int_type, ret, zero, zero}); - } - return ret; - } - - const auto packed_bits = EmitImageAddressValueLoad(state, inst, inst.src[0], layout.offset); - const auto packed_i32 = state.builder.AllocateId(); - const auto offset_x = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, packed_i32, packed_bits}); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_x, packed_i32, - ConstantI32(state, 0), ConstantI32(state, 6)}); - if (components == 1u) { - return offset_x; - } - const auto offset_y = state.builder.AllocateId(); - const auto offset = state.builder.AllocateId(); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_y, packed_i32, - ConstantI32(state, 8), ConstantI32(state, 6)}); - if (components == 3u) { - const auto offset_z = state.builder.AllocateId(); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_z, packed_i32, - ConstantI32(state, 16), ConstantI32(state, 6)}); - state.builder.AddFunction( - {OpCompositeConstruct, state.vec3_int_type, offset, offset_x, offset_y, offset_z}); - } else { - state.builder.AddFunction( - {OpCompositeConstruct, state.vec2_int_type, offset, offset_x, offset_y}); - } - return offset; -} - -uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst, ImageViewKind view) { - const auto x = EmitImageAddressValueLoad(state, inst, inst.src[1], 0); - const auto components = ImageViewCoordinateComponents(view); - if (components == 1u) { - return x; - } - const auto y = inst.memory.image_address_components > 1u - ? EmitImageAddressValueLoad(state, inst, inst.src[1], 1) - : ConstantU32(state, 0); - const auto coord = state.builder.AllocateId(); - if (components == 3u) { - const auto z = inst.memory.image_address_components > 2u - ? EmitImageAddressValueLoad(state, inst, inst.src[1], 2) - : ConstantU32(state, 0); - state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z}); - } else { - state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y}); - } - return coord; -} - -uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst, - ImageViewKind view) { - const auto x = EmitImageAddressValueLoad(state, inst, inst.src[0], 0); - const auto components = ImageViewCoordinateComponents(view); - if (components == 1u) { - return x; - } - const auto y = inst.memory.image_address_components > 1u - ? EmitImageAddressValueLoad(state, inst, inst.src[0], 1) - : ConstantU32(state, 0); - const auto coord = state.builder.AllocateId(); - if (components == 3u) { - const auto z = inst.memory.image_address_components > 2u - ? EmitImageAddressValueLoad(state, inst, inst.src[0], 2) - : ConstantU32(state, 0); - state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z}); - } else { - state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y}); - } - return coord; -} - -uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst, - const IR::Operand& address, ImageViewKind view) { - if (!inst.memory.image_has_mip || inst.memory.image_address_components == 0u) { - return ConstantU32(state, 0); - } - const auto lod_component = ImageViewCoordinateComponents(view); - if (inst.memory.image_address_components <= lod_component) { - return ConstantU32(state, 0); - } - return EmitImageAddressValueLoad(state, inst, address, lod_component); -} - -uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst, - ImageViewKind view) { - // OpImageQueryLod takes only the spatial coordinates, even for arrayed images. - return EmitImageCoordF32Impl(state, inst, inst.src[0], 0, ImageViewSpatialComponents(view)); -} - -uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) { - uint32_t index = 0; - for (uint32_t i = 0; i < component; i++) { - index += (dmask >> i) & 1u; - } - return index; -} - -uint32_t EmitImageStoreComponentF32(EmitterState& state, const IR::Instruction& inst, - uint32_t component) { - const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; - if (((dmask >> component) & 1u) == 0) { - return EmitZeroF32(state); - } - return EmitSequentialFloatLoad(state, inst.src[0], DmaskComponentIndex(dmask, component)); -} - -uint32_t InverseStorageSwizzleComponent(uint32_t swizzle, uint32_t component) { - const auto target = 4u + component; - for (uint32_t source = 0; source < 4u; source++) { - if (((swizzle >> (source * 3u)) & 0x7u) == target) { - return source; - } - } - return UINT32_MAX; -} - -uint32_t StorageImageSwizzle(const EmitterState& state, const IR::Instruction& inst) { - if (inst.memory.resource >= state.program.info.images.size()) { - EXIT("storage image instruction has no specialized swizzle\n"); - } - return state.program.info.images[inst.memory.resource].storage_swizzle; -} - -uint32_t EmitImageStoreTexelF32(EmitterState& state, const IR::Instruction& inst) { - const auto swizzle = StorageImageSwizzle(state, inst); - const auto component_value = [&](uint32_t component) { - const auto source = InverseStorageSwizzleComponent(swizzle, component); - return source < 4u ? EmitImageStoreComponentF32(state, inst, source) : EmitZeroF32(state); - }; - const auto x = component_value(0); - const auto y = component_value(1); - const auto z = component_value(2); - const auto w = component_value(3); - const auto texel = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, texel, x, y, z, w}); - return texel; -} - -uint32_t EmitImageStoreComponentU32(EmitterState& state, const IR::Instruction& inst, - uint32_t component) { - const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; - if (((dmask >> component) & 1u) == 0) { - return ConstantU32(state, 0); - } - return EmitSequentialValueLoad(state, inst.src[0], DmaskComponentIndex(dmask, component)); -} - -uint32_t EmitImageStoreTexelU32(EmitterState& state, const IR::Instruction& inst) { - const auto swizzle = StorageImageSwizzle(state, inst); - const auto component_value = [&](uint32_t component) { - const auto source = InverseStorageSwizzleComponent(swizzle, component); - return source < 4u ? EmitImageStoreComponentU32(state, inst, source) - : ConstantU32(state, 0); - }; - const auto x = component_value(0); - const auto y = component_value(1); - const auto z = component_value(2); - const auto w = component_value(3); - const auto texel = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, texel, x, y, z, w}); - return texel; -} - -} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageOps.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageOps.cpp deleted file mode 100644 index 9007caed0..000000000 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageOps.cpp +++ /dev/null @@ -1,342 +0,0 @@ -#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" - -namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { - -namespace { - -uint32_t ConstantVec2I32(EmitterState& state, int32_t x, int32_t y) { - const auto value = state.builder.AllocateId(); - state.builder.AddType({OpConstantComposite, state.vec2_int_type, value, ConstantI32(state, x), - ConstantI32(state, y)}); - return value; -} - -uint32_t ConstantImageGatherHorizontalOffsets(EmitterState& state, ImageViewKind view) { - if (ImageViewSpatialComponents(view) == 1u) { - const auto type = state.builder.AllocateId(); - state.builder.AddType({OpTypeArray, type, state.int_type, ConstantU32(state, 4)}); - const auto value = state.builder.AllocateId(); - state.builder.AddType({OpConstantComposite, type, value, ConstantI32(state, -1), - ConstantI32(state, 0), ConstantI32(state, 1), - ConstantI32(state, 2)}); - return value; - } - const auto offset0 = ConstantVec2I32(state, -1, 0); - const auto offset1 = ConstantVec2I32(state, 0, 0); - const auto offset2 = ConstantVec2I32(state, 1, 0); - const auto offset3 = ConstantVec2I32(state, 2, 0); - const auto type = state.builder.AllocateId(); - state.builder.AddType({OpTypeArray, type, state.vec2_int_type, ConstantU32(state, 4)}); - const auto value = state.builder.AllocateId(); - state.builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3}); - return value; -} - -uint32_t LoadStorageImageDescriptorAtIndex(EmitterState& state, uint32_t resource, - uint32_t array_index, bool uint_image, - ImageViewKind view) { - const auto kind = StorageBindingKind(uint_image, view); - const auto& descriptors = state.storage_images[StorageImageIndex(uint_image, view)]; - const auto pointer = - DescriptorElementPointer(state, descriptors.pointer_type, descriptors.variable, array_index, - kind, resource, "storage image descriptor array was not emitted"); - const auto image = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, descriptors.image_type, image, pointer}); - return image; -} - -} // namespace - -uint32_t EmitImageGetResinfoComponent(EmitterState& state, uint32_t image, uint32_t size, - uint32_t component_index, ImageViewKind view) { - const auto components = ImageViewCoordinateComponents(view); - switch (component_index) { - case 0: { - if (components == 1u) { - return size; - } - const auto width = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, width, size, 0}); - return width; - } - case 1: { - if (components < 2u) { - return ConstantU32(state, 0); - } - const auto height = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, height, size, 1}); - return height; - } - case 2: - if (components == 3u) { - const auto depth = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, depth, size, 2}); - return depth; - } - return ConstantU32(state, 0); - case 3: { - const auto levels = state.builder.AllocateId(); - state.builder.AddFunction({OpImageQueryLevels, state.uint_type, levels, image}); - return levels; - } - default: return ConstantU32(state, 0); - } -} - -void EmitImageGetResinfo(EmitterState& state, const IR::Instruction& inst) { - const auto view = SampledImageViewKind(state, inst.memory, inst.pc); - const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view); - const auto mip_level = EmitValueLoad(state, inst.src[0]); - - const auto size = state.builder.AllocateId(); - state.builder.AddFunction( - {OpImageQuerySizeLod, ImageViewSizeType(state, view), size, image, mip_level}); - - const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; - uint32_t dst_index = 0; - for (uint32_t component_index = 0; component_index < 4u; component_index++) { - if (((dmask >> component_index) & 1u) == 0) { - continue; - } - const auto component = - EmitImageGetResinfoComponent(state, image, size, component_index, view); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), component); - } -} - -void EmitImageGetLod(EmitterState& state, const IR::Instruction& inst) { - const auto view = SampledImageViewKind(state, inst.memory, inst.pc); - const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view); - - const auto lod = state.builder.AllocateId(); - state.builder.AddFunction({OpImageQueryLod, state.vec2_float_type, lod, sampled_image, - EmitImageQueryCoordF32(state, inst, view)}); - - const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; - uint32_t dst_index = 0; - for (uint32_t component_index = 0; component_index < 2u; component_index++) { - if (((dmask >> component_index) & 1u) == 0) { - continue; - } - const auto component = state.builder.AllocateId(); - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction( - {OpCompositeExtract, state.float_type, component, lod, component_index}); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, component}); - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits); - } -} - -void EmitImageLoad(EmitterState& state, const IR::Instruction& inst) { - const auto view = SampledImageViewKind(state, inst.memory, inst.pc); - const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view); - const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint; - - const auto color = state.builder.AllocateId(); - const auto coord = EmitImageLoadCoordU32(state, inst, view); - if (ImageSpirvMultisampled(view) != 0) { - const auto sample = EmitImageAddressValueLoad(state, inst, inst.src[0], - ImageViewCoordinateComponents(view)); - state.builder.AddFunction({OpImageFetch, - integer ? state.vec4_uint_type : state.vec4_float_type, color, - image, coord, ImageOperandsSampleMask, sample}); - } else { - state.builder.AddFunction( - {OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type, color, image, - coord, ImageOperandsLodMask, EmitImageMipLodU32(state, inst, inst.src[0], view)}); - } - - const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; - uint32_t dst_index = 0; - for (uint32_t component_index = 0; component_index < 4u; component_index++) { - if (((dmask >> component_index) & 1u) == 0) { - continue; - } - const auto component = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type, - component, color, component_index}); - const auto bits = integer ? component : state.builder.AllocateId(); - if (!integer) { - state.builder.AddFunction({OpBitcast, state.uint_type, bits, component}); - } - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits); - } -} - -void EmitImageStore(EmitterState& state, const IR::Instruction& inst) { - const auto uint_image = inst.memory.kind == IR::ResourceKind::StorageImageUint; - const auto view = StorageImageViewKind(state, inst.memory, uint_image, inst.pc); - const auto binding = - ResourceForDescriptor(state, StorageBindingKind(uint_image, view), inst.memory.resource); - const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource, - binding.array_index, uint_image, view); - - state.builder.AddFunction( - {OpImageWrite, image, EmitImageCoordU32(state, inst, view), - uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)}); -} - -void EmitImageSampleResult(EmitterState& state, const IR::Instruction& inst, uint32_t sample, - bool dref, bool integer) { - if (dref) { - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, sample}); - for (uint32_t i = 0; i < inst.memory.data_dwords; i++) { - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits); - } - return; - } - - const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u; - uint32_t dst_index = 0; - for (uint32_t component_index = 0; component_index < 4u; component_index++) { - if (((dmask >> component_index) & 1u) == 0) { - continue; - } - const auto component = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type, - component, sample, component_index}); - const auto bits = integer ? component : state.builder.AllocateId(); - if (!integer) { - state.builder.AddFunction({OpBitcast, state.uint_type, bits, component}); - } - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits); - } -} - -bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst) { - // RDNA2 plain IMAGE_SAMPLE is derivative-based in pixel shaders. Do not translate it - // to Lod(0): only _L/_LZ/_D variants supply explicit LOD/gradient information. - // SPIR-V implicit-lod sampling is fragment-only, so non-pixel stages still need - // an explicit operand even for the unsuffixed sample opcode. - if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative) || - HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod) || - HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero)) { - return true; - } - return state.stage != ShaderType::Pixel; -} - -void AddImageSampleOperands(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout, bool explicit_lod, - std::vector& words) { - uint32_t mask = 0; - std::vector operands; - - // Keep the SPIR-V operand class aligned with the RDNA2 opcode suffix. Grad/Lod - // operands belong to explicit-lod instructions; bias belongs to implicit-lod - // sampling because it modifies the hardware-computed LOD. - if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) { - mask |= ImageOperandsGradMask; - const auto view = SampledImageViewKind(state, inst.memory, inst.pc); - operands.push_back(EmitImageGradientF32(state, inst, layout.grad_x, view)); - operands.push_back(EmitImageGradientF32(state, inst, layout.grad_y, view)); - } else if (explicit_lod) { - mask |= ImageOperandsLodMask; - operands.push_back(EmitImageLodF32(state, inst, layout)); - } else if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) { - mask |= ImageOperandsBiasMask; - operands.push_back(EmitImageBiasF32(state, inst, layout)); - } - if (mask != 0) { - words.push_back(mask); - words.insert(words.end(), operands.begin(), operands.end()); - } -} - -uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst) { - const auto flags = inst.memory.image_sample_flags; - const bool dref = (flags & Decoder::ImageSampleFlagCompare) != 0; - const bool explicit_lod = ImageSampleNeedsExplicitLod(state, inst); - if (explicit_lod) { - return dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod; - } - return dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod; -} - -void EmitImageSample(EmitterState& state, const IR::Instruction& inst) { - const auto view = SampledImageViewKind(state, inst.memory, inst.pc); - const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view); - - const auto layout = MakeImageSampleLayout(inst, view); - const auto base_coord = EmitImageCoordF32(state, inst, layout, view); - const auto sample = state.builder.AllocateId(); - const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare); - const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint; - uint32_t result_type = state.vec4_float_type; - if (dref) { - result_type = state.float_type; - } else if (integer) { - result_type = state.vec4_uint_type; - } - const auto explicit_lod = ImageSampleNeedsExplicitLod(state, inst); - const auto opcode = ImageSampleOpcode(state, inst); - std::vector words = {opcode, result_type, sample, sampled_image, base_coord}; - if (dref) { - words.push_back(EmitImageDrefF32(state, inst, layout)); - } - AddImageSampleOperands(state, inst, layout, explicit_lod, words); - state.builder.AddFunction(words); - EmitImageSampleResult(state, inst, sample, dref, integer); -} - -uint32_t ImageGatherComponent(uint32_t dmask) { - switch (dmask) { - case 0x2u: return 1; - case 0x4u: return 2; - case 0x8u: return 3; - default: return 0; - } -} - -void AddImageGatherOperands(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout, ImageViewKind view, - std::vector& words) { - if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagGatherHorizontal)) { - words.push_back(ImageOperandsConstOffsetsMask); - words.push_back(ConstantImageGatherHorizontalOffsets(state, view)); - return; - } - if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) { - words.push_back(ImageOperandsOffsetMask); - words.push_back(EmitImagePackedOffsetI32(state, inst, layout, view)); - } -} - -void EmitImageGather4(EmitterState& state, const IR::Instruction& inst) { - const auto view = SampledImageViewKind(state, inst.memory, inst.pc); - const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view); - - const auto layout = MakeImageSampleLayout(inst, view); - const auto coord = EmitImageCoordF32(state, inst, layout, view); - const auto texels = state.builder.AllocateId(); - const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare); - const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint; - const auto result_type = integer ? state.vec4_uint_type : state.vec4_float_type; - std::vector words = - dref ? std::vector {OpImageDrefGather, - state.vec4_float_type, - texels, - sampled_image, - coord, - EmitImageDrefF32(state, inst, layout)} - : std::vector { - OpImageGather, result_type, - texels, sampled_image, - coord, ConstantU32(state, ImageGatherComponent(inst.memory.dmask))}; - AddImageGatherOperands(state, inst, layout, view, words); - state.builder.AddFunction(words); - - for (uint32_t i = 0; i < inst.memory.data_dwords; i++) { - const auto component = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type, - component, texels, i}); - const auto bits = integer ? component : state.builder.AllocateId(); - if (!integer) { - state.builder.AddFunction({OpBitcast, state.uint_type, bits, component}); - } - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits); - } -} - -} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index d9905e8b5..3e58faac6 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -599,28 +599,6 @@ struct F32Class { uint32_t quiet_bits = 0; }; -struct CubeF32Values { - uint32_t x = 0; - uint32_t y = 0; - uint32_t z = 0; - uint32_t nx = 0; - uint32_t ny = 0; - uint32_t nz = 0; - uint32_t z_face = 0; - uint32_t y_face = 0; - uint32_t x_neg = 0; - uint32_t y_neg = 0; - uint32_t z_neg = 0; -}; - -struct F16Class { - uint32_t bits = 0; - uint32_t nan = 0; - uint32_t snan = 0; - uint32_t zero = 0; - uint32_t quiet_bits = 0; -}; - uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr); uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr); @@ -645,18 +623,8 @@ void SetError(std::string* error, const char* message); void CollectRegister(std::vector& registers, IR::Register reg); -IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index); - -IR::Register SccRegister(); - -IR::Operand SccOperand(); - bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg); -bool IsMaskRegisterFile(IR::RegisterFile file); - -bool IsSccOperand(const IR::Operand& operand); - bool HasOutput(const std::vector& outputs, IR::StageOutputKind kind, uint32_t index); void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program); @@ -681,12 +649,8 @@ uint32_t PointerForRegister(const EmitterState& state, IR::Register reg); uint32_t ConstantU32(EmitterState& state, uint32_t value); -void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value); - uint32_t EmitSubgroupLocalInvocationId(EmitterState& state); -uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane); - [[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state, IR::DescriptorBindingKind kind, uint32_t resource, const char* reason); @@ -768,10 +732,6 @@ void AllocateRegisterVariables(EmitterState& state); void AllocateDescriptorVariables(EmitterState& state); -bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width); - -uint32_t EmitSdwaExtractU32(EmitterState& state, const IR::Operand& operand, uint32_t value); - uint32_t EmitTrueBool(EmitterState& state); DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control); @@ -785,34 +745,10 @@ DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control); -uint32_t EmitDppValueU32(EmitterState& state, const IR::Operand& operand, uint32_t value); - -uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand); - -uint32_t EmitRegisterLoad(EmitterState& state, IR::Register reg); - -uint32_t WaveMaskHighLoad(EmitterState& state, IR::RegisterFile file); - -uint32_t EmitLaneMaskPartU32(EmitterState& state, uint32_t part); - -uint32_t EmitThreadMaskBelowPartU32(EmitterState& state, uint32_t part); - -uint32_t EmitMaskActiveBool(EmitterState& state, IR::RegisterFile file); - -uint32_t EmitMaskZeroBool(EmitterState& state, IR::RegisterFile file, bool zero); - -uint32_t EmitMaskSummaryZeroBool(EmitterState& state, IR::RegisterFile file, bool zero); - -uint32_t EmitExecActiveBool(EmitterState& state); - -uint32_t EmitConditionBool(EmitterState& state, const IR::Operand& operand); - uint32_t EmitSubgroupLocalInvocationId(EmitterState& state); uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind); -uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location); - const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location); uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input, @@ -822,141 +758,26 @@ uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uin uint32_t EmitLocalInvocationIndex(EmitterState& state); -void EmitLoadInputF32(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitSccBool(EmitterState& state, bool non_zero); - -uint32_t EmitFloatLoad(EmitterState& state, const IR::Operand& operand); - -uint32_t ApplyResultModifiersF32(EmitterState& state, uint32_t value, const IR::Operand& dst); - -uint32_t EmitMixF32Load(EmitterState& state, const IR::Operand& operand); - -IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset); - -uint32_t EmitLaneMaskOperandActiveBool(EmitterState& state, const IR::Operand& operand); - uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t ballot, uint32_t lane); -uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane); - uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane); -uint32_t EmitSequentialValueLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset); - -uint32_t EmitSequentialFloatLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset); - -uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component); - -IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base, - uint32_t component); - -uint32_t EmitImageAddressValueLoad(EmitterState& state, const IR::Instruction& inst, - const IR::Operand& base, uint32_t component); - -uint32_t EmitImageAddressFloatLoad(EmitterState& state, const IR::Instruction& inst, - const IR::Operand& base, uint32_t component); - -uint32_t EmitZeroF32(EmitterState& state); - -bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag); - -ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view); - -uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout, ImageViewKind view); - -uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout); - -uint32_t EmitImageDrefF32(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout); - -uint32_t EmitImageBiasF32(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout); - -uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst, - uint32_t first_component, ImageViewKind view); - -uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout, ImageViewKind view); - -uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst, - ImageViewKind view = ImageViewKind::Dim2D); - -uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst, - ImageViewKind view); - -uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst, - const IR::Operand& address, ImageViewKind view); - -uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst, - ImageViewKind view); - -uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component); - -uint32_t EmitImageStoreComponentF32(EmitterState& state, const IR::Instruction& inst, - uint32_t component); - -uint32_t EmitImageStoreTexelF32(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitImageStoreTexelU32(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitDppWriteActiveBool(EmitterState& state, const IR::Operand& dst); - -uint32_t EmitSdwaDestinationMerge(EmitterState& state, uint32_t pointer, const IR::Operand& dst, - uint32_t value); - -void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value); - uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs); uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs); -uint32_t EmitNotEqualZeroBool(EmitterState& state, uint32_t value); - -uint32_t EmitSelectU32Value(EmitterState& state, uint32_t condition, uint32_t true_value, - uint32_t false_value); - uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index); bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index); -uint32_t EmitByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, - uint32_t src_count); - uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc); Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc); -bool ShouldApplyBufferAddTid(const IR::Instruction& inst); - -uint32_t EmitBufferIndexWithAddTid(EmitterState& state, const IR::Instruction& inst, uint32_t index, - uint32_t packed); - -uint32_t EmitOptionalLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs); - -bool IsStorageBufferMemoryKind(IR::ResourceKind kind); - void EmitStorageBufferOffsets(EmitterState& state); -uint32_t EmitBufferAddressFromParts(EmitterState& state, const IR::Instruction& inst, - uint32_t index, uint32_t offset, uint32_t soffset); - -uint32_t EmitBufferByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, - uint32_t src_count); - -uint32_t EmitMemoryByteAddress(EmitterState& state, const IR::Instruction& inst, - const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count); - -uint32_t EmitDwordIndex(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, - uint32_t src_count); - -uint32_t EmitMemoryDwordIndex(EmitterState& state, const IR::Instruction& inst, - const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count); - DescriptorResourceBinding StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc); @@ -982,37 +803,6 @@ uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index); uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, uint32_t use_pc); -uint32_t EmitMemoryLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst, - IR::ResourceKind kind, uint32_t first_src, uint32_t src_count); - -void EmitMemoryLoadU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, - uint32_t first_src, uint32_t src_count); - -uint32_t EmitMemoryLoadSubDwordValueU32(EmitterState& state, const IR::Instruction& inst, - IR::ResourceKind kind, uint32_t first_src, - uint32_t src_count, uint32_t data_bits, bool sign_extend); - -void EmitMemoryLoadSubDwordU32(EmitterState& state, const IR::Instruction& inst, - IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, - uint32_t data_bits, bool sign_extend); - -void EmitMemoryStoreU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, - uint32_t first_src, uint32_t src_count); - -void EmitMemoryStoreSubDwordU32(EmitterState& state, const IR::Instruction& inst, - IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, - uint32_t data_bits); - -uint32_t AddressSourceCount(const IR::Instruction& inst, uint32_t first_src); - -bool IsFormattedBufferComponent(const IR::Instruction& inst); - -Prospero::BufferFormat FormattedBufferFormat(const EmitterState& state, - const IR::Instruction& inst); - -IR::Instruction WithFormatComponentByteOffset(const IR::Instruction& inst, - Prospero::BufferFormat format); - uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value); uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value); @@ -1027,307 +817,37 @@ uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t bool IsSignedFormatComponent(Format::ComponentType type); -uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint32_t offset, - uint32_t bits, bool sign_extend); - uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw); uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits); -uint32_t EmitFormatRawComponent(EmitterState& state, const IR::Instruction& inst, - const Format::BufferFormatInfo& info); - uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info, uint32_t component, uint32_t raw); -uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst, - const Format::BufferFormatInfo& info); - -uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format, - uint32_t opcode_components); - -bool EmitBufferIntegerFormatStore(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitAtomicPointer(EmitterState& state, const IR::Instruction& inst); - void EmitDeviceAtomicMemoryBarrier(EmitterState& state); -void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitAtomicFMinF32(EmitterState& state, const IR::Instruction& inst); -void EmitAtomicFMaxF32(EmitterState& state, const IR::Instruction& inst); - -void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst); - -void EmitLoadSrtDword(EmitterState& state, const IR::Instruction& inst); - -void EmitBufferLoadUbyte(EmitterState& state, const IR::Instruction& inst); - -void EmitBufferLoadSbyte(EmitterState& state, const IR::Instruction& inst); - -void EmitBufferLoadUshort(EmitterState& state, const IR::Instruction& inst); - -void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst); - -void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst); - -void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions, - uint32_t count); - -void EmitBufferStoreDword(EmitterState& state, const IR::Instruction& inst); - -void EmitFlatLoadUbyte(EmitterState& state, const IR::Instruction& inst); - -void EmitFlatLoadSbyte(EmitterState& state, const IR::Instruction& inst); - -void EmitFlatLoadUshort(EmitterState& state, const IR::Instruction& inst); - -void EmitFlatLoadSshort(EmitterState& state, const IR::Instruction& inst); - -void EmitFlatLoadDword(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitDsAddtidDwordIndex(EmitterState& state, const IR::Instruction& inst, - uint32_t m0_src_index); - -void EmitDsWriteAddtidB32(EmitterState& state, const IR::Instruction& inst); - -void EmitDsReadAddtidB32(EmitterState& state, const IR::Instruction& inst); - -void EmitDsAppendConsume(EmitterState& state, const IR::Instruction& inst, uint32_t atomic_opcode); - -void EmitDsFloatMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool max_value); - uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control); -void EmitDsSwizzleB32(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitImageGetResinfoComponent(EmitterState& state, uint32_t image, uint32_t size, - uint32_t component_index); - -void EmitImageGetResinfo(EmitterState& state, const IR::Instruction& inst); - -void EmitImageGetLod(EmitterState& state, const IR::Instruction& inst); - -void EmitImageLoad(EmitterState& state, const IR::Instruction& inst); - -void EmitImageStore(EmitterState& state, const IR::Instruction& inst); - -void EmitImageSampleResult(EmitterState& state, const IR::Instruction& inst, uint32_t sample, - bool dref); - -bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst); - -void AddImageSampleOperands(EmitterState& state, const IR::Instruction& inst, - const ImageSampleLayout& layout, bool explicit_lod, - std::vector& words); - -uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst); - -void EmitImageSample(EmitterState& state, const IR::Instruction& inst); - -uint32_t ImageGatherComponent(uint32_t dmask); - -void EmitImageGather4(EmitterState& state, const IR::Instruction& inst); - -void EmitMoveU32(EmitterState& state, const IR::Instruction& inst); - -void EmitMoveF32Bits(EmitterState& state, const IR::Instruction& inst); - -uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base); - -void EmitMoveRelSourceU32(EmitterState& state, const IR::Instruction& inst); - -void EmitMoveRelDestU32(EmitterState& state, const IR::Instruction& inst); - -void EmitMoveU64(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src); - -void EmitWqmB64(EmitterState& state, const IR::Instruction& inst); - -void EmitSaveexecB32(EmitterState& state, const IR::Instruction& inst); - -void EmitSaveexecB64(EmitterState& state, const IR::Instruction& inst); - -void EmitReadFirstLaneU32(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitLaneIndex(EmitterState& state, const IR::Operand& operand); - -void EmitReadLaneU32(EmitterState& state, const IR::Instruction& inst); - -void EmitWriteLaneU32(EmitterState& state, const IR::Instruction& inst); - -void EmitPermlaneB32(EmitterState& state, const IR::Instruction& inst, bool x16); - -void EmitControlNop(EmitterState& state, const IR::Instruction& inst); - -void EmitWaitcnt(EmitterState& state, const IR::Instruction& inst); - -void EmitBarrier(EmitterState& state, const IR::Instruction& inst); - -void EmitAbsI32(EmitterState& state, const IR::Instruction& inst); - -void EmitUnaryU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitUnaryU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitFindLsbU32(EmitterState& state, const IR::Instruction& inst); - -void EmitFindMsbFromHighU32(EmitterState& state, const IR::Instruction& inst); - -void EmitFindMsbFromHighU64(EmitterState& state, const IR::Instruction& inst); - -void EmitBitCountU64(EmitterState& state, const IR::Instruction& inst); - -void EmitBitCountU32(EmitterState& state, const IR::Instruction& inst); - -void EmitBitCountAddU32(EmitterState& state, const IR::Instruction& inst); - -void EmitBinaryU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitBinaryNotRhsU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitBinaryThenNotU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitMaskedBitCountU32(EmitterState& state, const IR::Instruction& inst, uint32_t exec_index); - -uint32_t EmitMask5U32(EmitterState& state, uint32_t value); - -void EmitChainedBinaryU32(EmitterState& state, const IR::Instruction& inst, uint32_t first_op, - uint32_t second_op, bool mask_first_rhs = false, - bool mask_second_rhs = false); - -void EmitBinaryU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitBinaryNotRhsU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitBinaryThenNotU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitSelectU64(EmitterState& state, const IR::Instruction& inst); - uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value, uint32_t false_value); -void EmitStoreSccBool(EmitterState& state, uint32_t cond); - uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask); void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift, uint32_t& out_low, uint32_t& out_high); -void EmitShiftLeftLogicalU64(EmitterState& state, const IR::Instruction& inst); - void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift, uint32_t& out_low, uint32_t& out_high); -void EmitShiftRightLogicalU64(EmitterState& state, const IR::Instruction& inst); - -void EmitAdd3U32(EmitterState& state, const IR::Instruction& inst); - uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask); -void EmitShiftU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -uint32_t EmitU16LaneBits(EmitterState& state, const IR::Operand& operand, bool high_lane, - bool sign_extend = false); - -void EmitShiftU16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode, - bool arithmetic); - -void EmitBinaryU16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitMinMaxU16(EmitterState& state, const IR::Instruction& inst, bool signed_value, - bool max_value); - -uint32_t EmitXorConstant(EmitterState& state, uint32_t value, uint32_t mask); - AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs, uint32_t carry_in); -void EmitLaneMaskPairFromBool(EmitterState& state, const IR::Operand& dst, uint32_t value_bool); - -void EmitPerInvocationMask(EmitterState& state, const IR::Operand& dst, uint32_t value_bool); - -void EmitIAddCarryU32(EmitterState& state, const IR::Instruction& inst); - -void EmitISubBorrowU32(EmitterState& state, const IR::Instruction& inst); - -void EmitScalarAddCarryU32(EmitterState& state, const IR::Instruction& inst); - -void EmitScalarSubBorrowU32(EmitterState& state, const IR::Instruction& inst); - -void EmitScalarSubBorrowCarryU32(EmitterState& state, const IR::Instruction& inst); - -void EmitBitClearU32(EmitterState& state, const IR::Instruction& inst); - -void EmitBitSetU32(EmitterState& state, const IR::Instruction& inst); - -void EmitUMadU64U32(EmitterState& state, const IR::Instruction& inst); - -void EmitScalarSignedOverflowI32(EmitterState& state, const IR::Instruction& inst, bool subtract); - -void EmitScalarShiftLeftAddCarryU32(EmitterState& state, const IR::Instruction& inst); - -void EmitMulU24U32(EmitterState& state, const IR::Instruction& inst); - -void EmitMulI24U32(EmitterState& state, const IR::Instruction& inst); - -void EmitMulHighU32(EmitterState& state, const IR::Instruction& inst); - -void EmitMulHighI32(EmitterState& state, const IR::Instruction& inst); - -void EmitMadU32U24(EmitterState& state, const IR::Instruction& inst); - -void EmitMadI32I24(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitShiftLeftConstant(EmitterState& state, uint32_t value, uint32_t shift); - uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift); uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs); -uint32_t EmitBitReplicateHalfU32(EmitterState& state, uint32_t value); - -void EmitBitReplicateB64B32(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitBitFieldExtractConstant(EmitterState& state, uint32_t value, uint32_t offset, - uint32_t count); - -uint32_t EmitRightAlignedMaskU32(EmitterState& state, uint32_t count); - -void EmitBitFieldMaskU32(EmitterState& state, const IR::Instruction& inst); - -void EmitRightAlignedMaskU64(EmitterState& state, uint32_t count, uint32_t& out_low, - uint32_t& out_high); - -void EmitBitFieldMaskU64(EmitterState& state, const IR::Instruction& inst); - -void EmitBitFieldExtractU64(EmitterState& state, const IR::Instruction& inst); - -void EmitBitFieldExtractU32(EmitterState& state, const IR::Instruction& inst); - -void EmitBitFieldExtract3U32(EmitterState& state, const IR::Instruction& inst, bool signed_value); - -void EmitBitFieldInsertSelectU32(EmitterState& state, const IR::Instruction& inst); - -void EmitAlignBitU32(EmitterState& state, const IR::Instruction& inst); - -void EmitSelectU32(EmitterState& state, const IR::Instruction& inst); - -void EmitSelectMaskU32(EmitterState& state, const IR::Instruction& inst); - -void EmitSelectF32Bits(EmitterState& state, const IR::Instruction& inst); - -void EmitSelectMaskF32Bits(EmitterState& state, const IR::Instruction& inst); - -void EmitPackU16(EmitterState& state, const IR::Instruction& inst, bool high_src0, bool high_src1); - -void EmitPackU8F32(EmitterState& state, const IR::Instruction& inst); - -void EmitBinaryF32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitLdexpF32(EmitterState& state, const IR::Instruction& inst); - uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t constant); @@ -1335,25 +855,10 @@ uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32); -void EmitPackF32ToF16Rtz(EmitterState& state, const IR::Instruction& inst); - -void EmitPackNormalizedF32(EmitterState& state, const IR::Instruction& inst, uint32_t ext_inst); - uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value); -void EmitMinMaxU32(EmitterState& state, const IR::Instruction& inst, bool max_value); - -void EmitSadU32(EmitterState& state, const IR::Instruction& inst); - uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value); -void EmitMinMaxI32(EmitterState& state, const IR::Instruction& inst, bool max_value); - -void EmitMinMax3U32(EmitterState& state, const IR::Instruction& inst, bool signed_value, - bool max_value); - -void EmitMed3U32(EmitterState& state, const IR::Instruction& inst, bool signed_value); - uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value); uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value); @@ -1375,181 +880,18 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask); uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value); -void EmitMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool max_value); - -void EmitCompareResult(EmitterState& state, const IR::Operand& dst, uint32_t cond); - -void EmitCompareU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitCompareU16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitCompareI16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitBitCompareB32(EmitterState& state, const IR::Instruction& inst, bool bit_set); - -void EmitCompareU64(EmitterState& state, const IR::Instruction& inst); - -void EmitCompareConstant(EmitterState& state, const IR::Instruction& inst, bool value); - -void EmitCompareMaskU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitCompareF32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitCompareOrderedF32(EmitterState& state, const IR::Instruction& inst, bool ordered); - -void EmitCompareClassF32(EmitterState& state, const IR::Instruction& inst); - -void EmitCompareMaskF32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitConvertU32ToF32(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertI32ToF32(EmitterState& state, const IR::Instruction& inst); - uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value); -void EmitConvertF32ToU32(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertF32ToI32(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertF32ToF16(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertF16ToF32(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertU16ToF16(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertF16ToU16(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertI16ToF16(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertF16ToI16(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertRoundPlusInfF32ToI32(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertFloorF32ToI32(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertI4ToOffsetF32(EmitterState& state, const IR::Instruction& inst); - -void EmitConvertByteU32ToF32(EmitterState& state, const IR::Instruction& inst); - uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value); -void EmitRcpF32(EmitterState& state, const IR::Instruction& inst); - uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero); -void EmitFloatExtInst(EmitterState& state, const IR::Instruction& inst, uint32_t ext_inst, - bool scale_by_two_pi = false, bool flush_denorm = false); - -void EmitMadF32(EmitterState& state, const IR::Instruction& inst); - uint32_t EmitFNegateValue(EmitterState& state, uint32_t value); uint32_t EmitFAbsValue(EmitterState& state, uint32_t value); -uint32_t EmitFCompareValue(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs); - -uint32_t EmitLogicalAndValue(EmitterState& state, uint32_t lhs, uint32_t rhs); - -uint32_t EmitFSelectValue(EmitterState& state, uint32_t cond, uint32_t true_value, - uint32_t false_value); - -uint32_t EmitFMulConstant(EmitterState& state, uint32_t value, uint32_t bits); - -CubeF32Values EmitCubeF32Values(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitCubeIdF32(EmitterState& state, const CubeF32Values& cube); - -uint32_t EmitCubeScF32(EmitterState& state, const CubeF32Values& cube); - -uint32_t EmitCubeTcF32(EmitterState& state, const CubeF32Values& cube); - -uint32_t EmitCubeMaF32(EmitterState& state, const CubeF32Values& cube); - -void EmitCubeF32(EmitterState& state, const IR::Instruction& inst); - -void EmitDot2AccF32F16(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitPackedF16Lane(EmitterState& state, const IR::Operand& operand, bool high_lane); - -void EmitCompareF16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -uint32_t EmitPackedF16LaneBits(EmitterState& state, const IR::Operand& operand, bool high_lane); - uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits); -uint32_t EmitPackLowF32ToF16Bits(EmitterState& state, uint32_t value); - -void EmitPackB32F16(EmitterState& state, const IR::Instruction& inst); - -F16Class EmitClassifyF16Bits(EmitterState& state, uint32_t value); - -uint32_t EmitNegativeNumericF16Bool(EmitterState& state, const F16Class& cls); - -uint32_t EmitRcpF16Bits(EmitterState& state, uint32_t bits); - -uint32_t EmitSqrtF16Bits(EmitterState& state, uint32_t bits); - -uint32_t EmitRsqF16Bits(EmitterState& state, uint32_t bits); - -uint32_t EmitLog2F16Bits(EmitterState& state, uint32_t bits); - -uint32_t EmitExp2F16Bits(EmitterState& state, uint32_t bits); - -void EmitSpecialF16(EmitterState& state, const IR::Instruction& inst); - -void EmitBinaryF16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode); - -void EmitFmaF16(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitMinMaxF16Bits(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value); - -void EmitMinMaxF16(EmitterState& state, const IR::Instruction& inst, bool max_value); - -uint32_t EmitMinMax3F16Bits(EmitterState& state, uint32_t src0, uint32_t src1, uint32_t src2, - bool max_value); - -void EmitMinMax3F16(EmitterState& state, const IR::Instruction& inst, bool max_value); - -uint32_t EmitF16BitsEqualBool(EmitterState& state, uint32_t lhs, uint32_t rhs); - -uint32_t EmitAnyNanF16Bool(EmitterState& state, uint32_t src0, uint32_t src1, uint32_t src2); - -void EmitMed3F16(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitPackedMinMaxF16(EmitterState& state, const IR::Instruction& inst, bool max_value); - -uint32_t EmitPackedF16BinaryLane(EmitterState& state, const IR::Instruction& inst, bool high_lane, - uint32_t opcode); - -uint32_t EmitPackedF16FmaLane(EmitterState& state, const IR::Instruction& inst, bool high_lane); - -void EmitPackedF16(EmitterState& state, const IR::Instruction& inst); - -void EmitPackedInteger16(EmitterState& state, const IR::Instruction& inst); - -void EmitMadMixF16(EmitterState& state, const IR::Instruction& inst); - -void EmitMinMax3F32(EmitterState& state, const IR::Instruction& inst, bool max_value); - -void EmitMed3F32(EmitterState& state, const IR::Instruction& inst); - -uint32_t EmitExportComponentF32(EmitterState& state, const IR::Instruction& inst, - uint32_t component); - -uint32_t EmitExportVec4F32(EmitterState& state, const IR::Instruction& inst); - -bool ExportWritesData(const IR::Instruction& inst); - -void EmitExport(EmitterState& state, const IR::Instruction& inst); - -bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst); - -void EmitKillIfBoolFalse(EmitterState& state, uint32_t active); - -void EmitUpdatePixelValidMask(EmitterState& state); - -void EmitKillIfPixelValidMaskInactive(EmitterState& state); - uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg); bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst); @@ -1613,19 +955,6 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn& std::forward(fn)); } -template -void EmitGuardedByExec(EmitterState& state, Fn&& fn) { - const auto then_label = state.builder.AllocateId(); - const auto merge_label = state.builder.AllocateId(); - const auto condition = EmitExecActiveBool(state); - state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); - state.builder.AddFunction({OpBranchConditional, condition, then_label, merge_label}); - state.builder.AddFunction({OpLabel, then_label}); - fn(); - state.builder.AddFunction({OpBranch, merge_label}); - state.builder.AddFunction({OpLabel, merge_label}); -} - } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_INTERNAL_H_ */ diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp deleted file mode 100644 index 7e69bee3f..000000000 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ /dev/null @@ -1,1713 +0,0 @@ -#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" - -namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { - -uint32_t EmitDppWriteActiveBool(EmitterState& state, const IR::Operand& dst) { - const auto exec_active = EmitExecActiveBool(state); - if (!dst.dpp) { - return exec_active; - } - - const auto subid = EmitSubgroupLocalInvocationId(state); - const auto bank_shift = state.builder.AllocateId(); - const auto row_shift = state.builder.AllocateId(); - const auto bank = state.builder.AllocateId(); - const auto row = state.builder.AllocateId(); - const auto bank_bit = state.builder.AllocateId(); - const auto row_bit = state.builder.AllocateId(); - const auto bank_hit = state.builder.AllocateId(); - const auto row_hit = state.builder.AllocateId(); - const auto bank_active = state.builder.AllocateId(); - const auto row_active = state.builder.AllocateId(); - const auto dpp_active = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, bank_shift, subid, ConstantU32(state, 2)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, bank, bank_shift, ConstantU32(state, 3)}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, row_shift, subid, ConstantU32(state, 4)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, row, row_shift, ConstantU32(state, 3)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, dst.dpp_bank_mask), bank_bit}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, dst.dpp_row_mask), row_bit}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, bank_active, bank_hit, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, row_active, row_hit, ConstantU32(state, 0)}); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, dpp_active, bank_active, row_active}); - uint32_t write_active = dpp_active; - if (!dst.dpp_bound_ctrl) { - const auto target = EmitDppTargetLane(state, dst.dpp_ctrl); - const auto bounded = state.builder.AllocateId(); - state.builder.AddFunction( - {OpLogicalAnd, state.bool_type, bounded, write_active, target.valid}); - write_active = bounded; - } - state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, exec_active, write_active}); - return ret; -} - -uint32_t EmitSdwaDestinationMerge(EmitterState& state, uint32_t pointer, const IR::Operand& dst, - uint32_t value) { - if (dst.sdwa_sel == 6u) { - return value; - } - - uint32_t offset = 0; - uint32_t width = 0; - if (!SdwaSelectorOffsetWidth(dst.sdwa_sel, offset, width) || width == 32u) { - return value; - } - - const auto value_mask = width == 16u ? 0x0000ffffu : 0x000000ffu; - const auto field_mask = value_mask << offset; - const auto part = state.builder.AllocateId(); - const auto shifted = offset != 0u ? state.builder.AllocateId() : part; - const auto old_value = state.builder.AllocateId(); - const auto kept = state.builder.AllocateId(); - const auto merged = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, part, value, ConstantU32(state, value_mask)}); - if (offset != 0u) { - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, shifted, part, ConstantU32(state, offset)}); - } - if (dst.sdwa_dst_unused == 0u) { - return shifted; - } - if (dst.sdwa_dst_unused == 1u) { - const auto extended = state.builder.AllocateId(); - const auto result = offset != 0u ? state.builder.AllocateId() : extended; - state.builder.AddFunction({OpBitFieldSExtract, state.uint_type, extended, value, - ConstantU32(state, 0), ConstantU32(state, width)}); - if (offset != 0u) { - state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, result, extended, - ConstantU32(state, offset)}); - } - return result; - } - EXIT_IF(dst.sdwa_dst_unused != 2u); - state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, kept, old_value, ConstantU32(state, ~field_mask)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, merged, kept, shifted}); - return merged; -} - -void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value) { - if (dst.kind != IR::OperandKind::Register) { - return; - } - const auto pointer = PointerForRegister(state, dst.reg); - if (pointer != 0) { - const auto store_value = EmitSdwaDestinationMerge(state, pointer, dst, value); - const auto wave_value = - IsInactiveWave32ExecHigh(state, dst.reg) ? ConstantU32(state, 0) : store_value; - if (dst.reg.file == IR::RegisterFile::Vector) { - const auto old_value = state.builder.AllocateId(); - const auto selected = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer}); - state.builder.AddFunction({OpSelect, state.uint_type, selected, - EmitDppWriteActiveBool(state, dst), wave_value, old_value}); - state.builder.AddFunction({OpStore, pointer, selected}); - return; - } - state.builder.AddFunction({OpStore, pointer, wave_value}); - } -} - -uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpIAdd, state.uint_type, ret, lhs, rhs}); - return ret; -} - -uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs}); - return ret; -} - -uint32_t EmitNotEqualZeroBool(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpINotEqual, state.bool_type, ret, value, ConstantU32(state, 0)}); - return ret; -} - -uint32_t EmitSelectU32Value(EmitterState& state, uint32_t condition, uint32_t true_value, - uint32_t false_value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.uint_type, ret, condition, true_value, false_value}); - return ret; -} - -uint32_t EmitByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, - uint32_t src_count) { - uint32_t address = ConstantU32(state, inst.memory.offset); - for (uint32_t i = 0; i < src_count; i++) { - address = EmitAddU32(state, address, EmitValueLoad(state, inst.src[first_src + i])); - } - return address; -} - -uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem, - uint32_t use_pc) { - (void)use_pc; - if (mem.resource >= state.program.info.buffers.size()) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, - "buffer specialization is missing"); - } - return state.program.info.buffers[mem.resource].packed_stride; -} - -Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem, - uint32_t use_pc) { - (void)use_pc; - if (mem.resource >= state.program.info.buffers.size()) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, - "buffer specialization is missing"); - } - return state.program.info.buffers[mem.resource].descriptor_format; -} - -bool ShouldApplyBufferAddTid(const IR::Instruction& inst) { - return inst.memory.kind == IR::ResourceKind::Buffer && inst.memory.formatted && - !inst.memory.typed && inst.op == IR::Opcode::BufferStoreDword; -} - -uint32_t EmitBufferIndexWithAddTid(EmitterState& state, const IR::Instruction& inst, uint32_t index, - uint32_t packed) { - if (!ShouldApplyBufferAddTid(inst)) { - return index; - } - - const auto add_tid_bit = - EmitBinaryU32(state, OpBitwiseAnd, - EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 20)), - ConstantU32(state, 1)); - const auto add_tid = EmitNotEqualZeroBool(state, add_tid_bit); - const auto lane = - EmitBinaryU32(state, OpBitwiseAnd, EmitLocalInvocationIndex(state), ConstantU32(state, 63)); - const auto tid_index = EmitAddU32(state, index, lane); - return EmitSelectU32Value(state, add_tid, tid_index, index); -} - -uint32_t EmitOptionalLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { - if (lhs == 0) { - return rhs; - } - if (rhs == 0) { - return lhs; - } - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, lhs, rhs}); - return ret; -} - -bool IsStorageBufferMemoryKind(IR::ResourceKind kind) { - switch (kind) { - case IR::ResourceKind::ScalarBuffer: - case IR::ResourceKind::Buffer: - case IR::ResourceKind::Flat: - case IR::ResourceKind::Global: - case IR::ResourceKind::Scratch: return true; - default: return false; - } -} - -void EmitStorageBufferOffsets(EmitterState& state) { - for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) { - const auto word = - EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u); - const auto shift = ConstantU32(state, (i % 4u) * 8u + 2u); - state.storage_buffer_offsets[i] = EmitBinaryU32( - state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift), - ConstantU32(state, 0x3fu)); - } -} - -uint32_t EmitBufferAddressFromParts(EmitterState& state, const IR::Instruction& inst, - uint32_t index, uint32_t offset, uint32_t soffset) { - const auto& mem = inst.memory; - const auto packed = ConstantU32(state, StorageBufferPackedStride(state, mem, inst.pc)); - const auto address_index = EmitBufferIndexWithAddTid(state, inst, index, packed); - const auto stride = EmitBinaryU32(state, OpBitwiseAnd, packed, ConstantU32(state, 0x3fffu)); - const auto swizzle_bit = - EmitBinaryU32(state, OpBitwiseAnd, - EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 14)), - ConstantU32(state, 1)); - const auto stride_nonzero = EmitNotEqualZeroBool(state, stride); - const auto swizzle_nonzero = EmitNotEqualZeroBool(state, swizzle_bit); - const auto use_swizzle = state.builder.AllocateId(); - state.builder.AddFunction( - {OpLogicalAnd, state.bool_type, use_swizzle, swizzle_nonzero, stride_nonzero}); - - const auto index_stride_enum = - EmitBinaryU32(state, OpBitwiseAnd, - EmitBinaryU32(state, OpShiftRightLogical, packed, ConstantU32(state, 16)), - ConstantU32(state, 3)); - const auto index_shift = EmitAddU32(state, index_stride_enum, ConstantU32(state, 3)); - const auto index_stride = - EmitBinaryU32(state, OpShiftLeftLogical, ConstantU32(state, 8), index_stride_enum); - const auto index_stride_mask = - EmitBinaryU32(state, OpISub, index_stride, ConstantU32(state, 1)); - - const auto linear_index = EmitBinaryU32(state, OpIMul, address_index, stride); - const auto linear = EmitAddU32(state, linear_index, offset); - - const auto index_msb = EmitBinaryU32(state, OpShiftRightLogical, address_index, index_shift); - const auto index_lsb = EmitBinaryU32(state, OpBitwiseAnd, address_index, index_stride_mask); - const auto offset_msb_bytes = - EmitBinaryU32(state, OpBitwiseAnd, offset, ConstantU32(state, ~3u)); - const auto offset_lsb = EmitBinaryU32(state, OpBitwiseAnd, offset, ConstantU32(state, 3)); - const auto index_part = EmitBinaryU32(state, OpIMul, index_msb, stride); - const auto msb_sum = EmitAddU32(state, index_part, offset_msb_bytes); - const auto msb = EmitBinaryU32(state, OpIMul, msb_sum, index_stride); - const auto lsb_index = - EmitBinaryU32(state, OpShiftLeftLogical, index_lsb, ConstantU32(state, 2)); - const auto lsb = EmitAddU32(state, lsb_index, offset_lsb); - const auto swizzled = EmitAddU32(state, msb, lsb); - const auto buffer_offset = EmitSelectU32Value(state, use_swizzle, swizzled, linear); - return EmitAddU32(state, buffer_offset, soffset); -} - -uint32_t EmitBufferByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, - uint32_t src_count) { - const auto end = first_src + src_count; - auto cursor = first_src; - auto LoadAddressSource = [&]() { - if (cursor >= end) { - return ConstantU32(state, 0); - } - return EmitValueLoad(state, inst.src[cursor++]); - }; - - uint32_t index = ConstantU32(state, 0); - if (inst.memory.idxen) { - index = LoadAddressSource(); - } - uint32_t offset = ConstantU32(state, inst.memory.offset); - if (inst.memory.offen) { - offset = EmitAddU32(state, offset, LoadAddressSource()); - } - const auto soffset = LoadAddressSource(); - return EmitBufferAddressFromParts(state, inst, index, offset, soffset); -} - -bool IsAddressMemoryKind(IR::ResourceKind kind) { - return kind == IR::ResourceKind::ScalarAddress || kind == IR::ResourceKind::Flat || - kind == IR::ResourceKind::Global || kind == IR::ResourceKind::Scratch; -} - -uint32_t EmitRelativeAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, - uint32_t src_count, bool vector_only = true, - bool align_components = false) { - const auto delta = state.program.info.addresses[inst.memory.resource].specialized_base; - auto immediate = static_cast(inst.memory.offset); - if (align_components) { - immediate = static_cast(static_cast(immediate) & ~3u); - } - const auto initial = delta + static_cast(static_cast(immediate)); - uint32_t low = ConstantU32(state, static_cast(initial)); - uint32_t high = ConstantU32(state, static_cast(initial >> 32u)); - for (uint32_t i = 0; i < src_count; i++) { - const auto& src = inst.src[first_src + i]; - if (!vector_only || - (src.kind == IR::OperandKind::Register && src.reg.file == IR::RegisterFile::Vector)) { - auto value = EmitValueLoad(state, src); - if (align_components) { - value = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~3u)); - } - const auto next = EmitAddU32(state, low, value); - const auto carry = state.builder.AllocateId(); - state.builder.AddFunction({OpULessThan, state.bool_type, carry, next, low}); - high = EmitAddU32( - state, high, - EmitSelectU32Value(state, carry, ConstantU32(state, 1), ConstantU32(state, 0))); - low = next; - } - } - const auto valid = state.builder.AllocateId(); - state.builder.AddFunction({OpIEqual, state.bool_type, valid, high, ConstantU32(state, 0)}); - return EmitSelectU32Value(state, valid, low, ConstantU32(state, UINT32_MAX)); -} - -uint32_t EmitFlatVirtualAddress(EmitterState& state, const IR::Instruction& inst, - uint32_t first_src, uint32_t src_count) { - if (inst.memory.resource >= state.resources.addresses.size() || src_count < 2) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, - inst.memory.resource, "flat address snapshot is missing"); - } - const auto low0 = EmitValueLoad(state, inst.src[first_src]); - const auto high0 = EmitValueLoad(state, inst.src[first_src + 1u]); - const auto immediate = ConstantU32(state, inst.memory.offset); - const auto low = EmitAddU32(state, low0, immediate); - const auto carry_bool = state.builder.AllocateId(); - state.builder.AddFunction({OpULessThan, state.bool_type, carry_bool, low, low0}); - const auto carry = - EmitSelectU32Value(state, carry_bool, ConstantU32(state, 1), ConstantU32(state, 0)); - const auto immediate_high = - ConstantU32(state, (inst.memory.offset & 0x80000000u) != 0 ? UINT32_MAX : 0u); - const auto high = EmitAddU32(state, EmitAddU32(state, high0, immediate_high), carry); - - const auto base = state.program.info.addresses[inst.memory.resource].specialized_base; - const auto base_low = static_cast(base); - const auto base_high = static_cast(base >> 32u); - const auto relative_low = EmitBinaryU32(state, OpISub, low, ConstantU32(state, base_low)); - const auto borrow_bool = state.builder.AllocateId(); - state.builder.AddFunction( - {OpULessThan, state.bool_type, borrow_bool, low, ConstantU32(state, base_low)}); - const auto borrow = - EmitSelectU32Value(state, borrow_bool, ConstantU32(state, 1), ConstantU32(state, 0)); - const auto relative_high = EmitBinaryU32( - state, OpISub, EmitBinaryU32(state, OpISub, high, ConstantU32(state, base_high)), borrow); - const auto valid = state.builder.AllocateId(); - state.builder.AddFunction( - {OpIEqual, state.bool_type, valid, relative_high, ConstantU32(state, 0)}); - return EmitSelectU32Value(state, valid, relative_low, ConstantU32(state, UINT32_MAX)); -} - -uint32_t EmitMemoryByteAddress(EmitterState& state, const IR::Instruction& inst, - const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count) { - if (mem.kind == IR::ResourceKind::Buffer) { - return EmitBufferByteAddress(state, inst, first_src, src_count); - } - if (mem.kind == IR::ResourceKind::Flat || - ((mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) && - state.program.info.addresses[mem.resource].unbased)) { - return EmitFlatVirtualAddress(state, inst, first_src, src_count); - } - if (mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) { - return EmitRelativeAddress(state, inst, first_src, src_count); - } - return EmitByteAddress(state, inst, first_src, src_count); -} - -uint32_t EmitDwordIndex(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, - uint32_t src_count) { - const auto address = EmitByteAddress(state, inst, first_src, src_count); - const auto index = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); - return index; -} - -uint32_t EmitMemoryDwordIndex(EmitterState& state, const IR::Instruction& inst, - const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count) { - const auto address = EmitMemoryByteAddress(state, inst, mem, first_src, src_count); - const auto index = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); - return index; -} - -DescriptorResourceBinding -StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) { - (void)use_pc; - const auto binding = - ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource); - if (state.storage_buffer_variable == 0) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, - "storage buffer descriptor array was not emitted"); - } - return binding; -} - -uint32_t EmitStorageBufferIndex(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, - uint32_t use_pc) { - if (IsAddressMemoryKind(mem.kind)) { - return index; - } - const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); - return EmitAddU32(state, index, state.storage_buffer_offsets[binding.array_index]); -} - -uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t use_pc) { - if (IsAddressMemoryKind(mem.kind)) { - if (state.address_memory_variable == 0) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, - mem.resource, "address memory binding was not emitted"); - } - const auto binding = - ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource); - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer, - state.address_memory_variable, - ConstantU32(state, binding.array_index)}); - return pointer; - } - const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer, - state.storage_buffer_variable, - ConstantU32(state, binding.array_index)}); - return pointer; -} - -uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t index, uint32_t use_pc) { - index = EmitStorageBufferIndex(state, mem, index, use_pc); - const auto object = EmitStorageBufferObjectPointer(state, mem, use_pc); - const auto length = state.builder.AllocateId(); - const auto in_bounds = state.builder.AllocateId(); - state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0}); - state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length}); - return in_bounds; -} - -uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t index, uint32_t use_pc) { - index = EmitStorageBufferIndex(state, mem, index, use_pc); - if (IsAddressMemoryKind(mem.kind)) { - if (state.address_memory_variable == 0) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, - mem.resource, "address memory binding was not emitted"); - } - const auto binding = - ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource); - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction( - {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable, - ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); - return pointer; - } - const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction( - {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.storage_buffer_variable, - ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); - return pointer; -} - -uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) { - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction( - {OpAccessChain, state.ptr_workgroup_uint, pointer, state.lds_variable, index}); - return pointer; -} - -uint32_t LdsDwordCount(const EmitterState& state) { - return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u; -} - -uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) { - const auto in_bounds = state.builder.AllocateId(); - state.builder.AddFunction( - {OpULessThan, state.bool_type, in_bounds, index, ConstantU32(state, LdsDwordCount(state))}); - return in_bounds; -} - -uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, - uint32_t use_pc) { - if (mem.kind == IR::ResourceKind::Lds) { - return EmitLdsElementPointer(state, index); - } - if (mem.kind == IR::ResourceKind::Gds) { - return EmitGdsElementPointer(state, index); - } - return EmitStorageBufferElementPointer(state, mem, index, use_pc); -} - -uint32_t EmitMemoryLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst, - IR::ResourceKind kind, uint32_t first_src, - uint32_t src_count) { - auto mem = inst.memory; - mem.kind = kind; - const auto index = EmitMemoryDwordIndex(state, inst, mem, first_src, src_count); - auto LoadFromPointer = [&](uint32_t pointer) { - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); - return value; - }; - if (mem.kind == IR::ResourceKind::Lds) { - return LoadFromPointer(EmitLdsElementPointer(state, index)); - } - if (mem.kind == IR::ResourceKind::Gds) { - const auto in_bounds = EmitGdsElementInBounds(state, index); - return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { - return LoadFromPointer(EmitGdsElementPointer(state, index)); - }); - } - if (IsStorageBufferMemoryKind(mem.kind)) { - const auto in_bounds = EmitStorageBufferElementInBounds(state, mem, index, inst.pc); - return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { - return LoadFromPointer(EmitStorageBufferElementPointer(state, mem, index, inst.pc)); - }); - } - return ConstantU32(state, 0); -} - -void EmitMemoryLoadU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, - uint32_t first_src, uint32_t src_count) { - const auto value = EmitMemoryLoadDwordValueU32(state, inst, kind, first_src, src_count); - EmitStoreU32(state, inst.dst, value); -} - -void EmitMemoryLoadDwordsU32(EmitterState& state, const IR::Instruction& inst, - IR::ResourceKind kind, uint32_t first_src, uint32_t src_count) { - const auto count = std::max(1u, inst.memory.data_dwords); - if (count == 1u) { - EmitMemoryLoadU32(state, inst, kind, first_src, src_count); - return; - } - - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; i++) { - auto load_inst = inst; - load_inst.memory.offset += i * 4u; - load_inst.memory.data_dwords = 1u; - load_inst.memory.component_index = i; - load_inst.memory.component_count = count; - values.push_back(EmitMemoryLoadDwordValueU32(state, load_inst, kind, first_src, src_count)); - } - for (uint32_t i = 0; i < count; i++) { - EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), values[i]); - } -} - -uint32_t EmitMemoryLoadSubDwordValueU32(EmitterState& state, const IR::Instruction& inst, - IR::ResourceKind kind, uint32_t first_src, - uint32_t src_count, uint32_t data_bits, bool sign_extend) { - auto mem = inst.memory; - mem.kind = kind; - const auto address = EmitMemoryByteAddress(state, inst, mem, first_src, src_count); - const auto index = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); - auto LoadFromPointer = [&](uint32_t pointer) { - const auto word = state.builder.AllocateId(); - const auto byte_in_word = state.builder.AllocateId(); - const auto shift = state.builder.AllocateId(); - const auto shifted = state.builder.AllocateId(); - const auto masked = state.builder.AllocateId(); - const auto value = sign_extend ? state.builder.AllocateId() : masked; - const auto mask = data_bits == 8u ? 0xffu : 0xffffu; - state.builder.AddFunction({OpLoad, state.uint_type, word, pointer}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, byte_in_word, address, ConstantU32(state, 3)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, shift, byte_in_word, ConstantU32(state, 3)}); - state.builder.AddFunction({OpShiftRightLogical, state.uint_type, shifted, word, shift}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, masked, shifted, ConstantU32(state, mask)}); - if (sign_extend) { - const auto left = state.builder.AllocateId(); - const auto sign_shift = 32u - data_bits; - state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, left, masked, - ConstantU32(state, sign_shift)}); - state.builder.AddFunction({OpShiftRightArithmetic, state.uint_type, value, left, - ConstantU32(state, sign_shift)}); - } - return value; - }; - if (mem.kind == IR::ResourceKind::Lds) { - return LoadFromPointer(EmitLdsElementPointer(state, index)); - } - if (mem.kind == IR::ResourceKind::Gds) { - const auto in_bounds = EmitGdsElementInBounds(state, index); - return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { - return LoadFromPointer(EmitGdsElementPointer(state, index)); - }); - } - if (IsStorageBufferMemoryKind(mem.kind)) { - const auto in_bounds = EmitStorageBufferElementInBounds(state, mem, index, inst.pc); - return EmitValueOrZeroIfCondition(state, in_bounds, [&]() { - return LoadFromPointer(EmitStorageBufferElementPointer(state, mem, index, inst.pc)); - }); - } - return ConstantU32(state, 0); -} - -void EmitMemoryLoadSubDwordU32(EmitterState& state, const IR::Instruction& inst, - IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, - uint32_t data_bits, bool sign_extend) { - const auto value = EmitMemoryLoadSubDwordValueU32(state, inst, kind, first_src, src_count, - data_bits, sign_extend); - EmitStoreU32(state, inst.dst, value); -} - -void EmitMemoryStoreU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind, - uint32_t first_src, uint32_t src_count) { - auto store_inst = inst; - store_inst.memory.kind = kind; - const auto value = EmitValueLoad(state, inst.src[0]); - uint32_t in_bounds = 0; - const auto index = - EmitMemoryDwordIndex(state, store_inst, store_inst.memory, first_src, src_count); - if (IsStorageBufferMemoryKind(store_inst.memory.kind)) { - in_bounds = EmitOptionalLogicalAndBool( - state, in_bounds, - EmitStorageBufferElementInBounds(state, store_inst.memory, index, inst.pc)); - } else if (store_inst.memory.kind == IR::ResourceKind::Gds) { - in_bounds = EmitGdsElementInBounds(state, index); - } - EmitIfCondition(state, in_bounds, [&]() { - const auto pointer = EmitMemoryElementPointer(state, store_inst.memory, index, inst.pc); - state.builder.AddFunction({OpStore, pointer, value}); - }); -} - -template -uint32_t EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, - Fn&& desired_value) { - const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; - const auto memory = kind == IR::ResourceKind::Lds ? MemorySemanticsWorkgroupMemory - : MemorySemanticsUniformMemory; - const auto preheader = state.builder.AllocateId(); - const auto header = state.builder.AllocateId(); - const auto continue_label = state.builder.AllocateId(); - const auto merge = state.builder.AllocateId(); - const auto initial = state.builder.AllocateId(); - const auto observed = state.builder.AllocateId(); - const auto exchanged = state.builder.AllocateId(); - - state.builder.AddFunction({OpBranch, preheader}); - state.builder.AddFunction({OpLabel, preheader}); - state.builder.AddFunction({OpAtomicLoad, state.uint_type, initial, pointer, - ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)}); - state.builder.AddFunction({OpBranch, header}); - state.builder.AddFunction({OpLabel, header}); - state.builder.AddFunction( - {OpPhi, state.uint_type, observed, initial, preheader, exchanged, continue_label}); - const auto desired = desired_value(observed); - state.builder.AddFunction({OpAtomicCompareExchange, state.uint_type, exchanged, pointer, - ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone), - ConstantU32(state, MemorySemanticsNone), desired, observed}); - const auto success = state.builder.AllocateId(); - state.builder.AddFunction({OpIEqual, state.bool_type, success, exchanged, observed}); - state.builder.AddFunction({OpLoopMerge, merge, continue_label, LoopControlNone}); - state.builder.AddFunction({OpBranchConditional, success, merge, continue_label}); - state.builder.AddFunction({OpLabel, continue_label}); - state.builder.AddFunction({OpBranch, header}); - state.builder.AddFunction({OpLabel, merge}); - const auto semantics = MemorySemanticsAcquireRelease | memory; - state.builder.AddFunction( - {OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)}); - return observed; -} - -void EmitMemoryStoreSubDwordU32(EmitterState& state, const IR::Instruction& inst, - IR::ResourceKind kind, uint32_t first_src, uint32_t src_count, - uint32_t data_bits) { - auto store_inst = inst; - store_inst.memory.kind = kind; - uint32_t in_bounds = 0; - const auto address = - EmitMemoryByteAddress(state, store_inst, store_inst.memory, first_src, src_count); - const auto index = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); - if (IsStorageBufferMemoryKind(store_inst.memory.kind)) { - in_bounds = EmitOptionalLogicalAndBool( - state, in_bounds, - EmitStorageBufferElementInBounds(state, store_inst.memory, index, inst.pc)); - } else if (store_inst.memory.kind == IR::ResourceKind::Gds) { - in_bounds = EmitGdsElementInBounds(state, index); - } - EmitIfCondition(state, in_bounds, [&]() { - const auto pointer = EmitMemoryElementPointer(state, store_inst.memory, index, inst.pc); - const auto value = EmitValueLoad(state, inst.src[0]); - const auto byte_in_word = state.builder.AllocateId(); - const auto shift = state.builder.AllocateId(); - const auto field_mask = state.builder.AllocateId(); - const auto value_masked = state.builder.AllocateId(); - const auto value_shifted = state.builder.AllocateId(); - const auto mask = data_bits == 8u ? 0xffu : 0xffffu; - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, byte_in_word, address, ConstantU32(state, 3)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, shift, byte_in_word, ConstantU32(state, 3)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, field_mask, ConstantU32(state, mask), shift}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, value_masked, value, ConstantU32(state, mask)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, value_shifted, value_masked, shift}); - auto Merge = [&](uint32_t word) { - const auto inverted = state.builder.AllocateId(); - const auto cleared = state.builder.AllocateId(); - const auto merged = state.builder.AllocateId(); - state.builder.AddFunction({OpNot, state.uint_type, inverted, field_mask}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, cleared, word, inverted}); - state.builder.AddFunction( - {OpBitwiseOr, state.uint_type, merged, cleared, value_shifted}); - return merged; - }; - if (store_inst.memory.kind == IR::ResourceKind::Lds || - store_inst.memory.kind == IR::ResourceKind::Gds) { - EmitAtomicUpdateU32(state, pointer, store_inst.memory.kind, Merge); - } else { - const auto word = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, word, pointer}); - state.builder.AddFunction({OpStore, pointer, Merge(word)}); - } - }); -} - -uint32_t AddressSourceCount(const IR::Instruction& inst, uint32_t first_src) { - return inst.src_count > first_src ? inst.src_count - first_src : 0u; -} - -bool IsFormattedBufferComponent(const IR::Instruction& inst) { - return inst.memory.kind == IR::ResourceKind::Buffer && inst.memory.formatted && - inst.memory.data_dwords == 1u; -} - -Prospero::BufferFormat FormattedBufferFormat(const EmitterState& state, - const IR::Instruction& inst) { - return inst.memory.typed - ? Format::DecodeTBufferFormat(inst.memory.data_format, inst.memory.number_format) - : StorageBufferFormat(state, inst.memory, inst.pc); -} - -IR::Instruction WithFormatComponentByteOffset(const IR::Instruction& inst, - Prospero::BufferFormat format) { - auto rebased = inst; - if (inst.memory.typed || inst.memory.component_index == 0u) { - return rebased; - } - - const auto default_component_offset = inst.memory.component_index * 4u; - if (rebased.memory.offset >= default_component_offset) { - rebased.memory.offset = - rebased.memory.offset - default_component_offset + - Format::GetFormatComponentByteOffset(format, inst.memory.component_index); - } - return rebased; -} - -uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, ret, value}); - return ret; -} - -uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.float_type, ret, value}); - return ret; -} - -uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, ret, value}); - return ret; -} - -uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value, - uint32_t constant) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)}); - return ret; -} - -uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value, - uint32_t false_value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpSelect, state.float_type, ret, condition, true_value, false_value}); - return ret; -} - -bool IsSignedFormatComponent(Format::ComponentType type) { - return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm || - type == Format::ComponentType::Sscaled; -} - -uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint32_t offset, - uint32_t bits, bool sign_extend) { - if (bits == 32u && offset == 0u) { - return raw_word; - } - - if (sign_extend) { - const auto signed_word = EmitTBufferBitcastU32ToI32(state, raw_word); - const auto extracted = state.builder.AllocateId(); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_word, - ConstantU32(state, offset), ConstantU32(state, bits)}); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, ret, extracted}); - return ret; - } - - const auto extracted = state.builder.AllocateId(); - state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, raw_word, - ConstantU32(state, offset), ConstantU32(state, bits)}); - return extracted; -} - -uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw) { - const auto unpacked = state.builder.AllocateId(); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, raw}); - state.builder.AddFunction({OpCompositeExtract, state.float_type, value, unpacked, 0}); - return EmitTBufferBitcastF32ToU32(state, value); -} - -uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) { - const auto mantissa_bits = bits == 11u ? 6u : 5u; - const auto mantissa_mask = (1u << mantissa_bits) - 1u; - const auto mantissa = - EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask)); - const auto exponent = EmitBinaryU32( - state, OpBitwiseAnd, - EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)), - ConstantU32(state, 0x1fu)); - - const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u)); - const auto exponent_bits = - EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23)); - const auto mantissa_bits_32 = - EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits)); - const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32); - const auto normal = EmitTBufferBitcastU32ToF32(state, normal_bits); - - const auto special_bits = - EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32); - const auto special = EmitTBufferBitcastU32ToF32(state, special_bits); - - const auto mantissa_f32 = state.builder.AllocateId(); - const auto subnormal = state.builder.AllocateId(); - state.builder.AddFunction({OpConvertUToF, state.float_type, mantissa_f32, mantissa}); - state.builder.AddFunction( - {OpFMul, state.float_type, subnormal, mantissa_f32, - ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast(mantissa_bits)))}); - - const auto zero_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 0); - const auto special_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 31); - const auto finite = EmitTBufferSelectF32(state, zero_exp, subnormal, normal); - const auto result = EmitTBufferSelectF32(state, special_exp, special, finite); - return EmitTBufferBitcastF32ToU32(state, result); -} - -uint32_t EmitFormatRawComponent(EmitterState& state, const IR::Instruction& inst, - const Format::BufferFormatInfo& info) { - const auto component = inst.memory.component_index; - if (component >= info.component_count) { - return ConstantU32(state, 0); - } - - auto load_inst = WithFormatComponentByteOffset(inst, info.format); - const auto bits = info.component_bits[component]; - const auto sign_extend = IsSignedFormatComponent(info.type); - if (info.packed_bitfield) { - const auto raw_word = EmitMemoryLoadDwordValueU32( - state, load_inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(load_inst, 0)); - return EmitExtractFormatFieldU32(state, raw_word, info.component_bit_offset[component], - bits, sign_extend); - } - - if (bits == 32u) { - return EmitMemoryLoadDwordValueU32(state, load_inst, IR::ResourceKind::Buffer, 0, - AddressSourceCount(load_inst, 0)); - } - - return EmitMemoryLoadSubDwordValueU32(state, load_inst, IR::ResourceKind::Buffer, 0, - AddressSourceCount(load_inst, 0), bits, sign_extend); -} - -uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info, - uint32_t component, uint32_t raw) { - const auto bits = info.component_bits[component]; - switch (info.type) { - case Format::ComponentType::Uint: - case Format::ComponentType::Sint: return raw; - case Format::ComponentType::Uscaled: { - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw}); - return EmitTBufferBitcastF32ToU32(state, value); - } - case Format::ComponentType::Sscaled: { - const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw}); - return EmitTBufferBitcastF32ToU32(state, value); - } - case Format::ComponentType::Unorm: { - const auto value = state.builder.AllocateId(); - const auto normalized = state.builder.AllocateId(); - const auto max_value = static_cast((1u << bits) - 1u); - state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw}); - state.builder.AddFunction( - {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)}); - return EmitTBufferBitcastF32ToU32(state, normalized); - } - case Format::ComponentType::Snorm: { - const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw); - const auto value = state.builder.AllocateId(); - const auto normalized = state.builder.AllocateId(); - const auto clamped = state.builder.AllocateId(); - const auto max_value = static_cast((1u << (bits - 1u)) - 1u); - state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw}); - state.builder.AddFunction( - {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)}); - state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450, - GlslFMax, normalized, ConstantF32Value(state, -1.0f)}); - return EmitTBufferBitcastF32ToU32(state, clamped); - } - case Format::ComponentType::Float: - if (bits == 32u) { - return raw; - } - if (bits == 16u) { - return EmitHalfToF32Bits(state, raw); - } - return EmitUFloatToF32Bits(state, raw, bits); - default: return raw; - } -} - -uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst, - const Format::BufferFormatInfo& info) { - const auto raw = EmitFormatRawComponent(state, inst, info); - return NormalizeFormatComponent(state, info, inst.memory.component_index, raw); -} - -bool EmitFormattedBufferLoadValueU32(EmitterState& state, const IR::Instruction& inst, - uint32_t& value) { - if (!IsFormattedBufferComponent(inst)) { - return false; - } - - const auto format = FormattedBufferFormat(state, inst); - if (!Format::IsKnownFormat(format)) { - return false; - } - - const auto info = Format::GetFormatInfo(format); - if (inst.memory.component_index >= info.component_count) { - value = ConstantU32(state, 0); - return true; - } - - if (Format::CanUseTypedBufferLoad(info.format)) { - value = EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0, - AddressSourceCount(inst, 0)); - return true; - } - - value = UnpackTBufferFormat(state, inst, info); - return true; -} - -uint32_t EmitBufferLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst) { - uint32_t value = 0; - if (EmitFormattedBufferLoadValueU32(state, inst, value)) { - return value; - } - return EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0, - AddressSourceCount(inst, 0)); -} - -uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format, - uint32_t opcode_components) { - switch (format) { - case Prospero::BufferFormat::k32UInt: - case Prospero::BufferFormat::k32SInt: - case Prospero::BufferFormat::k32Float: return opcode_components >= 1u ? 1u : 0u; - case Prospero::BufferFormat::k32_32UInt: - case Prospero::BufferFormat::k32_32SInt: - case Prospero::BufferFormat::k32_32Float: return opcode_components >= 2u ? 2u : 0u; - case Prospero::BufferFormat::k32_32_32UInt: - case Prospero::BufferFormat::k32_32_32SInt: - case Prospero::BufferFormat::k32_32_32Float: return opcode_components >= 3u ? 3u : 0u; - case Prospero::BufferFormat::k32_32_32_32UInt: - case Prospero::BufferFormat::k32_32_32_32SInt: - case Prospero::BufferFormat::k32_32_32_32Float: return opcode_components >= 4u ? 4u : 0u; - default: return 0u; - } -} - -bool EmitBufferIntegerFormatStore(EmitterState& state, const IR::Instruction& inst) { - if (!IsFormattedBufferComponent(inst)) { - return false; - } - - const auto format = FormattedBufferFormat(state, inst); - if (!inst.memory.typed) { - const auto dword_components = - FormattedBufferDwordStoreComponentCount(format, inst.memory.component_count); - if (dword_components != 0u) { - if (inst.memory.component_index >= dword_components) { - return true; - } - EmitMemoryStoreU32(state, inst, IR::ResourceKind::Buffer, 1, - AddressSourceCount(inst, 1)); - return true; - } - - switch (format) { - case Prospero::BufferFormat::k8UInt: - case Prospero::BufferFormat::k8SInt: - if (inst.memory.component_index != 0u) { - return true; - } - EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, - AddressSourceCount(inst, 1), 8); - return true; - case Prospero::BufferFormat::k16UInt: - case Prospero::BufferFormat::k16SInt: - if (inst.memory.component_index != 0u) { - return true; - } - EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, - AddressSourceCount(inst, 1), 16); - return true; - case Prospero::BufferFormat::k8_8UInt: - case Prospero::BufferFormat::k8_8SInt: { - if (inst.memory.component_index >= 2u) { - return true; - } - const auto rebased = WithFormatComponentByteOffset(inst, format); - EmitMemoryStoreSubDwordU32(state, rebased, IR::ResourceKind::Buffer, 1, - AddressSourceCount(rebased, 1), 8); - return true; - } - case Prospero::BufferFormat::k16_16UInt: - case Prospero::BufferFormat::k16_16SInt: { - if (inst.memory.component_index >= 2u) { - return true; - } - const auto rebased = WithFormatComponentByteOffset(inst, format); - EmitMemoryStoreSubDwordU32(state, rebased, IR::ResourceKind::Buffer, 1, - AddressSourceCount(rebased, 1), 16); - return true; - } - default: return false; - } - } - - switch (format) { - case Prospero::BufferFormat::k8UInt: - case Prospero::BufferFormat::k8SInt: - EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, - AddressSourceCount(inst, 1), 8); - return true; - case Prospero::BufferFormat::k16UInt: - case Prospero::BufferFormat::k16SInt: - EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, - AddressSourceCount(inst, 1), 16); - return true; - case Prospero::BufferFormat::k8_8UInt: - case Prospero::BufferFormat::k8_8SInt: - EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, - AddressSourceCount(inst, 1), 8); - return true; - case Prospero::BufferFormat::k16_16UInt: - case Prospero::BufferFormat::k16_16SInt: - EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, - AddressSourceCount(inst, 1), 16); - return true; - default: break; - } - return false; -} - -uint32_t EmitAtomicPointer(EmitterState& state, const IR::Instruction& inst) { - switch (inst.memory.kind) { - case IR::ResourceKind::Lds: - return EmitLdsElementPointer(state, EmitDwordIndex(state, inst, 1, 1)); - case IR::ResourceKind::Gds: - return EmitGdsElementPointer(state, EmitDwordIndex(state, inst, 1, 1)); - case IR::ResourceKind::Buffer: { - const auto index = - EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1)); - return EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc); - } - case IR::ResourceKind::StorageImageUint: { - const auto view = StorageImageViewKind(state, inst.memory, true, inst.pc); - const auto image_pointer = - StorageImageDescriptorPointer(state, inst.memory.resource, true, inst.pc, view); - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpImageTexelPointer, state.ptr_image_uint, pointer, - image_pointer, EmitImageCoordU32(state, inst, view), - ConstantU32(state, 0)}); - return pointer; - } - default: return 0; - } -} - -void EmitDeviceAtomicMemoryBarrier(EmitterState& state) { - const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory; - state.builder.AddFunction( - {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)}); -} - -void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode) { - if (IsStorageBufferMemoryKind(inst.memory.kind)) { - const auto index = - EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1)); - const auto in_bounds = EmitStorageBufferElementInBounds(state, inst.memory, index, inst.pc); - const auto value = EmitValueLoad(state, inst.src[0]); - const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() { - const auto pointer = - EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc); - const auto result = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, result, pointer, - ConstantU32(state, ScopeDevice), - ConstantU32(state, MemorySemanticsNone), value}); - EmitDeviceAtomicMemoryBarrier(state); - return result; - }); - EmitStoreU32(state, inst.dst, old); - return; - } - if (inst.memory.kind == IR::ResourceKind::Gds) { - const auto index = EmitDwordIndex(state, inst, 1, 1); - const auto in_bounds = EmitGdsElementInBounds(state, index); - const auto value = EmitValueLoad(state, inst.src[0]); - const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() { - const auto pointer = EmitGdsElementPointer(state, index); - const auto result = state.builder.AllocateId(); - state.builder.AddFunction({opcode, state.uint_type, result, pointer, - ConstantU32(state, ScopeDevice), - ConstantU32(state, MemorySemanticsNone), value}); - EmitDeviceAtomicMemoryBarrier(state); - return result; - }); - EmitStoreU32(state, inst.dst, old); - return; - } - - const auto pointer = EmitAtomicPointer(state, inst); - if (pointer == 0) { - EmitStoreU32(state, inst.dst, ConstantU32(state, 0)); - return; - } - - const auto value = EmitValueLoad(state, inst.src[0]); - const auto old = state.builder.AllocateId(); - const auto scope = inst.memory.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; - state.builder.AddFunction({opcode, state.uint_type, old, pointer, ConstantU32(state, scope), - ConstantU32(state, MemorySemanticsNone), value}); - if (inst.memory.kind == IR::ResourceKind::StorageImageUint || - inst.memory.kind == IR::ResourceKind::Gds) { - EmitDeviceAtomicMemoryBarrier(state); - } - EmitStoreU32(state, inst.dst, old); -} - -namespace { - -uint32_t EmitF32BitsOrderedCompare(EmitterState& state, uint32_t lhs_bits, uint32_t rhs_bits, - uint32_t comparison_opcode) { - struct ClassifiedBits { - uint32_t nan = 0; - uint32_t zero = 0; - uint32_t key = 0; - }; - const auto Classify = [&](uint32_t bits) { - ClassifiedBits cls; - const auto abs_bits = EmitAndConstant(state, bits, 0x7fffffffu); - const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u); - const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu); - const auto exponent_max = - EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u); - const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0); - const auto negative = EmitCompareU32Constant(state, OpINotEqual, - EmitAndConstant(state, bits, 0x80000000u), 0); - const auto negative_key = state.builder.AllocateId(); - const auto positive_key = state.builder.AllocateId(); - // Map all non-NaN IEEE-754 encodings to monotonically increasing unsigned keys. - state.builder.AddFunction({OpNot, state.uint_type, negative_key, bits}); - state.builder.AddFunction( - {OpBitwiseXor, state.uint_type, positive_key, bits, ConstantU32(state, 0x80000000u)}); - cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero); - cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); - cls.key = EmitSelectValueU32(state, negative, negative_key, positive_key); - return cls; - }; - - const auto lhs = Classify(lhs_bits); - const auto rhs = Classify(rhs_bits); - const auto any_nan = EmitLogicalOrBool(state, lhs.nan, rhs.nan); - // The key transform orders -0 below +0, while the floating comparison treats them as equal. - const auto both_zero = EmitLogicalAndBool(state, lhs.zero, rhs.zero); - const auto ordered_nonzero = - EmitLogicalNotBool(state, EmitLogicalOrBool(state, any_nan, both_zero)); - const auto comparison = state.builder.AllocateId(); - state.builder.AddFunction({comparison_opcode, state.bool_type, comparison, lhs.key, rhs.key}); - return EmitLogicalAndBool(state, ordered_nonzero, comparison); -} - -void EmitAtomicFMinMaxF32(EmitterState& state, const IR::Instruction& inst, - uint32_t comparison_opcode) { - const auto index = - EmitMemoryDwordIndex(state, inst, inst.memory, 1, AddressSourceCount(inst, 1)); - const auto in_bounds = EmitStorageBufferElementInBounds(state, inst.memory, index, inst.pc); - const auto src_u32 = EmitValueLoad(state, inst.src[0]); - const auto old = EmitValueOrZeroIfCondition(state, in_bounds, [&]() { - const auto pointer = EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc); - return EmitAtomicUpdateU32(state, pointer, inst.memory.kind, [&](uint32_t old_u32) { - const auto replace_old = state.builder.AllocateId(); - const auto replace = - EmitF32BitsOrderedCompare(state, src_u32, old_u32, comparison_opcode); - state.builder.AddFunction( - {OpSelect, state.uint_type, replace_old, replace, src_u32, old_u32}); - return replace_old; - }); - }); - EmitStoreU32(state, inst.dst, old); -} - -} // namespace - -void EmitAtomicFMinF32(EmitterState& state, const IR::Instruction& inst) { - EmitAtomicFMinMaxF32(state, inst, OpULessThan); -} - -void EmitAtomicFMaxF32(EmitterState& state, const IR::Instruction& inst) { - EmitAtomicFMinMaxF32(state, inst, OpUGreaterThan); -} - -void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst) { - if (state.address_memory_variable == 0) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, - inst.memory.resource, - "scalar memory descriptor array was not emitted"); - } - const auto binding = ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, - inst.memory.resource); - const auto address = EmitRelativeAddress(state, inst, 0, 1, false, true); - const auto index = state.builder.AllocateId(); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); - const auto object = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, object, - state.address_memory_variable, - ConstantU32(state, binding.array_index)}); - const auto length = state.builder.AllocateId(); - const auto in_bounds = state.builder.AllocateId(); - state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0}); - state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length}); - const auto value = EmitValueOrZeroIfCondition(state, in_bounds, [&]() { - const auto pointer = state.builder.AllocateId(); - const auto loaded = state.builder.AllocateId(); - state.builder.AddFunction( - {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable, - ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); - state.builder.AddFunction({OpLoad, state.uint_type, loaded, pointer}); - return loaded; - }); - EmitStoreU32(state, inst.dst, value); -} - -void EmitLoadSrtDword(EmitterState& state, const IR::Instruction& inst) { - if (state.flattened_srt_variable == 0 || inst.src_count != 1 || - inst.src[0].kind != IR::OperandKind::ImmediateU32) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::FlattenedSrt, - inst.src_count != 0 ? inst.src[0].imm : 0, - "invalid flattened SRT load"); - } - const auto pointer = state.builder.AllocateId(); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer, - state.flattened_srt_variable, ConstantU32(state, 0), - ConstantU32(state, inst.src[0].imm)}); - state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); - EmitStoreU32(state, inst.dst, value); -} - -void EmitBufferLoadUbyte(EmitterState& state, const IR::Instruction& inst) { - EmitGuardedByExec(state, [&]() { - EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, - AddressSourceCount(inst, 0), 8, false); - }); -} - -void EmitBufferLoadSbyte(EmitterState& state, const IR::Instruction& inst) { - EmitGuardedByExec(state, [&]() { - EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, - AddressSourceCount(inst, 0), 8, true); - }); -} - -void EmitBufferLoadUshort(EmitterState& state, const IR::Instruction& inst) { - EmitGuardedByExec(state, [&]() { - EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, - AddressSourceCount(inst, 0), 16, false); - }); -} - -void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst) { - EmitGuardedByExec(state, [&]() { - EmitMemoryLoadSubDwordU32(state, inst, IR::ResourceKind::Buffer, 0, - AddressSourceCount(inst, 0), 16, true); - }); -} - -void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst) { - EmitGuardedByExec( - state, [&]() { EmitStoreU32(state, inst.dst, EmitBufferLoadDwordValueU32(state, inst)); }); -} - -void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions, - uint32_t count) { - if (instructions == nullptr || count == 0u) { - return; - } - - EmitGuardedByExec(state, [&]() { - // RDNA VMEM captures every VADDR component before making overlapping VDATA writes - // visible. Keep the split IR components instruction-atomic by deferring all stores. - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; i++) { - values.push_back(EmitBufferLoadDwordValueU32(state, instructions[i])); - } - for (uint32_t i = 0; i < count; i++) { - EmitStoreU32(state, instructions[i].dst, values[i]); - } - }); -} - -void EmitBufferStoreDword(EmitterState& state, const IR::Instruction& inst) { - if (EmitBufferIntegerFormatStore(state, inst)) { - return; - } - EmitMemoryStoreU32(state, inst, IR::ResourceKind::Buffer, 1, AddressSourceCount(inst, 1)); -} - -void EmitFlatLoadUbyte(EmitterState& state, const IR::Instruction& inst) { - EmitGuardedByExec(state, [&]() { - EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 8, false); - }); -} - -void EmitFlatLoadSbyte(EmitterState& state, const IR::Instruction& inst) { - EmitGuardedByExec(state, [&]() { - EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 8, true); - }); -} - -void EmitFlatLoadUshort(EmitterState& state, const IR::Instruction& inst) { - EmitGuardedByExec(state, [&]() { - EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 16, false); - }); -} - -void EmitFlatLoadSshort(EmitterState& state, const IR::Instruction& inst) { - EmitGuardedByExec(state, [&]() { - EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, inst.src_count, 16, true); - }); -} - -void EmitFlatLoadDword(EmitterState& state, const IR::Instruction& inst) { - EmitGuardedByExec(state, [&]() { - EmitMemoryLoadDwordsU32(state, inst, inst.memory.kind, 0, inst.src_count); - }); -} - -uint32_t EmitDsAddtidDwordIndex(EmitterState& state, const IR::Instruction& inst, - uint32_t m0_src_index) { - if (state.stage != ShaderType::Compute) { - const auto m0 = m0_src_index < inst.src_count ? EmitValueLoad(state, inst.src[m0_src_index]) - : ConstantU32(state, 0); - const auto m0_lo = state.builder.AllocateId(); - const auto base = state.builder.AllocateId(); - const auto index = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, m0_lo, m0, ConstantU32(state, 0x0000ffffu)}); - state.builder.AddFunction( - {OpIAdd, state.uint_type, base, m0_lo, ConstantU32(state, inst.memory.offset)}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, index, base, ConstantU32(state, 2)}); - return index; - } - - const auto local_index = EmitLocalInvocationIndex(state); - const auto lane = state.builder.AllocateId(); - const auto lane_bytes = state.builder.AllocateId(); - const auto m0 = m0_src_index < inst.src_count ? EmitValueLoad(state, inst.src[m0_src_index]) - : ConstantU32(state, 0); - const auto m0_lo = state.builder.AllocateId(); - const auto base = state.builder.AllocateId(); - const auto address = state.builder.AllocateId(); - const auto index = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, lane, local_index, ConstantU32(state, 63)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, lane_bytes, lane, ConstantU32(state, 2)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, m0_lo, m0, ConstantU32(state, 0x0000ffffu)}); - state.builder.AddFunction( - {OpIAdd, state.uint_type, base, m0_lo, ConstantU32(state, inst.memory.offset)}); - state.builder.AddFunction({OpIAdd, state.uint_type, address, base, lane_bytes}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); - return index; -} - -void EmitDsWriteAddtidB32(EmitterState& state, const IR::Instruction& inst) { - const auto value = EmitValueLoad(state, inst.src[0]); - const auto index = EmitDsAddtidDwordIndex(state, inst, 1); - const auto pointer = EmitLdsElementPointer(state, index); - state.builder.AddFunction({OpStore, pointer, value}); -} - -void EmitDsReadAddtidB32(EmitterState& state, const IR::Instruction& inst) { - const auto index = EmitDsAddtidDwordIndex(state, inst, 0); - const auto pointer = EmitLdsElementPointer(state, index); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); - EmitStoreU32(state, inst.dst, value); -} - -struct DsCounterAddress { - uint32_t index = 0; - uint32_t size = 0; - uint32_t in_lds_bounds = 0; -}; - -DsCounterAddress EmitAppendConsumeAddress(EmitterState& state, const IR::Instruction& inst) { - const auto m0 = inst.src_count > 0 ? EmitValueLoad(state, inst.src[0]) : ConstantU32(state, 0); - const auto base = state.builder.AllocateId(); - const auto size = state.builder.AllocateId(); - const auto address = state.builder.AllocateId(); - const auto index = state.builder.AllocateId(); - const auto in_bounds = state.builder.AllocateId(); - // RDNA2 puts the DS append/consume byte base in M0[31:16]. - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, base, m0, ConstantU32(state, 16)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, size, m0, ConstantU32(state, 0xffffu)}); - state.builder.AddFunction( - {OpIAdd, state.uint_type, address, base, ConstantU32(state, inst.memory.offset)}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)}); - state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, - ConstantU32(state, inst.memory.offset + 3u), size}); - return {index, size, in_bounds}; -} - -uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) { - const auto length = state.builder.AllocateId(); - const auto in_bounds = state.builder.AllocateId(); - state.builder.AddFunction({OpArrayLength, state.uint_type, length, state.gds_variable, 0}); - state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length}); - return in_bounds; -} - -uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) { - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer, - state.gds_variable, ConstantU32(state, 0), index}); - return pointer; -} - -uint32_t EmitBitCountU32Value(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitCount, state.uint_type, ret, value}); - return ret; -} - -uint32_t EmitFindLsbU32Value(EmitterState& state, uint32_t value) { - const auto i32 = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.int_type, i32, state.glsl_std450, GlslFindILsb, value}); - state.builder.AddFunction({OpBitcast, state.uint_type, ret, i32}); - return ret; -} - -struct ExecMaskInfo { - uint32_t active_count = 0; - uint32_t first_lane = 0; - uint32_t any_active = 0; -}; - -ExecMaskInfo EmitExecMaskInfo(EmitterState& state) { - uint32_t exec_lo = 0; - uint32_t exec_hi = 0; - if (state.per_invocation_masks) { - const auto ballot = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot, - ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)}); - exec_lo = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, exec_lo, ballot, 0}); - if (state.wave_size == 64u) { - exec_hi = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, exec_hi, ballot, 1}); - } else { - exec_hi = ConstantU32(state, 0); - } - } else { - exec_lo = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0}); - exec_hi = WaveMaskHighLoad(state, IR::RegisterFile::Exec); - } - const auto lo_count = EmitBitCountU32Value(state, exec_lo); - const auto hi_count = EmitBitCountU32Value(state, exec_hi); - const auto active_count = state.builder.AllocateId(); - state.builder.AddFunction({OpIAdd, state.uint_type, active_count, lo_count, hi_count}); - - const auto lo_nonzero = EmitNotEqualZeroBool(state, exec_lo); - const auto hi_nonzero = EmitNotEqualZeroBool(state, exec_hi); - const auto any_active = EmitLogicalOrBool(state, lo_nonzero, hi_nonzero); - const auto hi_lsb = EmitFindLsbU32Value(state, exec_hi); - const auto hi_lane = state.builder.AllocateId(); - state.builder.AddFunction({OpIAdd, state.uint_type, hi_lane, hi_lsb, ConstantU32(state, 32)}); - const auto lo_lsb = EmitFindLsbU32Value(state, exec_lo); - const auto selected = state.builder.AllocateId(); - state.builder.AddFunction({OpSelect, state.uint_type, selected, lo_nonzero, lo_lsb, hi_lane}); - const auto first_lane = state.builder.AllocateId(); - state.builder.AddFunction( - {OpSelect, state.uint_type, first_lane, any_active, selected, ConstantU32(state, 0)}); - return {active_count, first_lane, any_active}; -} - -void EmitDsAppendConsume(EmitterState& state, const IR::Instruction& inst, uint32_t atomic_opcode) { - const auto gds = inst.memory.kind == IR::ResourceKind::Gds; - if (gds && state.gds_variable == 0) { - EmitStoreU32(state, inst.dst, ConstantU32(state, 0)); - return; - } - - const auto address = EmitAppendConsumeAddress(state, inst); - const auto storage_bounds = gds ? EmitGdsElementInBounds(state, address.index) - : EmitLdsElementInBounds(state, address.index); - const auto m0_bounds = gds ? EmitNotEqualZeroBool(state, address.size) : address.in_lds_bounds; - const auto in_bounds = EmitLogicalAndBool(state, m0_bounds, storage_bounds); - const auto exec = EmitExecMaskInfo(state); - const auto subid = EmitSubgroupLocalInvocationId(state); - const auto first_lane = state.builder.AllocateId(); - const auto do_atomic = state.builder.AllocateId(); - state.builder.AddFunction({OpIEqual, state.bool_type, first_lane, subid, exec.first_lane}); - const auto first_active = EmitLogicalAndBool(state, first_lane, exec.any_active); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, do_atomic, first_active, in_bounds}); - - const auto atomic_value = EmitValueOrZeroIfCondition(state, do_atomic, [&]() { - const auto pointer = gds ? EmitGdsElementPointer(state, address.index) - : EmitLdsElementPointer(state, address.index); - const auto result = state.builder.AllocateId(); - state.builder.AddFunction({atomic_opcode, state.uint_type, result, pointer, - ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup), - ConstantU32(state, MemorySemanticsNone), exec.active_count}); - if (gds) { - EmitDeviceAtomicMemoryBarrier(state); - } else { - const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; - state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, ScopeWorkgroup), - ConstantU32(state, semantics)}); - } - return result; - }); - const auto broadcast = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, broadcast, - ConstantU32(state, ScopeSubgroup), atomic_value, exec.first_lane}); - const auto value = EmitSelectU32Value(state, exec.any_active, broadcast, ConstantU32(state, 0)); - EmitStoreU32(state, inst.dst, value); -} - -void EmitDsFloatMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool max_value) { - const auto index = EmitDwordIndex(state, inst, 1, 1); - const auto in_bounds = - inst.memory.kind == IR::ResourceKind::Gds ? EmitGdsElementInBounds(state, index) : 0u; - EmitIfCondition(state, in_bounds, [&]() { - const auto pointer = EmitMemoryElementPointer(state, inst.memory, index, inst.pc); - const auto data_f32 = EmitFloatLoad(state, inst.src[0]); - const auto cmp_f32 = EmitFloatLoad(state, inst.src[2]); - EmitAtomicUpdateU32(state, pointer, inst.memory.kind, [&](uint32_t old_u32) { - const auto old_f32 = state.builder.AllocateId(); - const auto store_src = state.builder.AllocateId(); - const auto value_f32 = state.builder.AllocateId(); - const auto value_u32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.float_type, old_f32, old_u32}); - state.builder.AddFunction({max_value ? OpFOrdGreaterThan : OpFOrdLessThan, - state.bool_type, store_src, max_value ? old_f32 : cmp_f32, - max_value ? cmp_f32 : old_f32}); - state.builder.AddFunction( - {OpSelect, state.float_type, value_f32, store_src, data_f32, old_f32}); - state.builder.AddFunction({OpBitcast, state.uint_type, value_u32, value_f32}); - return value_u32; - }); - }); -} - -uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) { - if ((control & 0xc000u) == 0xc000u) { - const uint32_t mask = control & 0x1fu; - const uint32_t rotate = (control >> 5u) & 0x1fu; - const uint32_t rotate_delta = (control & 0x400u) != 0u ? ((32u - rotate) & 0x1fu) : rotate; - const auto lane = EmitAndConstant(state, subid, 31); - const auto rotated_sum = EmitAddU32(state, lane, ConstantU32(state, rotate_delta)); - const auto rotated = EmitAndConstant(state, rotated_sum, 31); - const auto kept = EmitAndConstant(state, lane, mask); - const auto moved = EmitAndConstant(state, rotated, (~mask) & 31u); - const auto combined = EmitOrU32(state, kept, moved); - const auto base = EmitAndConstant(state, subid, 0xffffffe0u); - return EmitOrU32(state, base, combined); - } - - if ((control & 0x8000u) != 0) { - const auto lane2 = state.builder.AllocateId(); - const auto shift = state.builder.AllocateId(); - const auto perm0 = state.builder.AllocateId(); - const auto perm = state.builder.AllocateId(); - const auto base = state.builder.AllocateId(); - const auto target = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, lane2, subid, ConstantU32(state, 3)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, shift, lane2, ConstantU32(state, 1)}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, perm0, ConstantU32(state, control), shift}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, perm, perm0, ConstantU32(state, 3)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xfffffffcu)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, perm}); - return target; - } - - const auto lane = state.builder.AllocateId(); - const auto masked = state.builder.AllocateId(); - const auto ored = state.builder.AllocateId(); - const auto xored = state.builder.AllocateId(); - const auto base = state.builder.AllocateId(); - const auto target = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)}); - state.builder.AddFunction( - {OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)}); - state.builder.AddFunction( - {OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored}); - return target; -} - -void EmitDsSwizzleB32(EmitterState& state, const IR::Instruction& inst) { - const auto source = EmitValueLoad(state, inst.src[0]); - const auto control = inst.src_count > 1u ? inst.src[1].imm & 0xffffu : 0u; - const auto subid = EmitSubgroupLocalInvocationId(state); - const auto target = EmitDsSwizzleTargetLane(state, subid, control); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value, - ConstantU32(state, ScopeSubgroup), source, target}); - const auto exec_active = EmitLaneIndexActiveBool(state, target); - const auto subgroup_active = EmitSubgroupLaneActiveBool(state, target); - const auto source_active = state.builder.AllocateId(); - const auto selected = state.builder.AllocateId(); - // RDNA2 DS_SWIZZLE returns zero when the selected source thread is invalid. - state.builder.AddFunction( - {OpLogicalAnd, state.bool_type, source_active, exec_active, subgroup_active}); - state.builder.AddFunction( - {OpSelect, state.uint_type, selected, source_active, value, ConstantU32(state, 0)}); - EmitStoreU32(state, inst.dst, selected); -} - -} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValue.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValue.cpp index e88b8f5cc..aac13a265 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValue.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValue.cpp @@ -9,6 +9,32 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { namespace { +void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) { + const auto kill_label = state.builder.AllocateId(); + const auto merge_label = state.builder.AllocateId(); + const auto inactive = state.builder.AllocateId(); + state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active}); + state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); + state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label}); + state.builder.AddFunction({OpLabel, kill_label}); + state.builder.AddFunction({OpKill}); + state.builder.AddFunction({OpLabel, merge_label}); +} + +void EmitKillIfPixelValidMaskInactive(EmitterState& state) { + if (state.pixel_valid_mask_variable == 0) { + return; + } + + const auto mask_value = state.builder.AllocateId(); + const auto active = state.builder.AllocateId(); + state.builder.AddFunction( + {OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable}); + state.builder.AddFunction( + {OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)}); + EmitKillIfBoolFalse(state, active); +} + uint32_t BoolConstant(EmitterState& state, bool value) { const auto id = state.builder.AllocateId(); state.builder.AddType({value ? 41u : 42u, state.bool_type, id}); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueAluHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueAluHelpers.cpp new file mode 100644 index 000000000..efd67f910 --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueAluHelpers.cpp @@ -0,0 +1,423 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value, + uint32_t false_value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, true_value, false_value}); + return ret; +} + +uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask); + +void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift, + uint32_t& out_low, uint32_t& out_high) { + const auto amount = EmitAndConstant(state, shift, 63u); + const auto dword_shift = EmitAndConstant(state, amount, 31u); + const auto less32 = state.builder.AllocateId(); + const auto non_zero = state.builder.AllocateId(); + const auto carry_count_base = state.builder.AllocateId(); + const auto carry_count = state.builder.AllocateId(); + const auto low_part = state.builder.AllocateId(); + const auto high_part = state.builder.AllocateId(); + const auto carry_part = state.builder.AllocateId(); + const auto carry = state.builder.AllocateId(); + const auto high_less = state.builder.AllocateId(); + const auto high_ge = state.builder.AllocateId(); + state.builder.AddFunction( + {OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)}); + state.builder.AddFunction( + {OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)}); + state.builder.AddFunction( + {OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)}); + state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, low_part, low, dword_shift}); + state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_part, high, dword_shift}); + state.builder.AddFunction({OpShiftRightLogical, state.uint_type, carry_part, low, carry_count}); + state.builder.AddFunction( + {OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, high_less, high_part, carry}); + state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_ge, low, dword_shift}); + out_low = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0)); + out_high = EmitSelectValueU32(state, less32, high_less, high_ge); +} + +void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, + uint32_t shift, uint32_t& out_low, uint32_t& out_high) { + const auto amount = EmitAndConstant(state, shift, 63u); + const auto dword_shift = EmitAndConstant(state, amount, 31u); + const auto less32 = state.builder.AllocateId(); + const auto non_zero = state.builder.AllocateId(); + const auto carry_count_base = state.builder.AllocateId(); + const auto carry_count = state.builder.AllocateId(); + const auto low_part = state.builder.AllocateId(); + const auto high_part = state.builder.AllocateId(); + const auto carry_part = state.builder.AllocateId(); + const auto carry = state.builder.AllocateId(); + const auto low_less = state.builder.AllocateId(); + const auto low_ge = state.builder.AllocateId(); + state.builder.AddFunction( + {OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)}); + state.builder.AddFunction( + {OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)}); + state.builder.AddFunction( + {OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)}); + state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_part, low, dword_shift}); + state.builder.AddFunction({OpShiftRightLogical, state.uint_type, high_part, high, dword_shift}); + state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, carry_part, high, carry_count}); + state.builder.AddFunction( + {OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, low_less, low_part, carry}); + state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_ge, high, dword_shift}); + out_low = EmitSelectValueU32(state, less32, low_less, low_ge); + out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0)); +} + +uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, ret, value, ConstantU32(state, mask)}); + return ret; +} + +AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs, + uint32_t carry_in) { + const auto pair0 = state.builder.AllocateId(); + const auto sum0 = state.builder.AllocateId(); + const auto carry0 = state.builder.AllocateId(); + const auto pair1 = state.builder.AllocateId(); + const auto sum1 = state.builder.AllocateId(); + const auto carry1 = state.builder.AllocateId(); + const auto carry = state.builder.AllocateId(); + state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair0, lhs, rhs}); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum0, pair0, 0}); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry0, pair0, 1}); + state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair1, sum0, carry_in}); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum1, pair1, 0}); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry1, pair1, 1}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, carry, carry0, carry1}); + return {sum1, carry}; +} + +uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction( + {OpShiftRightLogical, state.uint_type, ret, value, ConstantU32(state, shift)}); + return ret; +} + +uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value, + uint32_t constant) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)}); + return ret; +} + +uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpISub, state.uint_type, ret, ConstantU32(state, constant), value}); + return ret; +} + +uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) { + const auto bits = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.uint_type, bits, f32}); + + const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u); + const auto exp = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu); + const auto mant = EmitAndConstant(state, bits, 0x007fffffu); + + const auto half_exp = state.builder.AllocateId(); + const auto normal_exp = state.builder.AllocateId(); + const auto normal_mant = EmitShiftRightConstant(state, mant, 13); + const auto normal_payload = state.builder.AllocateId(); + const auto normal = state.builder.AllocateId(); + state.builder.AddFunction({OpISub, state.uint_type, half_exp, exp, ConstantU32(state, 112)}); + state.builder.AddFunction( + {OpShiftLeftLogical, state.uint_type, normal_exp, half_exp, ConstantU32(state, 10)}); + state.builder.AddFunction( + {OpBitwiseOr, state.uint_type, normal_payload, normal_exp, normal_mant}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, normal, sign, normal_payload}); + + const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u)); + const auto raw_sub_shift = EmitSubConstantMinusU32(state, 126, exp); + const auto exp_lt_103 = EmitCompareU32Constant(state, OpULessThan, exp, 103); + const auto exp_gt_112 = EmitCompareU32Constant(state, OpUGreaterThan, exp, 112); + const auto sub_shift_low = + EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift); + const auto sub_shift = + EmitSelectValueU32(state, exp_gt_112, ConstantU32(state, 14), sub_shift_low); + const auto sub_mant = state.builder.AllocateId(); + const auto subnormal = state.builder.AllocateId(); + state.builder.AddFunction( + {OpShiftRightLogical, state.uint_type, sub_mant, mant_with_hidden, sub_shift}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, subnormal, sign, sub_mant}); + + const auto nan_payload = + EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u)); + const auto nan = + EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload)); + const auto inf = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u)); + const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu)); + const auto mant_zero = EmitCompareU32Constant(state, OpIEqual, mant, 0); + const auto special = EmitSelectValueU32(state, mant_zero, inf, nan); + + const auto exp_le_112 = EmitCompareU32Constant(state, OpULessThanEqual, exp, 112); + const auto exp_ge_143 = EmitCompareU32Constant(state, OpUGreaterThanEqual, exp, 143); + const auto exp_eq_255 = EmitCompareU32Constant(state, OpIEqual, exp, 255); + const auto finite0 = EmitSelectValueU32(state, exp_le_112, subnormal, normal); + const auto finite1 = EmitSelectValueU32(state, exp_lt_103, sign, finite0); + const auto finite2 = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1); + return EmitAndConstant(state, EmitSelectValueU32(state, exp_eq_255, special, finite2), 0xffffu); +} + +uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) { + const auto cond = state.builder.AllocateId(); + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction( + {max_value ? OpUGreaterThan : OpULessThan, state.bool_type, cond, lhs, rhs}); + state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs}); + return ret; +} + +uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) { + const auto cond = state.builder.AllocateId(); + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction( + {max_value ? OpSGreaterThan : OpSLessThan, state.bool_type, cond, lhs, rhs}); + state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs}); + return ret; +} + +uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.uint_type, ret, value}); + return ret; +} + +uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.float_type, ret, value}); + return ret; +} + +uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpLogicalOr, state.bool_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpLogicalNot, state.bool_type, ret, value}); + return ret; +} + +F32Class EmitClassifyF32(EmitterState& state, uint32_t value) { + F32Class cls; + cls.bits = EmitBitcastF32ToU32(state, value); + const auto abs_bits = EmitAndConstant(state, cls.bits, 0x7fffffffu); + const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u); + const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu); + const auto exponent_max = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u); + const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0); + const auto quiet_bit_clear = + EmitCompareU32Constant(state, OpIEqual, EmitAndConstant(state, cls.bits, 0x00400000u), 0); + cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero); + cls.snan = EmitLogicalAndBool(state, cls.nan, quiet_bit_clear); + cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); + cls.quiet_bits = EmitOrU32(state, cls.bits, ConstantU32(state, 0x00400000u)); + return cls; +} + +uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit, + uint32_t class_match) { + const auto selected = + EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0); + return EmitLogicalAndBool(state, selected, class_match); +} + +uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) { + const auto bits = EmitBitcastF32ToU32(state, value); + const auto sign_bits = EmitAndConstant(state, bits, 0x80000000u); + const auto abs_bits = EmitAndConstant(state, bits, 0x7fffffffu); + const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u); + const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu); + const auto quiet_bits = EmitAndConstant(state, mantissa_bits, 0x00400000u); + + const auto sign = EmitCompareU32Constant(state, OpINotEqual, sign_bits, 0); + const auto positive = EmitLogicalNotBool(state, sign); + const auto exponent_zero = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0); + const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero); + const auto exponent_inf = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u); + const auto finite_exponent = EmitLogicalNotBool(state, exponent_inf); + const auto mantissa_zero = EmitCompareU32Constant(state, OpIEqual, mantissa_bits, 0); + const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero); + const auto quiet = EmitCompareU32Constant(state, OpINotEqual, quiet_bits, 0); + + const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero); + const auto snan = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet)); + const auto qnan = EmitLogicalAndBool(state, nan_common, quiet); + const auto inf = EmitLogicalAndBool(state, exponent_inf, mantissa_zero); + const auto normal = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent); + const auto denorm = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero); + const auto zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); + + uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan); + match = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan)); + match = EmitLogicalOrBool( + state, match, EmitClassMaskBitMatch(state, mask, 2, EmitLogicalAndBool(state, inf, sign))); + match = EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 3, EmitLogicalAndBool(state, normal, sign))); + match = EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 4, EmitLogicalAndBool(state, denorm, sign))); + match = EmitLogicalOrBool( + state, match, EmitClassMaskBitMatch(state, mask, 5, EmitLogicalAndBool(state, zero, sign))); + match = EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 6, EmitLogicalAndBool(state, zero, positive))); + match = EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 7, EmitLogicalAndBool(state, denorm, positive))); + match = EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 8, EmitLogicalAndBool(state, normal, positive))); + return EmitLogicalOrBool( + state, match, + EmitClassMaskBitMatch(state, mask, 9, EmitLogicalAndBool(state, inf, positive))); +} + +uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) { + const auto lhs_class = EmitClassifyF32(state, lhs); + const auto rhs_class = EmitClassifyF32(state, rhs); + + const auto numeric_cond = state.builder.AllocateId(); + state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, state.bool_type, + numeric_cond, lhs, rhs}); + const auto ordered_bits = + EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits); + + const auto both_zero = EmitLogicalAndBool(state, lhs_class.zero, rhs_class.zero); + const auto zero_bits = max_value ? EmitAndU32(state, lhs_class.bits, rhs_class.bits) + : EmitOrU32(state, lhs_class.bits, rhs_class.bits); + const auto numeric_bits = EmitSelectValueU32(state, both_zero, zero_bits, ordered_bits); + + const auto rhs_nan_bits = + EmitSelectValueU32(state, rhs_class.nan, lhs_class.bits, numeric_bits); + const auto non_snan_bits = + EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits); + const auto rhs_snan_bits = + EmitSelectValueU32(state, rhs_class.snan, rhs_class.quiet_bits, non_snan_bits); + const auto result_bits = + EmitSelectValueU32(state, lhs_class.snan, lhs_class.quiet_bits, rhs_snan_bits); + return EmitBitcastU32ToF32(state, result_bits); +} + +uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction( + {OpExtInst, state.float_type, ret, state.glsl_std450, GlslTrunc, value}); + return ret; +} + +uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) { + const auto bits = state.builder.AllocateId(); + const auto abs_bits = state.builder.AllocateId(); + const auto sign_bits = state.builder.AllocateId(); + const auto non_zero = state.builder.AllocateId(); + const auto subnormal = state.builder.AllocateId(); + const auto flush = state.builder.AllocateId(); + const auto selected = state.builder.AllocateId(); + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.uint_type, bits, value}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, sign_bits, bits, ConstantU32(state, 0x80000000u)}); + state.builder.AddFunction( + {OpINotEqual, state.bool_type, non_zero, abs_bits, ConstantU32(state, 0)}); + state.builder.AddFunction( + {OpULessThan, state.bool_type, subnormal, abs_bits, ConstantU32(state, 0x00800000u)}); + state.builder.AddFunction({OpLogicalAnd, state.bool_type, flush, non_zero, subnormal}); + state.builder.AddFunction({OpSelect, state.uint_type, selected, flush, sign_bits, bits}); + state.builder.AddFunction({OpBitcast, state.float_type, ret, selected}); + return ret; +} + +uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero) { + const auto fract = state.builder.AllocateId(); + const auto bits = state.builder.AllocateId(); + const auto abs_bits = state.builder.AllocateId(); + const auto large = state.builder.AllocateId(); + const auto finite = state.builder.AllocateId(); + const auto large_finite = state.builder.AllocateId(); + const auto reduced = state.builder.AllocateId(); + state.builder.AddFunction( + {OpExtInst, state.float_type, fract, state.glsl_std450, GlslFract, src}); + state.builder.AddFunction({OpBitcast, state.uint_type, bits, src}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)}); + state.builder.AddFunction( + {OpUGreaterThanEqual, state.bool_type, large, abs_bits, ConstantU32(state, 0x4b000000u)}); + state.builder.AddFunction( + {OpULessThan, state.bool_type, finite, abs_bits, ConstantU32(state, 0x7f800000u)}); + state.builder.AddFunction({OpLogicalAnd, state.bool_type, large_finite, large, finite}); + state.builder.AddFunction( + {OpSelect, state.float_type, reduced, large_finite, ConstantF32(state, 0), fract}); + if (!preserve_signed_zero) { + return reduced; + } + const auto zero = state.builder.AllocateId(); + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpIEqual, state.bool_type, zero, abs_bits, ConstantU32(state, 0)}); + state.builder.AddFunction({OpSelect, state.float_type, ret, zero, src, reduced}); + return ret; +} + +uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpFNegate, state.float_type, ret, value}); + return ret; +} + +uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction( + {OpExtInst, state.float_type, ret, state.glsl_std450, GlslFAbs, value}); + return ret; +} + +uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) { + const auto unpacked = state.builder.AllocateId(); + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction( + {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, bits}); + state.builder.AddFunction({OpCompositeExtract, state.float_type, ret, unpacked, 0}); + return ret; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueFlow.cpp index 3ce4e1f5d..380be7616 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueFlow.cpp @@ -5,6 +5,26 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { namespace { +uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) { + uint32_t ret = ConstantU32(state, 0); + for (uint32_t i = 0; i < 8u; i++) { + const auto mask = 0x0fu << (i * 4u); + const auto masked = state.builder.AllocateId(); + const auto non_zero = state.builder.AllocateId(); + const auto expanded = state.builder.AllocateId(); + const auto combined = state.builder.AllocateId(); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)}); + state.builder.AddFunction( + {OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)}); + state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero, + ConstantU32(state, mask), ConstantU32(state, 0)}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded}); + ret = combined; + } + return ret; +} + uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) { const auto result = ctx.state.builder.AllocateId(); ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, result, value, @@ -407,20 +427,13 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) { state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)}); return true; case IR::ValueOpcode::DppMoveU32: { - const auto flags = inst.Flags(); - IR::Operand operand; - operand.dpp = true; - operand.dpp_ctrl = flags.control; - operand.dpp_row_mask = flags.row_mask; - operand.dpp_bank_mask = flags.bank_mask; - operand.dpp_fetch_inactive = flags.fetch_inactive; - operand.dpp_bound_ctrl = flags.bound_control; - const auto target = EmitDppTargetLane(state, operand.dpp_ctrl); - const auto shuffled = state.builder.AllocateId(); + const auto flags = inst.Flags(); + const auto target = EmitDppTargetLane(state, flags.control); + const auto shuffled = state.builder.AllocateId(); state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled, ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), target.lane}); - if (operand.dpp_fetch_inactive) { + if (flags.fetch_inactive) { ctx.Define(inst, shuffled); return true; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueHelpers.cpp new file mode 100644 index 000000000..160dcc966 --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueHelpers.cpp @@ -0,0 +1,247 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +uint32_t EmitTrueBool(EmitterState& state) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction( + {OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)}); + return ret; +} + +DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) { + const auto quad_base = state.builder.AllocateId(); + const auto lane = state.builder.AllocateId(); + const auto shift = state.builder.AllocateId(); + const auto selected0 = state.builder.AllocateId(); + const auto selected = state.builder.AllocateId(); + const auto target = state.builder.AllocateId(); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)}); + state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)}); + state.builder.AddFunction( + {OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)}); + state.builder.AddFunction( + {OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected}); + return {target, EmitTrueBool(state)}; +} + +DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount, + bool left) { + const auto row = state.builder.AllocateId(); + const auto lane = state.builder.AllocateId(); + const auto lane_shifted = state.builder.AllocateId(); + const auto target = state.builder.AllocateId(); + const auto valid = state.builder.AllocateId(); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)}); + state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)}); + if (left) { + state.builder.AddFunction( + {OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)}); + state.builder.AddFunction( + {OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)}); + } else { + state.builder.AddFunction( + {OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)}); + state.builder.AddFunction( + {OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)}); + } + state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted}); + return {target, valid}; +} + +DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid, + uint32_t amount) { + const auto row = state.builder.AllocateId(); + const auto lane = state.builder.AllocateId(); + const auto in_high = state.builder.AllocateId(); + const auto minus = state.builder.AllocateId(); + const auto plus = state.builder.AllocateId(); + const auto selected = state.builder.AllocateId(); + const auto target = state.builder.AllocateId(); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)}); + state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)}); + state.builder.AddFunction( + {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)}); + state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)}); + state.builder.AddFunction( + {OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)}); + state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected}); + return {target, EmitTrueBool(state)}; +} + +DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) { + const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u; + const auto lane_mask = half_row ? 7u : 15u; + const auto base = state.builder.AllocateId(); + const auto lane = state.builder.AllocateId(); + const auto mirrored = state.builder.AllocateId(); + const auto target = state.builder.AllocateId(); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)}); + state.builder.AddFunction( + {OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored}); + return {target, EmitTrueBool(state)}; +} + +DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) { + const auto subid = EmitSubgroupLocalInvocationId(state); + if (control <= 0xffu) { + return EmitDppQuadPermTargetLane(state, subid, control); + } + if (control >= 0x101u && control <= 0x10fu) { + return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true); + } + if (control >= 0x111u && control <= 0x11fu) { + return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false); + } + if (control >= 0x121u && control <= 0x12fu) { + return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu); + } + if (control == 0x140u) { + return EmitDppMirrorTargetLane(state, subid, false); + } + if (control == 0x141u) { + return EmitDppMirrorTargetLane(state, subid, true); + } + return {subid, EmitTrueBool(state)}; +} + +uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) { + if (state.subgroup_local_invocation_id_variable == 0) { + return ConstantU32(state, 0); + } + const auto value = state.builder.AllocateId(); + state.builder.AddFunction( + {OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable}); + return value; +} + +uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) { + for (const auto& input: state.inputs) { + if (input.kind == kind) { + return input.variable_id; + } + } + return 0; +} + +const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) { + for (const auto& input: state.inputs) { + if (input.kind == IR::StageInputKind::Parameter && input.location == location) { + return &input; + } + } + return nullptr; +} + +uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) { + const auto variable = InputVariableForKind(state, kind); + if (variable == 0) { + return ConstantU32(state, 0); + } + const auto pointer = state.builder.AllocateId(); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction( + {OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)}); + state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); + return value; +} + +uint32_t EmitLocalInvocationIndex(EmitterState& state) { + const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex); + if (variable == 0) { + return ConstantU32(state, 0); + } + const auto value = state.builder.AllocateId(); + state.builder.AddFunction({OpLoad, state.uint_type, value, variable}); + return value; +} + +uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind, + uint32_t component) { + if ((component & 3u) != 3u) { + return ConstantU32(state, 0); + } + return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u); +} + +uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input, + uint32_t component) { + const auto count = VertexParameterComponentCount(state, input); + const auto kind = VertexParameterScalarKind(state, input.location); + if (component >= count) { + return VertexInputDefaultComponentU32(state, kind, component); + } + + const auto scalar_type = VertexParameterScalarType(state, kind); + uint32_t raw = state.builder.AllocateId(); + if (count == 1u) { + state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id}); + } else { + const auto pointer_type = VertexParameterScalarPointerType(state, kind); + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id, + ConstantU32(state, component)}); + state.builder.AddFunction({OpLoad, scalar_type, raw, pointer}); + } + + if (kind == VertexInputScalarKind::Uint) { + return raw; + } + + const auto bits = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw}); + return bits; +} + +uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) { + const auto active_ballot = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot, + ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)}); + return EmitBallotLaneActiveBool(state, active_ballot, lane); +} +uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) { + const auto low = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0}); + uint32_t mask = low; + if (state.wave_size == 64u) { + const auto high = state.builder.AllocateId(); + const auto in_high = state.builder.AllocateId(); + const auto selected = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1}); + state.builder.AddFunction( + {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)}); + state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low}); + mask = selected; + } + + const auto lane_low = state.builder.AllocateId(); + const auto bit = state.builder.AllocateId(); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)}); + state.builder.AddFunction( + {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low}); + + const auto hit = state.builder.AllocateId(); + const auto active = state.builder.AllocateId(); + const auto in_range = state.builder.AllocateId(); + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit}); + state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)}); + state.builder.AddFunction( + {OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)}); + state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range}); + return ret; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueImage.cpp index 075f08e24..9528f830f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueImage.cpp @@ -6,6 +6,23 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { namespace { +uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) { + uint32_t index = 0; + for (uint32_t i = 0; i < component; i++) { + index += (dmask >> i) & 1u; + } + return index; +} + +uint32_t ImageGatherComponent(uint32_t dmask) { + switch (dmask) { + case 0x2u: return 1; + case 0x4u: return 2; + case 0x8u: return 3; + default: return 0; + } +} + uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) { const auto result = state.builder.AllocateId(); state.builder.AddFunction({opcode, type, result, value}); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemoryHelpers.cpp new file mode 100644 index 000000000..3d5325fa1 --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemoryHelpers.cpp @@ -0,0 +1,371 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpIAdd, state.uint_type, ret, lhs, rhs}); + return ret; +} + +uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs}); + return ret; +} + +uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem, + uint32_t use_pc) { + (void)use_pc; + if (mem.resource >= state.program.info.buffers.size()) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, + "buffer specialization is missing"); + } + return state.program.info.buffers[mem.resource].packed_stride; +} + +Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem, + uint32_t use_pc) { + (void)use_pc; + if (mem.resource >= state.program.info.buffers.size()) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, + "buffer specialization is missing"); + } + return state.program.info.buffers[mem.resource].descriptor_format; +} + +void EmitStorageBufferOffsets(EmitterState& state) { + for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) { + const auto word = + EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u); + const auto shift = ConstantU32(state, (i % 4u) * 8u + 2u); + state.storage_buffer_offsets[i] = EmitBinaryU32( + state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift), + ConstantU32(state, 0x3fu)); + } +} + +bool IsAddressMemoryKind(IR::ResourceKind kind) { + return kind == IR::ResourceKind::ScalarAddress || kind == IR::ResourceKind::Flat || + kind == IR::ResourceKind::Global || kind == IR::ResourceKind::Scratch; +} + +DescriptorResourceBinding +StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) { + (void)use_pc; + const auto binding = + ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource); + if (state.storage_buffer_variable == 0) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, + "storage buffer descriptor array was not emitted"); + } + return binding; +} + +uint32_t EmitStorageBufferIndex(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, + uint32_t use_pc) { + if (IsAddressMemoryKind(mem.kind)) { + return index; + } + const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); + return EmitAddU32(state, index, state.storage_buffer_offsets[binding.array_index]); +} + +uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem, + uint32_t use_pc) { + if (IsAddressMemoryKind(mem.kind)) { + if (state.address_memory_variable == 0) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, + mem.resource, "address memory binding was not emitted"); + } + const auto binding = + ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource); + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer, + state.address_memory_variable, + ConstantU32(state, binding.array_index)}); + return pointer; + } + const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer, + state.storage_buffer_variable, + ConstantU32(state, binding.array_index)}); + return pointer; +} + +uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem, + uint32_t index, uint32_t use_pc) { + index = EmitStorageBufferIndex(state, mem, index, use_pc); + const auto object = EmitStorageBufferObjectPointer(state, mem, use_pc); + const auto length = state.builder.AllocateId(); + const auto in_bounds = state.builder.AllocateId(); + state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0}); + state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length}); + return in_bounds; +} + +uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem, + uint32_t index, uint32_t use_pc) { + index = EmitStorageBufferIndex(state, mem, index, use_pc); + if (IsAddressMemoryKind(mem.kind)) { + if (state.address_memory_variable == 0) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, + mem.resource, "address memory binding was not emitted"); + } + const auto binding = + ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource); + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction( + {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable, + ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); + return pointer; + } + const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction( + {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.storage_buffer_variable, + ConstantU32(state, binding.array_index), ConstantU32(state, 0), index}); + return pointer; +} + +uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) { + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction( + {OpAccessChain, state.ptr_workgroup_uint, pointer, state.lds_variable, index}); + return pointer; +} + +uint32_t LdsDwordCount(const EmitterState& state) { + return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u; +} + +uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) { + const auto in_bounds = state.builder.AllocateId(); + state.builder.AddFunction( + {OpULessThan, state.bool_type, in_bounds, index, ConstantU32(state, LdsDwordCount(state))}); + return in_bounds; +} + +uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, + uint32_t use_pc) { + if (mem.kind == IR::ResourceKind::Lds) { + return EmitLdsElementPointer(state, index); + } + if (mem.kind == IR::ResourceKind::Gds) { + return EmitGdsElementPointer(state, index); + } + return EmitStorageBufferElementPointer(state, mem, index, use_pc); +} + +uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.uint_type, ret, value}); + return ret; +} + +uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.float_type, ret, value}); + return ret; +} + +uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.int_type, ret, value}); + return ret; +} + +uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value, + uint32_t constant) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)}); + return ret; +} + +uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value, + uint32_t false_value) { + const auto ret = state.builder.AllocateId(); + state.builder.AddFunction( + {OpSelect, state.float_type, ret, condition, true_value, false_value}); + return ret; +} + +bool IsSignedFormatComponent(Format::ComponentType type) { + return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm || + type == Format::ComponentType::Sscaled; +} + +uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw) { + const auto unpacked = state.builder.AllocateId(); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction( + {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, raw}); + state.builder.AddFunction({OpCompositeExtract, state.float_type, value, unpacked, 0}); + return EmitTBufferBitcastF32ToU32(state, value); +} + +uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) { + const auto mantissa_bits = bits == 11u ? 6u : 5u; + const auto mantissa_mask = (1u << mantissa_bits) - 1u; + const auto mantissa = + EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask)); + const auto exponent = EmitBinaryU32( + state, OpBitwiseAnd, + EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)), + ConstantU32(state, 0x1fu)); + + const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u)); + const auto exponent_bits = + EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23)); + const auto mantissa_bits_32 = + EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits)); + const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32); + const auto normal = EmitTBufferBitcastU32ToF32(state, normal_bits); + + const auto special_bits = + EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32); + const auto special = EmitTBufferBitcastU32ToF32(state, special_bits); + + const auto mantissa_f32 = state.builder.AllocateId(); + const auto subnormal = state.builder.AllocateId(); + state.builder.AddFunction({OpConvertUToF, state.float_type, mantissa_f32, mantissa}); + state.builder.AddFunction( + {OpFMul, state.float_type, subnormal, mantissa_f32, + ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast(mantissa_bits)))}); + + const auto zero_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 0); + const auto special_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 31); + const auto finite = EmitTBufferSelectF32(state, zero_exp, subnormal, normal); + const auto result = EmitTBufferSelectF32(state, special_exp, special, finite); + return EmitTBufferBitcastF32ToU32(state, result); +} + +uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info, + uint32_t component, uint32_t raw) { + const auto bits = info.component_bits[component]; + switch (info.type) { + case Format::ComponentType::Uint: + case Format::ComponentType::Sint: return raw; + case Format::ComponentType::Uscaled: { + const auto value = state.builder.AllocateId(); + state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw}); + return EmitTBufferBitcastF32ToU32(state, value); + } + case Format::ComponentType::Sscaled: { + const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw}); + return EmitTBufferBitcastF32ToU32(state, value); + } + case Format::ComponentType::Unorm: { + const auto value = state.builder.AllocateId(); + const auto normalized = state.builder.AllocateId(); + const auto max_value = static_cast((1u << bits) - 1u); + state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw}); + state.builder.AddFunction( + {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)}); + return EmitTBufferBitcastF32ToU32(state, normalized); + } + case Format::ComponentType::Snorm: { + const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw); + const auto value = state.builder.AllocateId(); + const auto normalized = state.builder.AllocateId(); + const auto clamped = state.builder.AllocateId(); + const auto max_value = static_cast((1u << (bits - 1u)) - 1u); + state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw}); + state.builder.AddFunction( + {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)}); + state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450, + GlslFMax, normalized, ConstantF32Value(state, -1.0f)}); + return EmitTBufferBitcastF32ToU32(state, clamped); + } + case Format::ComponentType::Float: + if (bits == 32u) { + return raw; + } + if (bits == 16u) { + return EmitHalfToF32Bits(state, raw); + } + return EmitUFloatToF32Bits(state, raw, bits); + default: return raw; + } +} + +void EmitDeviceAtomicMemoryBarrier(EmitterState& state) { + const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory; + state.builder.AddFunction( + {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)}); +} + +uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) { + const auto length = state.builder.AllocateId(); + const auto in_bounds = state.builder.AllocateId(); + state.builder.AddFunction({OpArrayLength, state.uint_type, length, state.gds_variable, 0}); + state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length}); + return in_bounds; +} + +uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) { + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer, + state.gds_variable, ConstantU32(state, 0), index}); + return pointer; +} + +uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) { + if ((control & 0xc000u) == 0xc000u) { + const uint32_t mask = control & 0x1fu; + const uint32_t rotate = (control >> 5u) & 0x1fu; + const uint32_t rotate_delta = (control & 0x400u) != 0u ? ((32u - rotate) & 0x1fu) : rotate; + const auto lane = EmitAndConstant(state, subid, 31); + const auto rotated_sum = EmitAddU32(state, lane, ConstantU32(state, rotate_delta)); + const auto rotated = EmitAndConstant(state, rotated_sum, 31); + const auto kept = EmitAndConstant(state, lane, mask); + const auto moved = EmitAndConstant(state, rotated, (~mask) & 31u); + const auto combined = EmitOrU32(state, kept, moved); + const auto base = EmitAndConstant(state, subid, 0xffffffe0u); + return EmitOrU32(state, base, combined); + } + + if ((control & 0x8000u) != 0) { + const auto lane2 = state.builder.AllocateId(); + const auto shift = state.builder.AllocateId(); + const auto perm0 = state.builder.AllocateId(); + const auto perm = state.builder.AllocateId(); + const auto base = state.builder.AllocateId(); + const auto target = state.builder.AllocateId(); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, lane2, subid, ConstantU32(state, 3)}); + state.builder.AddFunction( + {OpShiftLeftLogical, state.uint_type, shift, lane2, ConstantU32(state, 1)}); + state.builder.AddFunction( + {OpShiftRightLogical, state.uint_type, perm0, ConstantU32(state, control), shift}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, perm, perm0, ConstantU32(state, 3)}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xfffffffcu)}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, perm}); + return target; + } + + const auto lane = state.builder.AllocateId(); + const auto masked = state.builder.AllocateId(); + const auto ored = state.builder.AllocateId(); + const auto xored = state.builder.AllocateId(); + const auto base = state.builder.AllocateId(); + const auto target = state.builder.AllocateId(); + state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)}); + state.builder.AddFunction( + {OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)}); + state.builder.AddFunction( + {OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored}); + return target; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValues.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValues.cpp deleted file mode 100644 index f62731ace..000000000 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValues.cpp +++ /dev/null @@ -1,839 +0,0 @@ -#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" - -namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { - -bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width) { - if (sel <= 3u) { - offset = sel * 8u; - width = 8u; - return true; - } - if (sel == 4u || sel == 5u) { - offset = sel == 5u ? 16u : 0u; - width = 16u; - return true; - } - if (sel == 6u) { - offset = 0; - width = 32u; - return true; - } - return false; -} - -uint32_t EmitSdwaExtractU32(EmitterState& state, const IR::Operand& operand, uint32_t value) { - if (operand.sdwa_sel == 6u && !operand.sdwa_sext) { - return value; - } - - uint32_t offset = 0; - uint32_t width = 0; - if (!SdwaSelectorOffsetWidth(operand.sdwa_sel, offset, width) || width == 32u) { - return value; - } - - if (operand.sdwa_sext) { - const auto signed_value = state.builder.AllocateId(); - const auto extracted = state.builder.AllocateId(); - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.int_type, signed_value, value}); - state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_value, - ConstantU32(state, offset), ConstantU32(state, width)}); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, extracted}); - return bits; - } - - const auto extracted = state.builder.AllocateId(); - state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, value, - ConstantU32(state, offset), ConstantU32(state, width)}); - return extracted; -} - -uint32_t EmitTrueBool(EmitterState& state) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)}); - return ret; -} - -DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) { - const auto quad_base = state.builder.AllocateId(); - const auto lane = state.builder.AllocateId(); - const auto shift = state.builder.AllocateId(); - const auto selected0 = state.builder.AllocateId(); - const auto selected = state.builder.AllocateId(); - const auto target = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)}); - state.builder.AddFunction( - {OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected}); - return {target, EmitTrueBool(state)}; -} - -DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount, - bool left) { - const auto row = state.builder.AllocateId(); - const auto lane = state.builder.AllocateId(); - const auto lane_shifted = state.builder.AllocateId(); - const auto target = state.builder.AllocateId(); - const auto valid = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)}); - if (left) { - state.builder.AddFunction( - {OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)}); - state.builder.AddFunction( - {OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)}); - } else { - state.builder.AddFunction( - {OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)}); - state.builder.AddFunction( - {OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)}); - } - state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted}); - return {target, valid}; -} - -DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid, - uint32_t amount) { - const auto row = state.builder.AllocateId(); - const auto lane = state.builder.AllocateId(); - const auto in_high = state.builder.AllocateId(); - const auto minus = state.builder.AllocateId(); - const auto plus = state.builder.AllocateId(); - const auto selected = state.builder.AllocateId(); - const auto target = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)}); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)}); - state.builder.AddFunction( - {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)}); - state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)}); - state.builder.AddFunction( - {OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)}); - state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected}); - return {target, EmitTrueBool(state)}; -} - -DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) { - const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u; - const auto lane_mask = half_row ? 7u : 15u; - const auto base = state.builder.AllocateId(); - const auto lane = state.builder.AllocateId(); - const auto mirrored = state.builder.AllocateId(); - const auto target = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)}); - state.builder.AddFunction( - {OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored}); - return {target, EmitTrueBool(state)}; -} - -DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) { - const auto subid = EmitSubgroupLocalInvocationId(state); - if (control <= 0xffu) { - return EmitDppQuadPermTargetLane(state, subid, control); - } - if (control >= 0x101u && control <= 0x10fu) { - return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true); - } - if (control >= 0x111u && control <= 0x11fu) { - return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false); - } - if (control >= 0x121u && control <= 0x12fu) { - return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu); - } - if (control == 0x140u) { - return EmitDppMirrorTargetLane(state, subid, false); - } - if (control == 0x141u) { - return EmitDppMirrorTargetLane(state, subid, true); - } - return {subid, EmitTrueBool(state)}; -} - -uint32_t EmitDppValueU32(EmitterState& state, const IR::Operand& operand, uint32_t value) { - if (!operand.dpp) { - return value; - } - - const auto target = EmitDppTargetLane(state, operand.dpp_ctrl); - const auto shuffled = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled, - ConstantU32(state, ScopeSubgroup), value, target.lane}); - if (operand.dpp_fetch_inactive) { - return shuffled; - } - - const auto source_active = EmitLaneIndexActiveBool(state, target.lane); - const auto can_fetch = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active}); - state.builder.AddFunction( - {OpSelect, state.uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)}); - return ret; -} - -uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand) { - uint32_t value = 0; - switch (operand.kind) { - case IR::OperandKind::ImmediateU32: - case IR::OperandKind::PcRelativeU32: - case IR::OperandKind::PcRelativeHighU32: value = ConstantU32(state, operand.imm); break; - case IR::OperandKind::Register: { - const auto pointer = PointerForRegister(state, operand.reg); - if (pointer == 0) { - value = ConstantU32(state, 0); - break; - } - value = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); - break; - } - case IR::OperandKind::Null: value = ConstantU32(state, 0); break; - default: value = ConstantU32(state, 0); break; - } - value = EmitDppValueU32(state, operand, value); - return EmitSdwaExtractU32(state, operand, value); -} - -uint32_t EmitRegisterLoad(EmitterState& state, IR::Register reg) { - if (IsInactiveWave32ExecHigh(state, reg)) { - return ConstantU32(state, 0); - } - const auto pointer = PointerForRegister(state, reg); - if (pointer == 0) { - return ConstantU32(state, 0); - } - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); - return value; -} - -uint32_t WaveMaskHighLoad(EmitterState& state, IR::RegisterFile file) { - if (state.wave_size == 32u && IsMaskRegisterFile(file)) { - return ConstantU32(state, 0); - } - return EmitRegisterLoad(state, {file, 1}); -} - -uint32_t EmitLaneMaskPartU32(EmitterState& state, uint32_t part) { - const auto subid = EmitSubgroupLocalInvocationId(state); - const auto lane = state.builder.AllocateId(); - const auto bit = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane}); - if (part == 0) { - const auto in_low = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpULessThan, state.bool_type, in_low, subid, ConstantU32(state, 32)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, ret, in_low, bit, ConstantU32(state, 0)}); - return ret; - } - if (state.wave_size == 32u) { - return ConstantU32(state, 0); - } - const auto in_high = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, ret, in_high, bit, ConstantU32(state, 0)}); - return ret; -} - -uint32_t EmitThreadMaskBelowPartU32(EmitterState& state, uint32_t part) { - const auto subid = EmitSubgroupLocalInvocationId(state); - const auto lane = state.builder.AllocateId(); - const auto bit = state.builder.AllocateId(); - const auto below = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane}); - state.builder.AddFunction({OpISub, state.uint_type, below, bit, ConstantU32(state, 1)}); - - if (part == 0) { - const auto in_high = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below}); - return ret; - } - if (state.wave_size == 32u) { - return ConstantU32(state, 0); - } - const auto in_high = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)}); - state.builder.AddFunction( - {OpSelect, state.uint_type, ret, in_high, below, ConstantU32(state, 0)}); - return ret; -} - -uint32_t EmitMaskActiveBool(EmitterState& state, IR::RegisterFile file) { - const auto low = EmitRegisterLoad(state, {file, 0}); - const auto condition = state.builder.AllocateId(); - if (state.per_invocation_masks) { - const auto combined = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseOr, state.uint_type, combined, low, WaveMaskHighLoad(state, file)}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, condition, combined, ConstantU32(state, 0)}); - return condition; - } - if (state.exact_subgroup_operations) { - const auto high = WaveMaskHighLoad(state, file); - const auto low_hit = state.builder.AllocateId(); - const auto high_hit = state.builder.AllocateId(); - const auto active = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, active, low_hit, high_hit}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, condition, active, ConstantU32(state, 0)}); - return condition; - } - state.builder.AddFunction( - {OpINotEqual, state.bool_type, condition, low, ConstantU32(state, 0)}); - return condition; -} - -uint32_t EmitMaskZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) { - const auto active = EmitMaskActiveBool(state, file); - if (!zero) { - return active; - } - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalNot, state.bool_type, cond, active}); - return cond; -} - -uint32_t EmitMaskSummaryZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) { - if (state.per_invocation_masks) { - return EmitMaskZeroBool(state, file, zero); - } - if (state.exact_subgroup_operations) { - const auto low = EmitRegisterLoad(state, {file, 0}); - const auto high = WaveMaskHighLoad(state, file); - const auto combined = state.builder.AllocateId(); - const auto active = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)}); - if (!zero) { - return active; - } - const auto inactive = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active}); - return inactive; - } - const auto active = EmitMaskActiveBool(state, file); - if (!zero) { - return active; - } - const auto inactive = state.builder.AllocateId(); - state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active}); - return inactive; -} - -uint32_t EmitExecActiveBool(EmitterState& state) { - return EmitMaskZeroBool(state, IR::RegisterFile::Exec, false); -} - -uint32_t EmitConditionBool(EmitterState& state, const IR::Operand& operand) { - if (operand.kind == IR::OperandKind::Register) { - if (operand.reg.file == IR::RegisterFile::Exec || - operand.reg.file == IR::RegisterFile::Vcc) { - return EmitMaskActiveBool(state, operand.reg.file); - } - } - const auto cond_value = EmitValueLoad(state, operand); - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, cond, cond_value, ConstantU32(state, 0)}); - return cond; -} - -uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) { - if (state.subgroup_local_invocation_id_variable == 0) { - return ConstantU32(state, 0); - } - const auto value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable}); - return value; -} - -uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) { - for (const auto& input: state.inputs) { - if (input.kind == kind) { - return input.variable_id; - } - } - return 0; -} - -uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location) { - for (const auto& input: state.inputs) { - if (input.kind == IR::StageInputKind::Parameter && input.location == location) { - return input.variable_id; - } - } - return 0; -} - -const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) { - for (const auto& input: state.inputs) { - if (input.kind == IR::StageInputKind::Parameter && input.location == location) { - return &input; - } - } - return nullptr; -} - -uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) { - const auto variable = InputVariableForKind(state, kind); - if (variable == 0) { - return ConstantU32(state, 0); - } - const auto pointer = state.builder.AllocateId(); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)}); - state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); - return value; -} - -uint32_t EmitLocalInvocationIndex(EmitterState& state) { - const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex); - if (variable == 0) { - return ConstantU32(state, 0); - } - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, value, variable}); - return value; -} - -uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind, - uint32_t component) { - if ((component & 3u) != 3u) { - return ConstantU32(state, 0); - } - return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u); -} - -uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input, - uint32_t component) { - const auto count = VertexParameterComponentCount(state, input); - const auto kind = VertexParameterScalarKind(state, input.location); - if (component >= count) { - return VertexInputDefaultComponentU32(state, kind, component); - } - - const auto scalar_type = VertexParameterScalarType(state, kind); - uint32_t raw = state.builder.AllocateId(); - if (count == 1u) { - state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id}); - } else { - const auto pointer_type = VertexParameterScalarPointerType(state, kind); - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id, - ConstantU32(state, component)}); - state.builder.AddFunction({OpLoad, scalar_type, raw, pointer}); - } - - if (kind == VertexInputScalarKind::Uint) { - return raw; - } - - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw}); - return bits; -} - -void EmitLoadInputF32(EmitterState& state, const IR::Instruction& inst) { - const auto* input = InputBindingForParameter(state, inst.input_info.attr); - if (input == nullptr || input->variable_id == 0) { - EmitStoreU32(state, inst.dst, ConstantU32(state, 0)); - return; - } - - if (state.stage == ShaderType::Vertex) { - EmitStoreU32(state, inst.dst, - EmitVertexParameterComponentU32(state, *input, inst.input_info.chan & 3u)); - return; - } - - const auto input_value = state.builder.AllocateId(); - const auto component = state.builder.AllocateId(); - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.vec4_float_type, input_value, input->variable_id}); - state.builder.AddFunction( - {OpCompositeExtract, state.float_type, component, input_value, inst.input_info.chan & 3u}); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, component}); - EmitStoreU32(state, inst.dst, bits); -} - -uint32_t EmitSccBool(EmitterState& state, bool non_zero) { - const auto value = EmitRegisterLoad(state, SccRegister()); - const auto cond = state.builder.AllocateId(); - state.builder.AddFunction( - {non_zero ? OpINotEqual : OpIEqual, state.bool_type, cond, value, ConstantU32(state, 0)}); - return cond; -} - -uint32_t EmitFloatLoad(EmitterState& state, const IR::Operand& operand) { - const auto uint_value = EmitValueLoad(state, operand); - auto float_value = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.float_type, float_value, uint_value}); - if (operand.absolute) { - const auto abs_value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, float_value}); - float_value = abs_value; - } - if (operand.negate) { - const auto neg_value = state.builder.AllocateId(); - state.builder.AddFunction({OpFNegate, state.float_type, neg_value, float_value}); - float_value = neg_value; - } - return float_value; -} - -uint32_t ApplyResultModifiersF32(EmitterState& state, uint32_t value, const IR::Operand& dst) { - uint32_t result = value; - if (dst.omod != 0) { - uint32_t bits = 0x3f000000u; - switch (dst.omod) { - case 1u: bits = 0x40000000u; break; - case 2u: bits = 0x40800000u; break; - default: break; - } - const auto scaled = state.builder.AllocateId(); - state.builder.AddFunction( - {OpFMul, state.float_type, scaled, result, ConstantF32(state, bits)}); - result = scaled; - } - if (!dst.clamp) { - return result; - } - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpExtInst, state.float_type, ret, state.glsl_std450, GlslFClamp, - result, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)}); - return ret; -} - -uint32_t EmitMixF32Load(EmitterState& state, const IR::Operand& operand) { - auto value_operand = operand; - value_operand.negate_hi = false; - - if (operand.op_sel_hi) { - auto raw = operand; - raw.sdwa_sel = 6; - raw.sdwa_sext = false; - raw.op_sel = false; - raw.op_sel_hi = false; - raw.negate = false; - raw.negate_hi = false; - raw.absolute = false; - raw.dpp = false; - raw.dpp_fetch_inactive = false; - raw.dpp_bound_ctrl = false; - - const auto bits = EmitValueLoad(state, raw); - const auto unpacked = state.builder.AllocateId(); - auto f32 = state.builder.AllocateId(); - state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, - GlslUnpackHalf2x16, bits}); - state.builder.AddFunction( - {OpCompositeExtract, state.float_type, f32, unpacked, operand.op_sel ? 1u : 0u}); - if (operand.absolute) { - const auto abs_value = state.builder.AllocateId(); - state.builder.AddFunction( - {OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, f32}); - f32 = abs_value; - } - if (value_operand.negate) { - const auto neg_value = state.builder.AllocateId(); - state.builder.AddFunction({OpFNegate, state.float_type, neg_value, f32}); - f32 = neg_value; - } - return f32; - } - - value_operand.op_sel = false; - value_operand.op_sel_hi = false; - return EmitFloatLoad(state, value_operand); -} - -IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset) { - if (offset == 0 || operand.kind != IR::OperandKind::Register) { - return operand; - } - auto ret = operand; - ret.reg.index += offset; - ret.sdwa_sel = 6; - ret.omod = 0; - ret.sdwa_sext = false; - ret.op_sel = false; - ret.op_sel_hi = false; - ret.negate = false; - ret.negate_hi = false; - ret.absolute = false; - ret.clamp = false; - ret.dpp_ctrl = 0; - ret.dpp_row_mask = 0xf; - ret.dpp_bank_mask = 0xf; - ret.dpp_fetch_inactive = false; - ret.dpp_bound_ctrl = false; - ret.dpp = false; - return ret; -} - -uint32_t EmitLaneMaskOperandActiveBool(EmitterState& state, const IR::Operand& operand) { - const auto low = EmitValueLoad(state, operand); - const auto active = state.builder.AllocateId(); - if (state.per_invocation_masks || state.exact_subgroup_operations) { - uint32_t high = ConstantU32(state, 0); - if (operand.kind == IR::OperandKind::Register && - (operand.reg.file == IR::RegisterFile::Scalar || - operand.reg.file == IR::RegisterFile::Vcc || - operand.reg.file == IR::RegisterFile::Exec)) { - high = IsMaskRegisterFile(operand.reg.file) - ? WaveMaskHighLoad(state, operand.reg.file) - : EmitValueLoad(state, OffsetRegisterOperand(operand, 1)); - } - if (state.per_invocation_masks) { - const auto combined = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)}); - return active; - } - const auto low_hit = state.builder.AllocateId(); - const auto high_hit = state.builder.AllocateId(); - const auto hit = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)}); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, hit, low_hit, high_hit}); - state.builder.AddFunction( - {OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)}); - return active; - } - state.builder.AddFunction({OpINotEqual, state.bool_type, active, low, ConstantU32(state, 0)}); - return active; -} - -uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane) { - if (state.per_invocation_masks) { - const auto ballot = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot, - ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)}); - return EmitBallotLaneActiveBool(state, ballot, lane); - } - const auto lane_low = state.builder.AllocateId(); - const auto bit = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low}); - - uint32_t mask = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0}); - if (state.wave_size == 64u) { - const auto high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1}); - const auto in_high = state.builder.AllocateId(); - const auto selected = state.builder.AllocateId(); - state.builder.AddFunction( - {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)}); - state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, mask}); - mask = selected; - } - - const auto hit = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit}); - state.builder.AddFunction({OpINotEqual, state.bool_type, ret, hit, ConstantU32(state, 0)}); - return ret; -} - -uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) { - const auto active_ballot = state.builder.AllocateId(); - state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot, - ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)}); - return EmitBallotLaneActiveBool(state, active_ballot, lane); -} -uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) { - const auto low = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0}); - uint32_t mask = low; - if (state.wave_size == 64u) { - const auto high = state.builder.AllocateId(); - const auto in_high = state.builder.AllocateId(); - const auto selected = state.builder.AllocateId(); - state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1}); - state.builder.AddFunction( - {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)}); - state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low}); - mask = selected; - } - - const auto lane_low = state.builder.AllocateId(); - const auto bit = state.builder.AllocateId(); - state.builder.AddFunction( - {OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low}); - - const auto hit = state.builder.AllocateId(); - const auto active = state.builder.AllocateId(); - const auto in_range = state.builder.AllocateId(); - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit}); - state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)}); - state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range}); - return ret; -} - -uint32_t EmitSequentialValueLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) { - if (offset == 0 || operand.kind == IR::OperandKind::Register) { - return EmitValueLoad(state, OffsetRegisterOperand(operand, offset)); - } - if (operand.kind == IR::OperandKind::ImmediateU32 && offset == 1) { - return ConstantU32(state, operand.sext_64 ? 0xffffffffu : 0u); - } - return ConstantU32(state, 0); -} - -uint32_t EmitSequentialFloatLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) { - const auto value = EmitSequentialValueLoad(state, operand, offset); - const auto f32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.float_type, f32, value}); - return f32; -} - -bool ImageAddressUsesA16(const IR::Instruction& inst) { - return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagA16) != 0u; -} - -bool ImageAddressComponentUsesA16(const IR::Instruction& inst, uint32_t component) { - if (!ImageAddressUsesA16(inst)) { - return false; - } - - uint32_t cursor = 0; - if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagOffset) != 0u) { - if (component == cursor) { - return false; - } - cursor++; - } - if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagBias) != 0u) { - if (component == cursor) { - return true; - } - cursor++; - } - if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u && - component == cursor) { - return false; - } - return true; -} - -uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component) { - if (!ImageAddressUsesA16(inst)) { - return component * 2u; - } - - uint32_t half_component = 0; - for (uint32_t i = 0; i < component; i++) { - const auto half_width = ImageAddressComponentUsesA16(inst, i) ? 1u : 2u; - if (half_width == 2u && (half_component & 1u) != 0u) { - half_component++; - } - half_component += half_width; - } - if (!ImageAddressComponentUsesA16(inst, component) && (half_component & 1u) != 0u) { - half_component++; - } - return half_component; -} - -uint32_t ImageAddressPackedComponent(const IR::Instruction& inst, uint32_t component) { - return ImageAddressHalfComponent(inst, component) / 2u; -} - -IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base, - uint32_t component) { - const auto packed_component = ImageAddressPackedComponent(inst, component); - if (packed_component > 0 && inst.memory.image_nsa_dwords != 0) { - const auto nsa_index = packed_component - 1u; - const auto nsa_count = std::min(inst.memory.image_nsa_dwords * 4u, - Decoder::MaxImageNsaAddressComponents); - if (nsa_index < nsa_count) { - return MakeRegisterOperand(IR::RegisterFile::Vector, - inst.memory.image_nsa_addr[nsa_index]); - } - } - return OffsetRegisterOperand(base, packed_component); -} - -uint32_t EmitImageAddressValueLoad(EmitterState& state, const IR::Instruction& inst, - const IR::Operand& base, uint32_t component) { - const auto operand = ImageAddressOperand(inst, base, component); - const auto packed_component = ImageAddressPackedComponent(inst, component); - if (packed_component == 0 || operand.kind == IR::OperandKind::Register) { - auto value = EmitValueLoad(state, operand); - if (ImageAddressComponentUsesA16(inst, component)) { - if ((ImageAddressHalfComponent(inst, component) & 1u) != 0u) { - value = EmitShiftRightConstant(state, value, 16u); - } - value = EmitAndConstant(state, value, 0xffffu); - } - return value; - } - return ConstantU32(state, 0); -} - -uint32_t EmitImageAddressFloatLoad(EmitterState& state, const IR::Instruction& inst, - const IR::Operand& base, uint32_t component) { - const auto value = EmitImageAddressValueLoad(state, inst, base, component); - if (ImageAddressComponentUsesA16(inst, component)) { - return EmitF16BitsToF32(state, value); - } - const auto f32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.float_type, f32, value}); - return f32; -} - -uint32_t EmitZeroF32(EmitterState& state) { - const auto f32 = state.builder.AllocateId(); - state.builder.AddFunction({OpBitcast, state.float_type, f32, ConstantU32(state, 0)}); - return f32; -} - -} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 962a39710..82c254999 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -4142,79 +4142,6 @@ void TestNewShaderRecompilerImageSampleA16ExceptionComponents() { } } -void TestNewShaderRecompilerImageSampleA16Rdna2AddressOrder() { - namespace Decoder = Libs::Graphics::ShaderRecompiler::Decoder; - namespace Emitter = Libs::Graphics::ShaderRecompiler::Spirv::Emitter; - namespace IR = Libs::Graphics::ShaderRecompiler::IR; - - auto make_inst = [](uint32_t flags) { - IR::Instruction inst{}; - inst.src[0] = Emitter::MakeRegisterOperand(IR::RegisterFile::Vector, 20); - inst.memory.image_sample_flags = flags | Decoder::ImageSampleFlagA16; - inst.memory.image_address_components = 5; - return inst; - }; - auto check_vgpr = [](const IR::Operand &operand, uint32_t expected, - const char *message) { - Check(operand.kind == IR::OperandKind::Register && - operand.reg.file == IR::RegisterFile::Vector && - operand.reg.index == expected, - message); - }; - - { - auto inst = make_inst(Decoder::ImageSampleFlagBias | - Decoder::ImageSampleFlagCompare); - const auto layout = - Emitter::MakeImageSampleLayout(inst, Emitter::ImageViewKind::Dim2D); - Check(layout.bias == 0u, - "RDNA2 sample_c_b A16 bias must precede PCF reference"); - Check(layout.dref == 1u, - "RDNA2 sample_c_b A16 PCF reference must follow bias"); - Check(layout.coord == 2u, - "RDNA2 sample_c_b A16 body must follow bias and PCF"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.bias) == 0u, - "A16 bias should use the first packed half"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.dref) == 2u, - "32-bit PCF reference must align to the next VGPR after A16 bias"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.coord) == 4u, - "A16 coordinates must start after the aligned PCF reference"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.bias), 20, - "A16 sample_c_b bias should read v20"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.dref), 21, - "A16 sample_c_b PCF reference should read v21"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.coord), - 22, "A16 sample_c_b x/y coordinates should start in v22"); - } - - { - auto inst = make_inst(Decoder::ImageSampleFlagOffset | - Decoder::ImageSampleFlagBias | - Decoder::ImageSampleFlagCompare); - const auto layout = - Emitter::MakeImageSampleLayout(inst, Emitter::ImageViewKind::Dim2D); - Check(layout.offset == 0u && layout.bias == 1u && layout.dref == 2u && - layout.coord == 3u, - "RDNA2 sample_c_b_o order must be offset, bias, PCF, body"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.offset) == 0u, - "texel offset should stay a packed 32-bit VGPR"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.bias) == 2u, - "A16 bias should follow the 32-bit texel offset"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.dref) == 4u, - "PCF reference should align after A16 bias"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.coord) == 6u, - "A16 coordinates should start after offset, bias, and PCF"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.offset), - 20, "A16 sample_c_b_o offset should read v20"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.bias), 21, - "A16 sample_c_b_o bias should read v21"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.dref), 22, - "A16 sample_c_b_o PCF reference should read v22"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.coord), - 23, "A16 sample_c_b_o x/y coordinates should start in v23"); - } -} - void TestNewShaderRecompilerImageLoadA16UintCoords() { const uint32_t shader[] = { EncodeMimg0(0x00, 0x3),