mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-21 11:26:56 +02:00
shader_recompiler: remove legacy SPIR-V emitter
This commit is contained in:
nmzik
@@ -953,8 +953,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
|
||||
std::string emit_error;
|
||||
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
|
||||
GetDumpLabel(options), StageName(options.stage), options.shader_hash);
|
||||
if (!Spirv::EmitProgram(ir, resources, options.input_info, spirv, &emit_error,
|
||||
options.dump_ir)) {
|
||||
if (!Spirv::EmitProgram(ir, resources, options.input_info, spirv, &emit_error)) {
|
||||
LOGF("%s typed SPIR-V emit failed: %s\n", GetDumpLabel(options), emit_error.c_str());
|
||||
if (dispatcher_fallback && error != nullptr) {
|
||||
*error = fmt::format("dispatcher fallback failed after {}: {}",
|
||||
|
||||
@@ -312,8 +312,8 @@ bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
|
||||
}
|
||||
|
||||
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
|
||||
ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv, std::string* error,
|
||||
bool preserve_debug_values) {
|
||||
ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv,
|
||||
std::string* error) {
|
||||
using namespace Emitter;
|
||||
|
||||
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
|
||||
@@ -339,7 +339,6 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
|
||||
SetError(error, "SPIR-V emitter requires planned typed SSA");
|
||||
return false;
|
||||
}
|
||||
(void)preserve_debug_values;
|
||||
const auto& value_program = *program.values;
|
||||
if (!IR::ValidateValueProgram(value_program, true, error)) {
|
||||
return false;
|
||||
|
||||
@@ -12,8 +12,7 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv {
|
||||
bool ProgramRequiresExactSubgroupSize(const IR::Program& program);
|
||||
|
||||
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
|
||||
ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv, std::string* error,
|
||||
bool preserve_debug_values = false);
|
||||
ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv, std::string* error);
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -46,34 +46,10 @@ void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg)
|
||||
registers.push_back({reg, 0});
|
||||
}
|
||||
|
||||
IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index) {
|
||||
IR::Operand operand;
|
||||
operand.kind = IR::OperandKind::Register;
|
||||
operand.reg = {file, index};
|
||||
return operand;
|
||||
}
|
||||
|
||||
IR::Register SccRegister() {
|
||||
return {IR::RegisterFile::Scc, static_cast<uint32_t>(Decoder::OperandKind::Scc)};
|
||||
}
|
||||
|
||||
IR::Operand SccOperand() {
|
||||
return MakeRegisterOperand(IR::RegisterFile::Scc,
|
||||
static_cast<uint32_t>(Decoder::OperandKind::Scc));
|
||||
}
|
||||
|
||||
bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg) {
|
||||
return state.wave_size == 32u && reg.file == IR::RegisterFile::Exec && reg.index == 1u;
|
||||
}
|
||||
|
||||
bool IsMaskRegisterFile(IR::RegisterFile file) {
|
||||
return file == IR::RegisterFile::Exec || file == IR::RegisterFile::Vcc;
|
||||
}
|
||||
|
||||
bool IsSccOperand(const IR::Operand& operand) {
|
||||
return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc;
|
||||
}
|
||||
|
||||
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
|
||||
uint32_t index) {
|
||||
return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
|
||||
@@ -277,9 +253,6 @@ uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) {
|
||||
}
|
||||
|
||||
uint32_t ConstantU32(EmitterState& state, uint32_t value);
|
||||
void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value);
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
|
||||
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane);
|
||||
[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource,
|
||||
const char* reason) {
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
uint32_t EmitExportComponentF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
|
||||
if (!enabled || component >= inst.src_count || component >= 4u) {
|
||||
return ConstantF32(state, component == 3u ? 0x3f800000u : 0u);
|
||||
}
|
||||
return EmitFloatLoad(state, inst.src[component]);
|
||||
}
|
||||
|
||||
uint32_t EmitExportVec4F32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.export_info.compr) {
|
||||
uint32_t components[4] = {
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0x3f800000u),
|
||||
};
|
||||
for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
|
||||
pair_index++) {
|
||||
const auto raw = EmitValueLoad(state, inst.src[pair_index]);
|
||||
const auto unpacked = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
|
||||
state.glsl_std450, GlslUnpackHalf2x16, raw});
|
||||
for (uint32_t lane = 0; lane < 2u; lane++) {
|
||||
const auto component = pair_index * 2u + lane;
|
||||
if (((inst.export_info.en >> component) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
components[component] = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, components[component], unpacked, lane});
|
||||
}
|
||||
}
|
||||
const auto vec = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, components[0],
|
||||
components[1], components[2], components[3]});
|
||||
return vec;
|
||||
}
|
||||
|
||||
const auto x = EmitExportComponentF32(state, inst, 0);
|
||||
const auto y = EmitExportComponentF32(state, inst, 1);
|
||||
const auto z = EmitExportComponentF32(state, inst, 2);
|
||||
const auto w = EmitExportComponentF32(state, inst, 3);
|
||||
const auto vec = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, x, y, z, w});
|
||||
return vec;
|
||||
}
|
||||
|
||||
uint32_t EmitExportComponentU32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
|
||||
if (!enabled || component >= inst.src_count || component >= 4u) {
|
||||
return ConstantU32(state, component == 3u ? 1u : 0u);
|
||||
}
|
||||
return EmitValueLoad(state, inst.src[component]);
|
||||
}
|
||||
|
||||
uint32_t EmitExportVec4U32(EmitterState& state, const IR::Instruction& inst) {
|
||||
uint32_t components[4] = {
|
||||
ConstantU32(state, 0u),
|
||||
ConstantU32(state, 0u),
|
||||
ConstantU32(state, 0u),
|
||||
ConstantU32(state, 1u),
|
||||
};
|
||||
|
||||
if (inst.export_info.compr) {
|
||||
for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
|
||||
pair_index++) {
|
||||
const auto raw = EmitValueLoad(state, inst.src[pair_index]);
|
||||
for (uint32_t lane = 0; lane < 2u; lane++) {
|
||||
const auto component = pair_index * 2u + lane;
|
||||
if (((inst.export_info.en >> component) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
components[component] = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitFieldUExtract, state.uint_type, components[component], raw,
|
||||
ConstantU32(state, lane * 16u), ConstantU32(state, 16u)});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (uint32_t component = 0; component < 4u; component++) {
|
||||
components[component] = EmitExportComponentU32(state, inst, component);
|
||||
}
|
||||
}
|
||||
|
||||
const auto vec = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, vec, components[0],
|
||||
components[1], components[2], components[3]});
|
||||
return vec;
|
||||
}
|
||||
|
||||
static bool MrtUsesUintOutput(const EmitterState& state, const IR::Instruction& inst) {
|
||||
return state.stage == ShaderType::Pixel && inst.export_info.kind == IR::ExportTargetKind::Mrt &&
|
||||
inst.export_info.index < std::size(state.input_info.pixel->target_output_mode) &&
|
||||
state.input_info.pixel->target_output_mode[inst.export_info.index] == 7u;
|
||||
}
|
||||
|
||||
uint32_t ApplyMrtExportMapping(EmitterState& state, const IR::Instruction& inst, uint32_t value,
|
||||
uint32_t vector_type) {
|
||||
if (state.stage != ShaderType::Pixel || inst.export_info.kind != IR::ExportTargetKind::Mrt ||
|
||||
inst.export_info.index >= state.input_info.pixel->target_export_mapping.size()) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const auto mapping = state.input_info.pixel->target_export_mapping[inst.export_info.index];
|
||||
if (mapping.IsIdentity()) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const auto mapped = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value, mapping.Map(0),
|
||||
mapping.Map(1), mapping.Map(2), mapping.Map(3)});
|
||||
return mapped;
|
||||
}
|
||||
|
||||
bool ExportWritesData(const IR::Instruction& inst) {
|
||||
switch (inst.export_info.kind) {
|
||||
case IR::ExportTargetKind::Null:
|
||||
case IR::ExportTargetKind::Primitive: return false;
|
||||
case IR::ExportTargetKind::MrtZ: return (inst.export_info.en & 0x5u) != 0;
|
||||
default: return inst.export_info.en != 0;
|
||||
}
|
||||
}
|
||||
|
||||
void EmitMrtZExport(EmitterState& state, const IR::Instruction& inst) {
|
||||
if ((inst.export_info.en & 0x1u) != 0 && state.depth_variable != 0) {
|
||||
state.builder.AddFunction(
|
||||
{OpStore, state.depth_variable, EmitExportComponentF32(state, inst, 0)});
|
||||
}
|
||||
|
||||
if ((inst.export_info.en & 0x4u) != 0 && state.sample_mask_variable != 0) {
|
||||
const auto raw =
|
||||
inst.src_count > 2u ? EmitValueLoad(state, inst.src[2]) : ConstantU32(state, 0);
|
||||
const auto mask = state.builder.AllocateId();
|
||||
const auto ptr = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, mask, raw});
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_output_int, ptr,
|
||||
state.sample_mask_variable, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, ptr, mask});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitExport(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::Null ||
|
||||
inst.export_info.kind == IR::ExportTargetKind::Primitive) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ExportWritesData(inst)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::MrtZ) {
|
||||
EmitMrtZExport(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto variable = OutputVariableForExport(state, inst.export_info);
|
||||
if (variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto uint_output = MrtUsesUintOutput(state, inst);
|
||||
const auto vector_type = uint_output ? state.vec4_uint_type : state.vec4_float_type;
|
||||
const auto value = ApplyMrtExportMapping(
|
||||
state, inst, uint_output ? EmitExportVec4U32(state, inst) : EmitExportVec4F32(state, inst),
|
||||
vector_type);
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::Position) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, state.ptr_output_vec4_float, pointer, variable, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, pointer, value});
|
||||
return;
|
||||
}
|
||||
|
||||
state.builder.AddFunction({OpStore, variable, value});
|
||||
}
|
||||
|
||||
bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst) {
|
||||
return state.stage == ShaderType::Pixel && inst.export_info.vm && state.needs_pixel_valid_mask;
|
||||
}
|
||||
|
||||
void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
|
||||
const auto kill_label = state.builder.AllocateId();
|
||||
const auto merge_label = state.builder.AllocateId();
|
||||
const auto inactive = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
|
||||
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
|
||||
state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
|
||||
state.builder.AddFunction({OpLabel, kill_label});
|
||||
state.builder.AddFunction({OpKill});
|
||||
state.builder.AddFunction({OpLabel, merge_label});
|
||||
}
|
||||
|
||||
void EmitUpdatePixelValidMask(EmitterState& state) {
|
||||
if (state.pixel_valid_mask_variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto active = EmitExecActiveBool(state);
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
|
||||
}
|
||||
|
||||
void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
|
||||
if (state.pixel_valid_mask_variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto mask_value = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)});
|
||||
EmitKillIfBoolFalse(state, active);
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -1,379 +0,0 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
void EmitMoveU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
EmitStoreU32(state, inst.dst, value);
|
||||
}
|
||||
|
||||
void EmitMoveF32Bits(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto value = EmitFloatLoad(state, inst.src[0]);
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
|
||||
EmitStoreU32(state, inst.dst, bits);
|
||||
}
|
||||
|
||||
uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base) {
|
||||
uint32_t limit = base.index + 1u;
|
||||
for (const auto& binding: state.registers) {
|
||||
if (binding.reg.file == IR::RegisterFile::Vector) {
|
||||
limit = std::max(limit, binding.reg.index + 1u);
|
||||
}
|
||||
}
|
||||
return std::min(limit, 256u);
|
||||
}
|
||||
|
||||
void EmitMoveRelSourceU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.src[0].kind != IR::OperandKind::Register ||
|
||||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
|
||||
EmitMoveU32(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto base_reg = inst.src[0].reg;
|
||||
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
|
||||
auto selected = EmitRegisterLoad(state, base_reg);
|
||||
const auto limit = MoveRelRegisterLimit(state, base_reg);
|
||||
for (uint32_t reg = base_reg.index + 1u; reg < limit; reg++) {
|
||||
const auto candidate = EmitRegisterLoad(state, {IR::RegisterFile::Vector, reg});
|
||||
const auto match = state.builder.AllocateId();
|
||||
const auto next = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, next, match, candidate, selected});
|
||||
selected = next;
|
||||
}
|
||||
EmitStoreU32(state, inst.dst, selected);
|
||||
}
|
||||
|
||||
void EmitMoveRelDestU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.dst.kind != IR::OperandKind::Register ||
|
||||
inst.dst.reg.file != IR::RegisterFile::Vector) {
|
||||
EmitMoveU32(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto base_reg = inst.dst.reg;
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
|
||||
const auto exec_active = EmitExecActiveBool(state);
|
||||
const auto limit = MoveRelRegisterLimit(state, base_reg);
|
||||
for (uint32_t reg = base_reg.index; reg < limit; reg++) {
|
||||
const auto pointer = PointerForRegister(state, {IR::RegisterFile::Vector, reg});
|
||||
if (pointer == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto old_value = state.builder.AllocateId();
|
||||
const auto match = state.builder.AllocateId();
|
||||
const auto write = state.builder.AllocateId();
|
||||
const auto merged = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
|
||||
state.builder.AddFunction(
|
||||
{OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, write, exec_active, match});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, merged, write, value, old_value});
|
||||
state.builder.AddFunction({OpStore, pointer, merged});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitMoveU64(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
|
||||
IsMaskRegisterFile(inst.dst.reg.file)) {
|
||||
EmitPerInvocationMask(state, inst.dst, EmitLaneMaskOperandActiveBool(state, inst.src[0]));
|
||||
return;
|
||||
}
|
||||
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
EmitStoreU32(state, inst.dst, low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high);
|
||||
}
|
||||
|
||||
uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) {
|
||||
uint32_t ret = ConstantU32(state, 0);
|
||||
for (uint32_t i = 0; i < 8u; i++) {
|
||||
const auto mask = 0x0fu << (i * 4u);
|
||||
const auto masked = state.builder.AllocateId();
|
||||
const auto non_zero = state.builder.AllocateId();
|
||||
const auto expanded = state.builder.AllocateId();
|
||||
const auto combined = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero,
|
||||
ConstantU32(state, mask), ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded});
|
||||
ret = combined;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
void EmitWqmB64(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (!state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
|
||||
inst.dst.reg.file == IR::RegisterFile::Scalar) {
|
||||
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
const auto ret_low = EmitWqmLaneU32(state, low);
|
||||
const auto ret_high = EmitWqmLaneU32(state, high);
|
||||
EmitStoreU32(state, inst.dst, ret_low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high);
|
||||
return;
|
||||
}
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup),
|
||||
EmitLaneMaskOperandActiveBool(state, inst.src[0])});
|
||||
const auto low = state.builder.AllocateId();
|
||||
const auto high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
|
||||
const auto wqm_low = EmitWqmLaneU32(state, low);
|
||||
const auto wqm_high = EmitWqmLaneU32(state, high);
|
||||
const auto lane = EmitSubgroupLocalInvocationId(state);
|
||||
const auto high_half = state.builder.AllocateId();
|
||||
const auto mask = state.builder.AllocateId();
|
||||
const auto bit_index = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, high_half, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, mask, high_half, wqm_high, wqm_low});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, bit_index, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_index});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
|
||||
if (state.per_invocation_masks) {
|
||||
EmitPerInvocationMask(state, inst.dst, active);
|
||||
} else {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, result, active, ConstantU32(state, 1),
|
||||
ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, inst.dst, result);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ConstantU32(state, 0));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitSaveexecPerInvocation(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto old_exec = EmitExecActiveBool(state);
|
||||
const auto src = EmitLaneMaskOperandActiveBool(state, inst.src[0]);
|
||||
EmitPerInvocationMask(state, inst.dst, old_exec);
|
||||
|
||||
uint32_t lhs = old_exec;
|
||||
uint32_t rhs = src;
|
||||
bool use_or = false;
|
||||
switch (inst.saveexec_mode) {
|
||||
case IR::SaveexecMode::And: break;
|
||||
case IR::SaveexecMode::Andn1: rhs = EmitLogicalNotBool(state, src); break;
|
||||
case IR::SaveexecMode::Orn2:
|
||||
lhs = EmitLogicalNotBool(state, old_exec);
|
||||
use_or = true;
|
||||
break;
|
||||
}
|
||||
const auto result =
|
||||
use_or ? EmitLogicalOrBool(state, lhs, rhs) : EmitLogicalAndBool(state, lhs, rhs);
|
||||
EmitPerInvocationMask(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), result);
|
||||
EmitStoreSccBool(state, result);
|
||||
}
|
||||
|
||||
void EmitSaveexecB32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (state.per_invocation_masks) {
|
||||
EmitSaveexecPerInvocation(state, inst);
|
||||
return;
|
||||
}
|
||||
const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
const auto src_low = EmitValueLoad(state, inst.src[0]);
|
||||
|
||||
EmitStoreU32(state, inst.dst, old_low);
|
||||
|
||||
uint32_t rhs_low = src_low;
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
|
||||
rhs_low = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
|
||||
}
|
||||
|
||||
const auto new_low = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, new_low, rhs_low, old_low});
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
|
||||
|
||||
if (state.wave_size == 32u) {
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), ConstantU32(state, 0));
|
||||
}
|
||||
|
||||
const auto cond = state.builder.AllocateId();
|
||||
const auto scc = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, cond, new_low, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, SccOperand(), scc);
|
||||
}
|
||||
|
||||
void EmitSaveexecB64(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (state.per_invocation_masks) {
|
||||
EmitSaveexecPerInvocation(state, inst);
|
||||
return;
|
||||
}
|
||||
const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
const auto old_high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
|
||||
const auto src_low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto src_high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
|
||||
EmitStoreU32(state, inst.dst, old_low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), old_high);
|
||||
|
||||
uint32_t lhs_low = old_low;
|
||||
uint32_t lhs_high = old_high;
|
||||
uint32_t rhs_low = src_low;
|
||||
uint32_t rhs_high = src_high;
|
||||
uint32_t opcode = OpBitwiseAnd;
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
|
||||
rhs_low = state.builder.AllocateId();
|
||||
rhs_high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
|
||||
state.builder.AddFunction({OpNot, state.uint_type, rhs_high, src_high});
|
||||
}
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Orn2) {
|
||||
lhs_low = state.builder.AllocateId();
|
||||
lhs_high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpNot, state.uint_type, lhs_low, old_low});
|
||||
state.builder.AddFunction({OpNot, state.uint_type, lhs_high, old_high});
|
||||
opcode = OpBitwiseOr;
|
||||
}
|
||||
|
||||
const auto new_low = state.builder.AllocateId();
|
||||
const auto new_high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, state.uint_type, new_low, lhs_low, rhs_low});
|
||||
state.builder.AddFunction({opcode, state.uint_type, new_high, lhs_high, rhs_high});
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), new_high);
|
||||
|
||||
const auto active_new_high = state.wave_size == 32u ? ConstantU32(state, 0) : new_high;
|
||||
const auto mask = state.builder.AllocateId();
|
||||
const auto cond = state.builder.AllocateId();
|
||||
const auto scc = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, mask, new_low, active_new_high});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, cond, mask, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, SccOperand(), scc);
|
||||
}
|
||||
|
||||
void EmitReadFirstLaneU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto src = EmitValueLoad(state, inst.src[0]);
|
||||
const auto active = EmitExecActiveBool(state);
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
const auto first_lane = state.builder.AllocateId();
|
||||
const auto first_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), active});
|
||||
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first_lane,
|
||||
ConstantU32(state, ScopeSubgroup), ballot});
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, first_value,
|
||||
ConstantU32(state, ScopeSubgroup), src, first_lane});
|
||||
EmitStoreU32(state, inst.dst, first_value);
|
||||
}
|
||||
|
||||
uint32_t EmitLaneIndex(EmitterState& state, const IR::Operand& operand) {
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto mask = state.wave_size == 32u ? 31u : 63u;
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, EmitValueLoad(state, operand),
|
||||
ConstantU32(state, mask)});
|
||||
return lane;
|
||||
}
|
||||
|
||||
void EmitReadLaneU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto src = EmitValueLoad(state, inst.src[0]);
|
||||
const auto lane = EmitLaneIndex(state, inst.src[1]);
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value,
|
||||
ConstantU32(state, ScopeSubgroup), src, lane});
|
||||
EmitStoreU32(state, inst.dst, value);
|
||||
}
|
||||
|
||||
void EmitWriteLaneU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto lane = EmitLaneIndex(state, inst.src[1]);
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto pointer =
|
||||
inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(state, inst.dst.reg) : 0;
|
||||
const auto old = pointer != 0 ? state.builder.AllocateId() : ConstantU32(state, 0);
|
||||
const auto merged = state.builder.AllocateId();
|
||||
if (pointer != 0) {
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, old, pointer});
|
||||
}
|
||||
state.builder.AddFunction({OpIEqual, state.bool_type, hit, subid, lane});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, merged, hit, value, old});
|
||||
if (pointer != 0) {
|
||||
// RDNA2 V_WRITELANE_B32 ignores EXEC, unlike ordinary VGPR destination writes.
|
||||
state.builder.AddFunction({OpStore, pointer, merged});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitPermlaneB32(EmitterState& state, const IR::Instruction& inst, bool x16) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto lane8 = state.builder.AllocateId();
|
||||
const auto shift = state.builder.AllocateId();
|
||||
const auto upper = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto index0 = state.builder.AllocateId();
|
||||
const auto index1 = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto src1 = EmitValueLoad(state, inst.src[1]);
|
||||
const auto src2 = EmitValueLoad(state, inst.src[2]);
|
||||
uint32_t row_value = row;
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
if (x16) {
|
||||
row_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
|
||||
}
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, upper, src2, src1});
|
||||
state.builder.AddFunction({OpShiftRightLogical, state.uint_type, index0, selected, shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, index1, index0, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index1});
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), value, target});
|
||||
uint32_t ret = shuffled;
|
||||
if (!inst.dst.op_sel) {
|
||||
const auto source_active = EmitLaneIndexActiveBool(state, target);
|
||||
ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, source_active, shuffled, ConstantU32(state, 0)});
|
||||
}
|
||||
EmitStoreU32(state, inst.dst, ret);
|
||||
}
|
||||
|
||||
void EmitControlNop(EmitterState& state, const IR::Instruction& inst) {
|
||||
(void)state;
|
||||
(void)inst;
|
||||
}
|
||||
|
||||
void EmitWaitcnt(EmitterState& state, const IR::Instruction& inst) {
|
||||
(void)state;
|
||||
(void)inst;
|
||||
}
|
||||
|
||||
void EmitBarrier(EmitterState& state, const IR::Instruction& inst) {
|
||||
(void)inst;
|
||||
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
|
||||
state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
|
||||
ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)});
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -1,333 +0,0 @@
|
||||
#include "common/assert.h"
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
uint32_t EmitCubeAxisF32(EmitterState& state, uint32_t value) {
|
||||
const auto normalized = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpFSub, state.float_type, normalized, value, ConstantF32(state, 0x3f800000u)});
|
||||
return normalized;
|
||||
}
|
||||
|
||||
uint32_t EmitCubeLayerF32(EmitterState& state, uint32_t face_id) {
|
||||
// Sampled RDNA2 cubemaps encode face_id as slice * 8 + face. The native
|
||||
// 2D-array view stores six contiguous faces per slice, so remove the two
|
||||
// reserved face IDs from every preceding slice.
|
||||
const auto guest_layer = state.builder.AllocateId();
|
||||
const auto slice = state.builder.AllocateId();
|
||||
const auto padding = state.builder.AllocateId();
|
||||
const auto host_layer = state.builder.AllocateId();
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpConvertFToU, state.uint_type, guest_layer, face_id});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, slice, guest_layer, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, padding, slice, ConstantU32(state, 1)});
|
||||
state.builder.AddFunction({OpISub, state.uint_type, host_layer, guest_layer, padding});
|
||||
state.builder.AddFunction({OpConvertUToF, state.float_type, result, host_layer});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t EmitImageCoordF32Impl(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& address, uint32_t first_component,
|
||||
uint32_t components) {
|
||||
auto x = EmitImageAddressFloatLoad(state, inst, address, first_component);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
auto y = inst.memory.image_address_components > first_component + 1u
|
||||
? EmitImageAddressFloatLoad(state, inst, address, first_component + 1u)
|
||||
: EmitZeroF32(state);
|
||||
if (inst.memory.image_cube) {
|
||||
// RDNA2 sampled cubemap S/T coordinates are biased by +1 relative to
|
||||
// normalized 2D-array coordinates.
|
||||
x = EmitCubeAxisF32(state, x);
|
||||
y = EmitCubeAxisF32(state, y);
|
||||
}
|
||||
const auto coord = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
auto z = inst.memory.image_address_components > first_component + 2u
|
||||
? EmitImageAddressFloatLoad(state, inst, address, first_component + 2u)
|
||||
: EmitZeroF32(state);
|
||||
if (inst.memory.image_cube) {
|
||||
z = EmitCubeLayerF32(state, z);
|
||||
}
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag) {
|
||||
return (inst.memory.image_sample_flags & flag) != 0;
|
||||
}
|
||||
|
||||
ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view) {
|
||||
ImageSampleLayout layout;
|
||||
uint32_t cursor = 0;
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
|
||||
layout.offset = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
|
||||
layout.bias = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare)) {
|
||||
layout.dref = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
layout.grad_x = cursor;
|
||||
cursor += components;
|
||||
layout.grad_y = cursor;
|
||||
cursor += components;
|
||||
}
|
||||
layout.coord = cursor;
|
||||
cursor += ImageViewCoordinateComponents(view);
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod)) {
|
||||
layout.lod = cursor++;
|
||||
}
|
||||
return layout;
|
||||
}
|
||||
|
||||
uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view) {
|
||||
return EmitImageCoordF32Impl(state, inst, inst.src[0], layout.coord,
|
||||
ImageViewCoordinateComponents(view));
|
||||
}
|
||||
|
||||
uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero) ||
|
||||
layout.lod == NoImageComponent || inst.memory.image_address_components <= layout.lod) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.lod);
|
||||
}
|
||||
|
||||
uint32_t EmitImageDrefF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (layout.dref == NoImageComponent || inst.memory.image_address_components <= layout.dref) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.dref);
|
||||
}
|
||||
|
||||
uint32_t EmitImageBiasF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (layout.bias == NoImageComponent || inst.memory.image_address_components <= layout.bias) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.bias);
|
||||
}
|
||||
|
||||
uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t first_component, ImageViewKind view) {
|
||||
const auto x = inst.memory.image_address_components > first_component
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component)
|
||||
: EmitZeroF32(state);
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > first_component + 1u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 1u)
|
||||
: EmitZeroF32(state);
|
||||
const auto grad = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z =
|
||||
inst.memory.image_address_components > first_component + 2u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 2u)
|
||||
: EmitZeroF32(state);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, grad, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, grad, x, y});
|
||||
}
|
||||
return grad;
|
||||
}
|
||||
|
||||
uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view) {
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
if (layout.offset == NoImageComponent ||
|
||||
inst.memory.image_address_components <= layout.offset) {
|
||||
const auto zero = ConstantI32(state, 0);
|
||||
if (components == 1u) {
|
||||
return zero;
|
||||
}
|
||||
const auto ret = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec3_int_type, ret, zero, zero, zero});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_int_type, ret, zero, zero});
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
const auto packed_bits = EmitImageAddressValueLoad(state, inst, inst.src[0], layout.offset);
|
||||
const auto packed_i32 = state.builder.AllocateId();
|
||||
const auto offset_x = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, packed_i32, packed_bits});
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_x, packed_i32,
|
||||
ConstantI32(state, 0), ConstantI32(state, 6)});
|
||||
if (components == 1u) {
|
||||
return offset_x;
|
||||
}
|
||||
const auto offset_y = state.builder.AllocateId();
|
||||
const auto offset = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_y, packed_i32,
|
||||
ConstantI32(state, 8), ConstantI32(state, 6)});
|
||||
if (components == 3u) {
|
||||
const auto offset_z = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_z, packed_i32,
|
||||
ConstantI32(state, 16), ConstantI32(state, 6)});
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec3_int_type, offset, offset_x, offset_y, offset_z});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec2_int_type, offset, offset_x, offset_y});
|
||||
}
|
||||
return offset;
|
||||
}
|
||||
|
||||
uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst, ImageViewKind view) {
|
||||
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[1], 0);
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > 1u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[1], 1)
|
||||
: ConstantU32(state, 0);
|
||||
const auto coord = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z = inst.memory.image_address_components > 2u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[1], 2)
|
||||
: ConstantU32(state, 0);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst,
|
||||
ImageViewKind view) {
|
||||
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[0], 0);
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > 1u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[0], 1)
|
||||
: ConstantU32(state, 0);
|
||||
const auto coord = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z = inst.memory.image_address_components > 2u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[0], 2)
|
||||
: ConstantU32(state, 0);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& address, ImageViewKind view) {
|
||||
if (!inst.memory.image_has_mip || inst.memory.image_address_components == 0u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto lod_component = ImageViewCoordinateComponents(view);
|
||||
if (inst.memory.image_address_components <= lod_component) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitImageAddressValueLoad(state, inst, address, lod_component);
|
||||
}
|
||||
|
||||
uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst,
|
||||
ImageViewKind view) {
|
||||
// OpImageQueryLod takes only the spatial coordinates, even for arrayed images.
|
||||
return EmitImageCoordF32Impl(state, inst, inst.src[0], 0, ImageViewSpatialComponents(view));
|
||||
}
|
||||
|
||||
uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
|
||||
uint32_t index = 0;
|
||||
for (uint32_t i = 0; i < component; i++) {
|
||||
index += (dmask >> i) & 1u;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreComponentF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
if (((dmask >> component) & 1u) == 0) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitSequentialFloatLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
|
||||
}
|
||||
|
||||
uint32_t InverseStorageSwizzleComponent(uint32_t swizzle, uint32_t component) {
|
||||
const auto target = 4u + component;
|
||||
for (uint32_t source = 0; source < 4u; source++) {
|
||||
if (((swizzle >> (source * 3u)) & 0x7u) == target) {
|
||||
return source;
|
||||
}
|
||||
}
|
||||
return UINT32_MAX;
|
||||
}
|
||||
|
||||
uint32_t StorageImageSwizzle(const EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.memory.resource >= state.program.info.images.size()) {
|
||||
EXIT("storage image instruction has no specialized swizzle\n");
|
||||
}
|
||||
return state.program.info.images[inst.memory.resource].storage_swizzle;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreTexelF32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto swizzle = StorageImageSwizzle(state, inst);
|
||||
const auto component_value = [&](uint32_t component) {
|
||||
const auto source = InverseStorageSwizzleComponent(swizzle, component);
|
||||
return source < 4u ? EmitImageStoreComponentF32(state, inst, source) : EmitZeroF32(state);
|
||||
};
|
||||
const auto x = component_value(0);
|
||||
const auto y = component_value(1);
|
||||
const auto z = component_value(2);
|
||||
const auto w = component_value(3);
|
||||
const auto texel = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, texel, x, y, z, w});
|
||||
return texel;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreComponentU32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
if (((dmask >> component) & 1u) == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitSequentialValueLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreTexelU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto swizzle = StorageImageSwizzle(state, inst);
|
||||
const auto component_value = [&](uint32_t component) {
|
||||
const auto source = InverseStorageSwizzleComponent(swizzle, component);
|
||||
return source < 4u ? EmitImageStoreComponentU32(state, inst, source)
|
||||
: ConstantU32(state, 0);
|
||||
};
|
||||
const auto x = component_value(0);
|
||||
const auto y = component_value(1);
|
||||
const auto z = component_value(2);
|
||||
const auto w = component_value(3);
|
||||
const auto texel = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, texel, x, y, z, w});
|
||||
return texel;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -1,342 +0,0 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
namespace {
|
||||
|
||||
uint32_t ConstantVec2I32(EmitterState& state, int32_t x, int32_t y) {
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, state.vec2_int_type, value, ConstantI32(state, x),
|
||||
ConstantI32(state, y)});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t ConstantImageGatherHorizontalOffsets(EmitterState& state, ImageViewKind view) {
|
||||
if (ImageViewSpatialComponents(view) == 1u) {
|
||||
const auto type = state.builder.AllocateId();
|
||||
state.builder.AddType({OpTypeArray, type, state.int_type, ConstantU32(state, 4)});
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, type, value, ConstantI32(state, -1),
|
||||
ConstantI32(state, 0), ConstantI32(state, 1),
|
||||
ConstantI32(state, 2)});
|
||||
return value;
|
||||
}
|
||||
const auto offset0 = ConstantVec2I32(state, -1, 0);
|
||||
const auto offset1 = ConstantVec2I32(state, 0, 0);
|
||||
const auto offset2 = ConstantVec2I32(state, 1, 0);
|
||||
const auto offset3 = ConstantVec2I32(state, 2, 0);
|
||||
const auto type = state.builder.AllocateId();
|
||||
state.builder.AddType({OpTypeArray, type, state.vec2_int_type, ConstantU32(state, 4)});
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t LoadStorageImageDescriptorAtIndex(EmitterState& state, uint32_t resource,
|
||||
uint32_t array_index, bool uint_image,
|
||||
ImageViewKind view) {
|
||||
const auto kind = StorageBindingKind(uint_image, view);
|
||||
const auto& descriptors = state.storage_images[StorageImageIndex(uint_image, view)];
|
||||
const auto pointer =
|
||||
DescriptorElementPointer(state, descriptors.pointer_type, descriptors.variable, array_index,
|
||||
kind, resource, "storage image descriptor array was not emitted");
|
||||
const auto image = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, descriptors.image_type, image, pointer});
|
||||
return image;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
uint32_t EmitImageGetResinfoComponent(EmitterState& state, uint32_t image, uint32_t size,
|
||||
uint32_t component_index, ImageViewKind view) {
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
switch (component_index) {
|
||||
case 0: {
|
||||
if (components == 1u) {
|
||||
return size;
|
||||
}
|
||||
const auto width = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, width, size, 0});
|
||||
return width;
|
||||
}
|
||||
case 1: {
|
||||
if (components < 2u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto height = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, height, size, 1});
|
||||
return height;
|
||||
}
|
||||
case 2:
|
||||
if (components == 3u) {
|
||||
const auto depth = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, depth, size, 2});
|
||||
return depth;
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
case 3: {
|
||||
const auto levels = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpImageQueryLevels, state.uint_type, levels, image});
|
||||
return levels;
|
||||
}
|
||||
default: return ConstantU32(state, 0);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGetResinfo(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
|
||||
const auto mip_level = EmitValueLoad(state, inst.src[0]);
|
||||
|
||||
const auto size = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpImageQuerySizeLod, ImageViewSizeType(state, view), size, image, mip_level});
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component =
|
||||
EmitImageGetResinfoComponent(state, image, size, component_index, view);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), component);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGetLod(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto lod = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpImageQueryLod, state.vec2_float_type, lod, sampled_image,
|
||||
EmitImageQueryCoordF32(state, inst, view)});
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 2u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, component, lod, component_index});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageLoad(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
|
||||
const auto color = state.builder.AllocateId();
|
||||
const auto coord = EmitImageLoadCoordU32(state, inst, view);
|
||||
if (ImageSpirvMultisampled(view) != 0) {
|
||||
const auto sample = EmitImageAddressValueLoad(state, inst, inst.src[0],
|
||||
ImageViewCoordinateComponents(view));
|
||||
state.builder.AddFunction({OpImageFetch,
|
||||
integer ? state.vec4_uint_type : state.vec4_float_type, color,
|
||||
image, coord, ImageOperandsSampleMask, sample});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type, color, image,
|
||||
coord, ImageOperandsLodMask, EmitImageMipLodU32(state, inst, inst.src[0], view)});
|
||||
}
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
|
||||
component, color, component_index});
|
||||
const auto bits = integer ? component : state.builder.AllocateId();
|
||||
if (!integer) {
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageStore(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto uint_image = inst.memory.kind == IR::ResourceKind::StorageImageUint;
|
||||
const auto view = StorageImageViewKind(state, inst.memory, uint_image, inst.pc);
|
||||
const auto binding =
|
||||
ResourceForDescriptor(state, StorageBindingKind(uint_image, view), inst.memory.resource);
|
||||
const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource,
|
||||
binding.array_index, uint_image, view);
|
||||
|
||||
state.builder.AddFunction(
|
||||
{OpImageWrite, image, EmitImageCoordU32(state, inst, view),
|
||||
uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)});
|
||||
}
|
||||
|
||||
void EmitImageSampleResult(EmitterState& state, const IR::Instruction& inst, uint32_t sample,
|
||||
bool dref, bool integer) {
|
||||
if (dref) {
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, sample});
|
||||
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
|
||||
component, sample, component_index});
|
||||
const auto bits = integer ? component : state.builder.AllocateId();
|
||||
if (!integer) {
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst) {
|
||||
// RDNA2 plain IMAGE_SAMPLE is derivative-based in pixel shaders. Do not translate it
|
||||
// to Lod(0): only _L/_LZ/_D variants supply explicit LOD/gradient information.
|
||||
// SPIR-V implicit-lod sampling is fragment-only, so non-pixel stages still need
|
||||
// an explicit operand even for the unsuffixed sample opcode.
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative) ||
|
||||
HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod) ||
|
||||
HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero)) {
|
||||
return true;
|
||||
}
|
||||
return state.stage != ShaderType::Pixel;
|
||||
}
|
||||
|
||||
void AddImageSampleOperands(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, bool explicit_lod,
|
||||
std::vector<uint32_t>& words) {
|
||||
uint32_t mask = 0;
|
||||
std::vector<uint32_t> operands;
|
||||
|
||||
// Keep the SPIR-V operand class aligned with the RDNA2 opcode suffix. Grad/Lod
|
||||
// operands belong to explicit-lod instructions; bias belongs to implicit-lod
|
||||
// sampling because it modifies the hardware-computed LOD.
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
|
||||
mask |= ImageOperandsGradMask;
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
operands.push_back(EmitImageGradientF32(state, inst, layout.grad_x, view));
|
||||
operands.push_back(EmitImageGradientF32(state, inst, layout.grad_y, view));
|
||||
} else if (explicit_lod) {
|
||||
mask |= ImageOperandsLodMask;
|
||||
operands.push_back(EmitImageLodF32(state, inst, layout));
|
||||
} else if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
|
||||
mask |= ImageOperandsBiasMask;
|
||||
operands.push_back(EmitImageBiasF32(state, inst, layout));
|
||||
}
|
||||
if (mask != 0) {
|
||||
words.push_back(mask);
|
||||
words.insert(words.end(), operands.begin(), operands.end());
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto flags = inst.memory.image_sample_flags;
|
||||
const bool dref = (flags & Decoder::ImageSampleFlagCompare) != 0;
|
||||
const bool explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
|
||||
if (explicit_lod) {
|
||||
return dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod;
|
||||
}
|
||||
return dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod;
|
||||
}
|
||||
|
||||
void EmitImageSample(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto layout = MakeImageSampleLayout(inst, view);
|
||||
const auto base_coord = EmitImageCoordF32(state, inst, layout, view);
|
||||
const auto sample = state.builder.AllocateId();
|
||||
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
uint32_t result_type = state.vec4_float_type;
|
||||
if (dref) {
|
||||
result_type = state.float_type;
|
||||
} else if (integer) {
|
||||
result_type = state.vec4_uint_type;
|
||||
}
|
||||
const auto explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
|
||||
const auto opcode = ImageSampleOpcode(state, inst);
|
||||
std::vector<uint32_t> words = {opcode, result_type, sample, sampled_image, base_coord};
|
||||
if (dref) {
|
||||
words.push_back(EmitImageDrefF32(state, inst, layout));
|
||||
}
|
||||
AddImageSampleOperands(state, inst, layout, explicit_lod, words);
|
||||
state.builder.AddFunction(words);
|
||||
EmitImageSampleResult(state, inst, sample, dref, integer);
|
||||
}
|
||||
|
||||
uint32_t ImageGatherComponent(uint32_t dmask) {
|
||||
switch (dmask) {
|
||||
case 0x2u: return 1;
|
||||
case 0x4u: return 2;
|
||||
case 0x8u: return 3;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
void AddImageGatherOperands(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view,
|
||||
std::vector<uint32_t>& words) {
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagGatherHorizontal)) {
|
||||
words.push_back(ImageOperandsConstOffsetsMask);
|
||||
words.push_back(ConstantImageGatherHorizontalOffsets(state, view));
|
||||
return;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
|
||||
words.push_back(ImageOperandsOffsetMask);
|
||||
words.push_back(EmitImagePackedOffsetI32(state, inst, layout, view));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGather4(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto layout = MakeImageSampleLayout(inst, view);
|
||||
const auto coord = EmitImageCoordF32(state, inst, layout, view);
|
||||
const auto texels = state.builder.AllocateId();
|
||||
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
const auto result_type = integer ? state.vec4_uint_type : state.vec4_float_type;
|
||||
std::vector<uint32_t> words =
|
||||
dref ? std::vector<uint32_t> {OpImageDrefGather,
|
||||
state.vec4_float_type,
|
||||
texels,
|
||||
sampled_image,
|
||||
coord,
|
||||
EmitImageDrefF32(state, inst, layout)}
|
||||
: std::vector<uint32_t> {
|
||||
OpImageGather, result_type,
|
||||
texels, sampled_image,
|
||||
coord, ConstantU32(state, ImageGatherComponent(inst.memory.dmask))};
|
||||
AddImageGatherOperands(state, inst, layout, view, words);
|
||||
state.builder.AddFunction(words);
|
||||
|
||||
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
|
||||
const auto component = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
|
||||
component, texels, i});
|
||||
const auto bits = integer ? component : state.builder.AllocateId();
|
||||
if (!integer) {
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -599,28 +599,6 @@ struct F32Class {
|
||||
uint32_t quiet_bits = 0;
|
||||
};
|
||||
|
||||
struct CubeF32Values {
|
||||
uint32_t x = 0;
|
||||
uint32_t y = 0;
|
||||
uint32_t z = 0;
|
||||
uint32_t nx = 0;
|
||||
uint32_t ny = 0;
|
||||
uint32_t nz = 0;
|
||||
uint32_t z_face = 0;
|
||||
uint32_t y_face = 0;
|
||||
uint32_t x_neg = 0;
|
||||
uint32_t y_neg = 0;
|
||||
uint32_t z_neg = 0;
|
||||
};
|
||||
|
||||
struct F16Class {
|
||||
uint32_t bits = 0;
|
||||
uint32_t nan = 0;
|
||||
uint32_t snan = 0;
|
||||
uint32_t zero = 0;
|
||||
uint32_t quiet_bits = 0;
|
||||
};
|
||||
|
||||
uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr);
|
||||
|
||||
uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr);
|
||||
@@ -645,18 +623,8 @@ void SetError(std::string* error, const char* message);
|
||||
|
||||
void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg);
|
||||
|
||||
IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index);
|
||||
|
||||
IR::Register SccRegister();
|
||||
|
||||
IR::Operand SccOperand();
|
||||
|
||||
bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg);
|
||||
|
||||
bool IsMaskRegisterFile(IR::RegisterFile file);
|
||||
|
||||
bool IsSccOperand(const IR::Operand& operand);
|
||||
|
||||
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, uint32_t index);
|
||||
|
||||
void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program);
|
||||
@@ -681,12 +649,8 @@ uint32_t PointerForRegister(const EmitterState& state, IR::Register reg);
|
||||
|
||||
uint32_t ConstantU32(EmitterState& state, uint32_t value);
|
||||
|
||||
void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value);
|
||||
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
|
||||
|
||||
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane);
|
||||
|
||||
[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource,
|
||||
const char* reason);
|
||||
@@ -768,10 +732,6 @@ void AllocateRegisterVariables(EmitterState& state);
|
||||
|
||||
void AllocateDescriptorVariables(EmitterState& state);
|
||||
|
||||
bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width);
|
||||
|
||||
uint32_t EmitSdwaExtractU32(EmitterState& state, const IR::Operand& operand, uint32_t value);
|
||||
|
||||
uint32_t EmitTrueBool(EmitterState& state);
|
||||
|
||||
DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
|
||||
@@ -785,34 +745,10 @@ DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool
|
||||
|
||||
DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control);
|
||||
|
||||
uint32_t EmitDppValueU32(EmitterState& state, const IR::Operand& operand, uint32_t value);
|
||||
|
||||
uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand);
|
||||
|
||||
uint32_t EmitRegisterLoad(EmitterState& state, IR::Register reg);
|
||||
|
||||
uint32_t WaveMaskHighLoad(EmitterState& state, IR::RegisterFile file);
|
||||
|
||||
uint32_t EmitLaneMaskPartU32(EmitterState& state, uint32_t part);
|
||||
|
||||
uint32_t EmitThreadMaskBelowPartU32(EmitterState& state, uint32_t part);
|
||||
|
||||
uint32_t EmitMaskActiveBool(EmitterState& state, IR::RegisterFile file);
|
||||
|
||||
uint32_t EmitMaskZeroBool(EmitterState& state, IR::RegisterFile file, bool zero);
|
||||
|
||||
uint32_t EmitMaskSummaryZeroBool(EmitterState& state, IR::RegisterFile file, bool zero);
|
||||
|
||||
uint32_t EmitExecActiveBool(EmitterState& state);
|
||||
|
||||
uint32_t EmitConditionBool(EmitterState& state, const IR::Operand& operand);
|
||||
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
|
||||
|
||||
uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind);
|
||||
|
||||
uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location);
|
||||
|
||||
const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location);
|
||||
|
||||
uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
|
||||
@@ -822,141 +758,26 @@ uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uin
|
||||
|
||||
uint32_t EmitLocalInvocationIndex(EmitterState& state);
|
||||
|
||||
void EmitLoadInputF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitSccBool(EmitterState& state, bool non_zero);
|
||||
|
||||
uint32_t EmitFloatLoad(EmitterState& state, const IR::Operand& operand);
|
||||
|
||||
uint32_t ApplyResultModifiersF32(EmitterState& state, uint32_t value, const IR::Operand& dst);
|
||||
|
||||
uint32_t EmitMixF32Load(EmitterState& state, const IR::Operand& operand);
|
||||
|
||||
IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset);
|
||||
|
||||
uint32_t EmitLaneMaskOperandActiveBool(EmitterState& state, const IR::Operand& operand);
|
||||
|
||||
uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t ballot, uint32_t lane);
|
||||
|
||||
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane);
|
||||
|
||||
uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane);
|
||||
|
||||
uint32_t EmitSequentialValueLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset);
|
||||
|
||||
uint32_t EmitSequentialFloatLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset);
|
||||
|
||||
uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component);
|
||||
|
||||
IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base,
|
||||
uint32_t component);
|
||||
|
||||
uint32_t EmitImageAddressValueLoad(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& base, uint32_t component);
|
||||
|
||||
uint32_t EmitImageAddressFloatLoad(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& base, uint32_t component);
|
||||
|
||||
uint32_t EmitZeroF32(EmitterState& state);
|
||||
|
||||
bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag);
|
||||
|
||||
ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view);
|
||||
|
||||
uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view);
|
||||
|
||||
uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout);
|
||||
|
||||
uint32_t EmitImageDrefF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout);
|
||||
|
||||
uint32_t EmitImageBiasF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout);
|
||||
|
||||
uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t first_component, ImageViewKind view);
|
||||
|
||||
uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view);
|
||||
|
||||
uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst,
|
||||
ImageViewKind view = ImageViewKind::Dim2D);
|
||||
|
||||
uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst,
|
||||
ImageViewKind view);
|
||||
|
||||
uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& address, ImageViewKind view);
|
||||
|
||||
uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst,
|
||||
ImageViewKind view);
|
||||
|
||||
uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component);
|
||||
|
||||
uint32_t EmitImageStoreComponentF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component);
|
||||
|
||||
uint32_t EmitImageStoreTexelF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitImageStoreTexelU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitDppWriteActiveBool(EmitterState& state, const IR::Operand& dst);
|
||||
|
||||
uint32_t EmitSdwaDestinationMerge(EmitterState& state, uint32_t pointer, const IR::Operand& dst,
|
||||
uint32_t value);
|
||||
|
||||
void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value);
|
||||
|
||||
uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
|
||||
|
||||
uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs);
|
||||
|
||||
uint32_t EmitNotEqualZeroBool(EmitterState& state, uint32_t value);
|
||||
|
||||
uint32_t EmitSelectU32Value(EmitterState& state, uint32_t condition, uint32_t true_value,
|
||||
uint32_t false_value);
|
||||
|
||||
uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
|
||||
|
||||
bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index);
|
||||
|
||||
uint32_t EmitByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src,
|
||||
uint32_t src_count);
|
||||
|
||||
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t use_pc);
|
||||
|
||||
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t use_pc);
|
||||
|
||||
bool ShouldApplyBufferAddTid(const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitBufferIndexWithAddTid(EmitterState& state, const IR::Instruction& inst, uint32_t index,
|
||||
uint32_t packed);
|
||||
|
||||
uint32_t EmitOptionalLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs);
|
||||
|
||||
bool IsStorageBufferMemoryKind(IR::ResourceKind kind);
|
||||
|
||||
void EmitStorageBufferOffsets(EmitterState& state);
|
||||
|
||||
uint32_t EmitBufferAddressFromParts(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t index, uint32_t offset, uint32_t soffset);
|
||||
|
||||
uint32_t EmitBufferByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src,
|
||||
uint32_t src_count);
|
||||
|
||||
uint32_t EmitMemoryByteAddress(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count);
|
||||
|
||||
uint32_t EmitDwordIndex(EmitterState& state, const IR::Instruction& inst, uint32_t first_src,
|
||||
uint32_t src_count);
|
||||
|
||||
uint32_t EmitMemoryDwordIndex(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::MemoryInfo& mem, uint32_t first_src, uint32_t src_count);
|
||||
|
||||
DescriptorResourceBinding StorageBufferBindingForMemory(EmitterState& state,
|
||||
const IR::MemoryInfo& mem, uint32_t use_pc);
|
||||
|
||||
@@ -982,37 +803,6 @@ uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index);
|
||||
uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
|
||||
uint32_t use_pc);
|
||||
|
||||
uint32_t EmitMemoryLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst,
|
||||
IR::ResourceKind kind, uint32_t first_src, uint32_t src_count);
|
||||
|
||||
void EmitMemoryLoadU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind,
|
||||
uint32_t first_src, uint32_t src_count);
|
||||
|
||||
uint32_t EmitMemoryLoadSubDwordValueU32(EmitterState& state, const IR::Instruction& inst,
|
||||
IR::ResourceKind kind, uint32_t first_src,
|
||||
uint32_t src_count, uint32_t data_bits, bool sign_extend);
|
||||
|
||||
void EmitMemoryLoadSubDwordU32(EmitterState& state, const IR::Instruction& inst,
|
||||
IR::ResourceKind kind, uint32_t first_src, uint32_t src_count,
|
||||
uint32_t data_bits, bool sign_extend);
|
||||
|
||||
void EmitMemoryStoreU32(EmitterState& state, const IR::Instruction& inst, IR::ResourceKind kind,
|
||||
uint32_t first_src, uint32_t src_count);
|
||||
|
||||
void EmitMemoryStoreSubDwordU32(EmitterState& state, const IR::Instruction& inst,
|
||||
IR::ResourceKind kind, uint32_t first_src, uint32_t src_count,
|
||||
uint32_t data_bits);
|
||||
|
||||
uint32_t AddressSourceCount(const IR::Instruction& inst, uint32_t first_src);
|
||||
|
||||
bool IsFormattedBufferComponent(const IR::Instruction& inst);
|
||||
|
||||
Prospero::BufferFormat FormattedBufferFormat(const EmitterState& state,
|
||||
const IR::Instruction& inst);
|
||||
|
||||
IR::Instruction WithFormatComponentByteOffset(const IR::Instruction& inst,
|
||||
Prospero::BufferFormat format);
|
||||
|
||||
uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value);
|
||||
|
||||
uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value);
|
||||
@@ -1027,307 +817,37 @@ uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t
|
||||
|
||||
bool IsSignedFormatComponent(Format::ComponentType type);
|
||||
|
||||
uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint32_t offset,
|
||||
uint32_t bits, bool sign_extend);
|
||||
|
||||
uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw);
|
||||
|
||||
uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits);
|
||||
|
||||
uint32_t EmitFormatRawComponent(EmitterState& state, const IR::Instruction& inst,
|
||||
const Format::BufferFormatInfo& info);
|
||||
|
||||
uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info,
|
||||
uint32_t component, uint32_t raw);
|
||||
|
||||
uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst,
|
||||
const Format::BufferFormatInfo& info);
|
||||
|
||||
uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format,
|
||||
uint32_t opcode_components);
|
||||
|
||||
bool EmitBufferIntegerFormatStore(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitAtomicPointer(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitDeviceAtomicMemoryBarrier(EmitterState& state);
|
||||
|
||||
void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitAtomicFMinF32(EmitterState& state, const IR::Instruction& inst);
|
||||
void EmitAtomicFMaxF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitLoadSrtDword(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBufferLoadUbyte(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBufferLoadSbyte(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBufferLoadUshort(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions,
|
||||
uint32_t count);
|
||||
|
||||
void EmitBufferStoreDword(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitFlatLoadUbyte(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitFlatLoadSbyte(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitFlatLoadUshort(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitFlatLoadSshort(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitFlatLoadDword(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitDsAddtidDwordIndex(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t m0_src_index);
|
||||
|
||||
void EmitDsWriteAddtidB32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitDsReadAddtidB32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitDsAppendConsume(EmitterState& state, const IR::Instruction& inst, uint32_t atomic_opcode);
|
||||
|
||||
void EmitDsFloatMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool max_value);
|
||||
|
||||
uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
|
||||
|
||||
void EmitDsSwizzleB32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitImageGetResinfoComponent(EmitterState& state, uint32_t image, uint32_t size,
|
||||
uint32_t component_index);
|
||||
|
||||
void EmitImageGetResinfo(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitImageGetLod(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitImageLoad(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitImageStore(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitImageSampleResult(EmitterState& state, const IR::Instruction& inst, uint32_t sample,
|
||||
bool dref);
|
||||
|
||||
bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void AddImageSampleOperands(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, bool explicit_lod,
|
||||
std::vector<uint32_t>& words);
|
||||
|
||||
uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitImageSample(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t ImageGatherComponent(uint32_t dmask);
|
||||
|
||||
void EmitImageGather4(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMoveU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMoveF32Bits(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base);
|
||||
|
||||
void EmitMoveRelSourceU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMoveRelDestU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMoveU64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src);
|
||||
|
||||
void EmitWqmB64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitSaveexecB32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitSaveexecB64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitReadFirstLaneU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitLaneIndex(EmitterState& state, const IR::Operand& operand);
|
||||
|
||||
void EmitReadLaneU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitWriteLaneU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitPermlaneB32(EmitterState& state, const IR::Instruction& inst, bool x16);
|
||||
|
||||
void EmitControlNop(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitWaitcnt(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBarrier(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitAbsI32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitUnaryU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitUnaryU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitFindLsbU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitFindMsbFromHighU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitFindMsbFromHighU64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBitCountU64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBitCountU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBitCountAddU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBinaryU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitBinaryNotRhsU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitBinaryThenNotU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitMaskedBitCountU32(EmitterState& state, const IR::Instruction& inst, uint32_t exec_index);
|
||||
|
||||
uint32_t EmitMask5U32(EmitterState& state, uint32_t value);
|
||||
|
||||
void EmitChainedBinaryU32(EmitterState& state, const IR::Instruction& inst, uint32_t first_op,
|
||||
uint32_t second_op, bool mask_first_rhs = false,
|
||||
bool mask_second_rhs = false);
|
||||
|
||||
void EmitBinaryU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitBinaryNotRhsU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitBinaryThenNotU64(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitSelectU64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
|
||||
uint32_t false_value);
|
||||
|
||||
void EmitStoreSccBool(EmitterState& state, uint32_t cond);
|
||||
|
||||
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
|
||||
|
||||
void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
|
||||
uint32_t& out_low, uint32_t& out_high);
|
||||
|
||||
void EmitShiftLeftLogicalU64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
|
||||
uint32_t shift, uint32_t& out_low, uint32_t& out_high);
|
||||
|
||||
void EmitShiftRightLogicalU64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitAdd3U32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
|
||||
|
||||
void EmitShiftU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
uint32_t EmitU16LaneBits(EmitterState& state, const IR::Operand& operand, bool high_lane,
|
||||
bool sign_extend = false);
|
||||
|
||||
void EmitShiftU16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode,
|
||||
bool arithmetic);
|
||||
|
||||
void EmitBinaryU16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitMinMaxU16(EmitterState& state, const IR::Instruction& inst, bool signed_value,
|
||||
bool max_value);
|
||||
|
||||
uint32_t EmitXorConstant(EmitterState& state, uint32_t value, uint32_t mask);
|
||||
|
||||
AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs,
|
||||
uint32_t carry_in);
|
||||
|
||||
void EmitLaneMaskPairFromBool(EmitterState& state, const IR::Operand& dst, uint32_t value_bool);
|
||||
|
||||
void EmitPerInvocationMask(EmitterState& state, const IR::Operand& dst, uint32_t value_bool);
|
||||
|
||||
void EmitIAddCarryU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitISubBorrowU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitScalarAddCarryU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitScalarSubBorrowU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitScalarSubBorrowCarryU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBitClearU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBitSetU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitUMadU64U32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitScalarSignedOverflowI32(EmitterState& state, const IR::Instruction& inst, bool subtract);
|
||||
|
||||
void EmitScalarShiftLeftAddCarryU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMulU24U32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMulI24U32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMulHighU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMulHighI32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMadU32U24(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMadI32I24(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitShiftLeftConstant(EmitterState& state, uint32_t value, uint32_t shift);
|
||||
|
||||
uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift);
|
||||
|
||||
uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
|
||||
|
||||
uint32_t EmitBitReplicateHalfU32(EmitterState& state, uint32_t value);
|
||||
|
||||
void EmitBitReplicateB64B32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitBitFieldExtractConstant(EmitterState& state, uint32_t value, uint32_t offset,
|
||||
uint32_t count);
|
||||
|
||||
uint32_t EmitRightAlignedMaskU32(EmitterState& state, uint32_t count);
|
||||
|
||||
void EmitBitFieldMaskU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitRightAlignedMaskU64(EmitterState& state, uint32_t count, uint32_t& out_low,
|
||||
uint32_t& out_high);
|
||||
|
||||
void EmitBitFieldMaskU64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBitFieldExtractU64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBitFieldExtractU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBitFieldExtract3U32(EmitterState& state, const IR::Instruction& inst, bool signed_value);
|
||||
|
||||
void EmitBitFieldInsertSelectU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitAlignBitU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitSelectU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitSelectMaskU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitSelectF32Bits(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitSelectMaskF32Bits(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitPackU16(EmitterState& state, const IR::Instruction& inst, bool high_src0, bool high_src1);
|
||||
|
||||
void EmitPackU8F32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBinaryF32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitLdexpF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
|
||||
uint32_t constant);
|
||||
|
||||
@@ -1335,25 +855,10 @@ uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_
|
||||
|
||||
uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32);
|
||||
|
||||
void EmitPackF32ToF16Rtz(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitPackNormalizedF32(EmitterState& state, const IR::Instruction& inst, uint32_t ext_inst);
|
||||
|
||||
uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
|
||||
|
||||
void EmitMinMaxU32(EmitterState& state, const IR::Instruction& inst, bool max_value);
|
||||
|
||||
void EmitSadU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
|
||||
|
||||
void EmitMinMaxI32(EmitterState& state, const IR::Instruction& inst, bool max_value);
|
||||
|
||||
void EmitMinMax3U32(EmitterState& state, const IR::Instruction& inst, bool signed_value,
|
||||
bool max_value);
|
||||
|
||||
void EmitMed3U32(EmitterState& state, const IR::Instruction& inst, bool signed_value);
|
||||
|
||||
uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value);
|
||||
|
||||
uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value);
|
||||
@@ -1375,181 +880,18 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask);
|
||||
|
||||
uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
|
||||
|
||||
void EmitMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool max_value);
|
||||
|
||||
void EmitCompareResult(EmitterState& state, const IR::Operand& dst, uint32_t cond);
|
||||
|
||||
void EmitCompareU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitCompareU16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitCompareI16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitBitCompareB32(EmitterState& state, const IR::Instruction& inst, bool bit_set);
|
||||
|
||||
void EmitCompareU64(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitCompareConstant(EmitterState& state, const IR::Instruction& inst, bool value);
|
||||
|
||||
void EmitCompareMaskU32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitCompareF32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitCompareOrderedF32(EmitterState& state, const IR::Instruction& inst, bool ordered);
|
||||
|
||||
void EmitCompareClassF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitCompareMaskF32(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitConvertU32ToF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertI32ToF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value);
|
||||
|
||||
void EmitConvertF32ToU32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertF32ToI32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertF32ToF16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertF16ToF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertU16ToF16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertF16ToU16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertI16ToF16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertF16ToI16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertRoundPlusInfF32ToI32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertFloorF32ToI32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertI4ToOffsetF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitConvertByteU32ToF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value);
|
||||
|
||||
void EmitRcpF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero);
|
||||
|
||||
void EmitFloatExtInst(EmitterState& state, const IR::Instruction& inst, uint32_t ext_inst,
|
||||
bool scale_by_two_pi = false, bool flush_denorm = false);
|
||||
|
||||
void EmitMadF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitFNegateValue(EmitterState& state, uint32_t value);
|
||||
|
||||
uint32_t EmitFAbsValue(EmitterState& state, uint32_t value);
|
||||
|
||||
uint32_t EmitFCompareValue(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs);
|
||||
|
||||
uint32_t EmitLogicalAndValue(EmitterState& state, uint32_t lhs, uint32_t rhs);
|
||||
|
||||
uint32_t EmitFSelectValue(EmitterState& state, uint32_t cond, uint32_t true_value,
|
||||
uint32_t false_value);
|
||||
|
||||
uint32_t EmitFMulConstant(EmitterState& state, uint32_t value, uint32_t bits);
|
||||
|
||||
CubeF32Values EmitCubeF32Values(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitCubeIdF32(EmitterState& state, const CubeF32Values& cube);
|
||||
|
||||
uint32_t EmitCubeScF32(EmitterState& state, const CubeF32Values& cube);
|
||||
|
||||
uint32_t EmitCubeTcF32(EmitterState& state, const CubeF32Values& cube);
|
||||
|
||||
uint32_t EmitCubeMaF32(EmitterState& state, const CubeF32Values& cube);
|
||||
|
||||
void EmitCubeF32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitDot2AccF32F16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitPackedF16Lane(EmitterState& state, const IR::Operand& operand, bool high_lane);
|
||||
|
||||
void EmitCompareF16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
uint32_t EmitPackedF16LaneBits(EmitterState& state, const IR::Operand& operand, bool high_lane);
|
||||
|
||||
uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
|
||||
|
||||
uint32_t EmitPackLowF32ToF16Bits(EmitterState& state, uint32_t value);
|
||||
|
||||
void EmitPackB32F16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
F16Class EmitClassifyF16Bits(EmitterState& state, uint32_t value);
|
||||
|
||||
uint32_t EmitNegativeNumericF16Bool(EmitterState& state, const F16Class& cls);
|
||||
|
||||
uint32_t EmitRcpF16Bits(EmitterState& state, uint32_t bits);
|
||||
|
||||
uint32_t EmitSqrtF16Bits(EmitterState& state, uint32_t bits);
|
||||
|
||||
uint32_t EmitRsqF16Bits(EmitterState& state, uint32_t bits);
|
||||
|
||||
uint32_t EmitLog2F16Bits(EmitterState& state, uint32_t bits);
|
||||
|
||||
uint32_t EmitExp2F16Bits(EmitterState& state, uint32_t bits);
|
||||
|
||||
void EmitSpecialF16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitBinaryF16(EmitterState& state, const IR::Instruction& inst, uint32_t opcode);
|
||||
|
||||
void EmitFmaF16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitMinMaxF16Bits(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
|
||||
|
||||
void EmitMinMaxF16(EmitterState& state, const IR::Instruction& inst, bool max_value);
|
||||
|
||||
uint32_t EmitMinMax3F16Bits(EmitterState& state, uint32_t src0, uint32_t src1, uint32_t src2,
|
||||
bool max_value);
|
||||
|
||||
void EmitMinMax3F16(EmitterState& state, const IR::Instruction& inst, bool max_value);
|
||||
|
||||
uint32_t EmitF16BitsEqualBool(EmitterState& state, uint32_t lhs, uint32_t rhs);
|
||||
|
||||
uint32_t EmitAnyNanF16Bool(EmitterState& state, uint32_t src0, uint32_t src1, uint32_t src2);
|
||||
|
||||
void EmitMed3F16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitPackedMinMaxF16(EmitterState& state, const IR::Instruction& inst, bool max_value);
|
||||
|
||||
uint32_t EmitPackedF16BinaryLane(EmitterState& state, const IR::Instruction& inst, bool high_lane,
|
||||
uint32_t opcode);
|
||||
|
||||
uint32_t EmitPackedF16FmaLane(EmitterState& state, const IR::Instruction& inst, bool high_lane);
|
||||
|
||||
void EmitPackedF16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitPackedInteger16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMadMixF16(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitMinMax3F32(EmitterState& state, const IR::Instruction& inst, bool max_value);
|
||||
|
||||
void EmitMed3F32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
uint32_t EmitExportComponentF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component);
|
||||
|
||||
uint32_t EmitExportVec4F32(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
bool ExportWritesData(const IR::Instruction& inst);
|
||||
|
||||
void EmitExport(EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst);
|
||||
|
||||
void EmitKillIfBoolFalse(EmitterState& state, uint32_t active);
|
||||
|
||||
void EmitUpdatePixelValidMask(EmitterState& state);
|
||||
|
||||
void EmitKillIfPixelValidMaskInactive(EmitterState& state);
|
||||
|
||||
uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg);
|
||||
|
||||
bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
@@ -1613,19 +955,6 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&
|
||||
std::forward<Fn>(fn));
|
||||
}
|
||||
|
||||
template <typename Fn>
|
||||
void EmitGuardedByExec(EmitterState& state, Fn&& fn) {
|
||||
const auto then_label = state.builder.AllocateId();
|
||||
const auto merge_label = state.builder.AllocateId();
|
||||
const auto condition = EmitExecActiveBool(state);
|
||||
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
|
||||
state.builder.AddFunction({OpBranchConditional, condition, then_label, merge_label});
|
||||
state.builder.AddFunction({OpLabel, then_label});
|
||||
fn();
|
||||
state.builder.AddFunction({OpBranch, merge_label});
|
||||
state.builder.AddFunction({OpLabel, merge_label});
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
|
||||
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_INTERNAL_H_ */
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,6 +9,32 @@
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
|
||||
const auto kill_label = state.builder.AllocateId();
|
||||
const auto merge_label = state.builder.AllocateId();
|
||||
const auto inactive = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
|
||||
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
|
||||
state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
|
||||
state.builder.AddFunction({OpLabel, kill_label});
|
||||
state.builder.AddFunction({OpKill});
|
||||
state.builder.AddFunction({OpLabel, merge_label});
|
||||
}
|
||||
|
||||
void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
|
||||
if (state.pixel_valid_mask_variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto mask_value = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)});
|
||||
EmitKillIfBoolFalse(state, active);
|
||||
}
|
||||
|
||||
uint32_t BoolConstant(EmitterState& state, bool value) {
|
||||
const auto id = state.builder.AllocateId();
|
||||
state.builder.AddType({value ? 41u : 42u, state.bool_type, id});
|
||||
|
||||
@@ -0,0 +1,423 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
|
||||
uint32_t false_value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, true_value, false_value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
|
||||
|
||||
void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
|
||||
uint32_t& out_low, uint32_t& out_high) {
|
||||
const auto amount = EmitAndConstant(state, shift, 63u);
|
||||
const auto dword_shift = EmitAndConstant(state, amount, 31u);
|
||||
const auto less32 = state.builder.AllocateId();
|
||||
const auto non_zero = state.builder.AllocateId();
|
||||
const auto carry_count_base = state.builder.AllocateId();
|
||||
const auto carry_count = state.builder.AllocateId();
|
||||
const auto low_part = state.builder.AllocateId();
|
||||
const auto high_part = state.builder.AllocateId();
|
||||
const auto carry_part = state.builder.AllocateId();
|
||||
const auto carry = state.builder.AllocateId();
|
||||
const auto high_less = state.builder.AllocateId();
|
||||
const auto high_ge = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, low_part, low, dword_shift});
|
||||
state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_part, high, dword_shift});
|
||||
state.builder.AddFunction({OpShiftRightLogical, state.uint_type, carry_part, low, carry_count});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, high_less, high_part, carry});
|
||||
state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_ge, low, dword_shift});
|
||||
out_low = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
|
||||
out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
|
||||
}
|
||||
|
||||
void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
|
||||
uint32_t shift, uint32_t& out_low, uint32_t& out_high) {
|
||||
const auto amount = EmitAndConstant(state, shift, 63u);
|
||||
const auto dword_shift = EmitAndConstant(state, amount, 31u);
|
||||
const auto less32 = state.builder.AllocateId();
|
||||
const auto non_zero = state.builder.AllocateId();
|
||||
const auto carry_count_base = state.builder.AllocateId();
|
||||
const auto carry_count = state.builder.AllocateId();
|
||||
const auto low_part = state.builder.AllocateId();
|
||||
const auto high_part = state.builder.AllocateId();
|
||||
const auto carry_part = state.builder.AllocateId();
|
||||
const auto carry = state.builder.AllocateId();
|
||||
const auto low_less = state.builder.AllocateId();
|
||||
const auto low_ge = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_part, low, dword_shift});
|
||||
state.builder.AddFunction({OpShiftRightLogical, state.uint_type, high_part, high, dword_shift});
|
||||
state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, carry_part, high, carry_count});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, low_less, low_part, carry});
|
||||
state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_ge, high, dword_shift});
|
||||
out_low = EmitSelectValueU32(state, less32, low_less, low_ge);
|
||||
out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
|
||||
}
|
||||
|
||||
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, ret, value, ConstantU32(state, mask)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs,
|
||||
uint32_t carry_in) {
|
||||
const auto pair0 = state.builder.AllocateId();
|
||||
const auto sum0 = state.builder.AllocateId();
|
||||
const auto carry0 = state.builder.AllocateId();
|
||||
const auto pair1 = state.builder.AllocateId();
|
||||
const auto sum1 = state.builder.AllocateId();
|
||||
const auto carry1 = state.builder.AllocateId();
|
||||
const auto carry = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair0, lhs, rhs});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum0, pair0, 0});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry0, pair0, 1});
|
||||
state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair1, sum0, carry_in});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum1, pair1, 0});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry1, pair1, 1});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, carry, carry0, carry1});
|
||||
return {sum1, carry};
|
||||
}
|
||||
|
||||
uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, ret, value, ConstantU32(state, shift)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, ret, lhs, rhs});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
|
||||
uint32_t constant) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpISub, state.uint_type, ret, ConstantU32(state, constant), value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, f32});
|
||||
|
||||
const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u);
|
||||
const auto exp = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu);
|
||||
const auto mant = EmitAndConstant(state, bits, 0x007fffffu);
|
||||
|
||||
const auto half_exp = state.builder.AllocateId();
|
||||
const auto normal_exp = state.builder.AllocateId();
|
||||
const auto normal_mant = EmitShiftRightConstant(state, mant, 13);
|
||||
const auto normal_payload = state.builder.AllocateId();
|
||||
const auto normal = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpISub, state.uint_type, half_exp, exp, ConstantU32(state, 112)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, normal_exp, half_exp, ConstantU32(state, 10)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseOr, state.uint_type, normal_payload, normal_exp, normal_mant});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, normal, sign, normal_payload});
|
||||
|
||||
const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u));
|
||||
const auto raw_sub_shift = EmitSubConstantMinusU32(state, 126, exp);
|
||||
const auto exp_lt_103 = EmitCompareU32Constant(state, OpULessThan, exp, 103);
|
||||
const auto exp_gt_112 = EmitCompareU32Constant(state, OpUGreaterThan, exp, 112);
|
||||
const auto sub_shift_low =
|
||||
EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift);
|
||||
const auto sub_shift =
|
||||
EmitSelectValueU32(state, exp_gt_112, ConstantU32(state, 14), sub_shift_low);
|
||||
const auto sub_mant = state.builder.AllocateId();
|
||||
const auto subnormal = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, sub_mant, mant_with_hidden, sub_shift});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, subnormal, sign, sub_mant});
|
||||
|
||||
const auto nan_payload =
|
||||
EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u));
|
||||
const auto nan =
|
||||
EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload));
|
||||
const auto inf = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u));
|
||||
const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu));
|
||||
const auto mant_zero = EmitCompareU32Constant(state, OpIEqual, mant, 0);
|
||||
const auto special = EmitSelectValueU32(state, mant_zero, inf, nan);
|
||||
|
||||
const auto exp_le_112 = EmitCompareU32Constant(state, OpULessThanEqual, exp, 112);
|
||||
const auto exp_ge_143 = EmitCompareU32Constant(state, OpUGreaterThanEqual, exp, 143);
|
||||
const auto exp_eq_255 = EmitCompareU32Constant(state, OpIEqual, exp, 255);
|
||||
const auto finite0 = EmitSelectValueU32(state, exp_le_112, subnormal, normal);
|
||||
const auto finite1 = EmitSelectValueU32(state, exp_lt_103, sign, finite0);
|
||||
const auto finite2 = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1);
|
||||
return EmitAndConstant(state, EmitSelectValueU32(state, exp_eq_255, special, finite2), 0xffffu);
|
||||
}
|
||||
|
||||
uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
|
||||
const auto cond = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{max_value ? OpUGreaterThan : OpULessThan, state.bool_type, cond, lhs, rhs});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
|
||||
const auto cond = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{max_value ? OpSGreaterThan : OpSLessThan, state.bool_type, cond, lhs, rhs});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, ret, value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, ret, value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret, lhs, rhs});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, lhs, rhs});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalOr, state.bool_type, ret, lhs, rhs});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, ret, value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
F32Class EmitClassifyF32(EmitterState& state, uint32_t value) {
|
||||
F32Class cls;
|
||||
cls.bits = EmitBitcastF32ToU32(state, value);
|
||||
const auto abs_bits = EmitAndConstant(state, cls.bits, 0x7fffffffu);
|
||||
const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
|
||||
const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
|
||||
const auto exponent_max = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
|
||||
const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
|
||||
const auto quiet_bit_clear =
|
||||
EmitCompareU32Constant(state, OpIEqual, EmitAndConstant(state, cls.bits, 0x00400000u), 0);
|
||||
cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
|
||||
cls.snan = EmitLogicalAndBool(state, cls.nan, quiet_bit_clear);
|
||||
cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
|
||||
cls.quiet_bits = EmitOrU32(state, cls.bits, ConstantU32(state, 0x00400000u));
|
||||
return cls;
|
||||
}
|
||||
|
||||
uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
|
||||
uint32_t class_match) {
|
||||
const auto selected =
|
||||
EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0);
|
||||
return EmitLogicalAndBool(state, selected, class_match);
|
||||
}
|
||||
|
||||
uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
|
||||
const auto bits = EmitBitcastF32ToU32(state, value);
|
||||
const auto sign_bits = EmitAndConstant(state, bits, 0x80000000u);
|
||||
const auto abs_bits = EmitAndConstant(state, bits, 0x7fffffffu);
|
||||
const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
|
||||
const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
|
||||
const auto quiet_bits = EmitAndConstant(state, mantissa_bits, 0x00400000u);
|
||||
|
||||
const auto sign = EmitCompareU32Constant(state, OpINotEqual, sign_bits, 0);
|
||||
const auto positive = EmitLogicalNotBool(state, sign);
|
||||
const auto exponent_zero = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0);
|
||||
const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero);
|
||||
const auto exponent_inf = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
|
||||
const auto finite_exponent = EmitLogicalNotBool(state, exponent_inf);
|
||||
const auto mantissa_zero = EmitCompareU32Constant(state, OpIEqual, mantissa_bits, 0);
|
||||
const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero);
|
||||
const auto quiet = EmitCompareU32Constant(state, OpINotEqual, quiet_bits, 0);
|
||||
|
||||
const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero);
|
||||
const auto snan = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet));
|
||||
const auto qnan = EmitLogicalAndBool(state, nan_common, quiet);
|
||||
const auto inf = EmitLogicalAndBool(state, exponent_inf, mantissa_zero);
|
||||
const auto normal = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent);
|
||||
const auto denorm = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero);
|
||||
const auto zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
|
||||
|
||||
uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan);
|
||||
match = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan));
|
||||
match = EmitLogicalOrBool(
|
||||
state, match, EmitClassMaskBitMatch(state, mask, 2, EmitLogicalAndBool(state, inf, sign)));
|
||||
match = EmitLogicalOrBool(
|
||||
state, match,
|
||||
EmitClassMaskBitMatch(state, mask, 3, EmitLogicalAndBool(state, normal, sign)));
|
||||
match = EmitLogicalOrBool(
|
||||
state, match,
|
||||
EmitClassMaskBitMatch(state, mask, 4, EmitLogicalAndBool(state, denorm, sign)));
|
||||
match = EmitLogicalOrBool(
|
||||
state, match, EmitClassMaskBitMatch(state, mask, 5, EmitLogicalAndBool(state, zero, sign)));
|
||||
match = EmitLogicalOrBool(
|
||||
state, match,
|
||||
EmitClassMaskBitMatch(state, mask, 6, EmitLogicalAndBool(state, zero, positive)));
|
||||
match = EmitLogicalOrBool(
|
||||
state, match,
|
||||
EmitClassMaskBitMatch(state, mask, 7, EmitLogicalAndBool(state, denorm, positive)));
|
||||
match = EmitLogicalOrBool(
|
||||
state, match,
|
||||
EmitClassMaskBitMatch(state, mask, 8, EmitLogicalAndBool(state, normal, positive)));
|
||||
return EmitLogicalOrBool(
|
||||
state, match,
|
||||
EmitClassMaskBitMatch(state, mask, 9, EmitLogicalAndBool(state, inf, positive)));
|
||||
}
|
||||
|
||||
uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
|
||||
const auto lhs_class = EmitClassifyF32(state, lhs);
|
||||
const auto rhs_class = EmitClassifyF32(state, rhs);
|
||||
|
||||
const auto numeric_cond = state.builder.AllocateId();
|
||||
state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, state.bool_type,
|
||||
numeric_cond, lhs, rhs});
|
||||
const auto ordered_bits =
|
||||
EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits);
|
||||
|
||||
const auto both_zero = EmitLogicalAndBool(state, lhs_class.zero, rhs_class.zero);
|
||||
const auto zero_bits = max_value ? EmitAndU32(state, lhs_class.bits, rhs_class.bits)
|
||||
: EmitOrU32(state, lhs_class.bits, rhs_class.bits);
|
||||
const auto numeric_bits = EmitSelectValueU32(state, both_zero, zero_bits, ordered_bits);
|
||||
|
||||
const auto rhs_nan_bits =
|
||||
EmitSelectValueU32(state, rhs_class.nan, lhs_class.bits, numeric_bits);
|
||||
const auto non_snan_bits =
|
||||
EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits);
|
||||
const auto rhs_snan_bits =
|
||||
EmitSelectValueU32(state, rhs_class.snan, rhs_class.quiet_bits, non_snan_bits);
|
||||
const auto result_bits =
|
||||
EmitSelectValueU32(state, lhs_class.snan, lhs_class.quiet_bits, rhs_snan_bits);
|
||||
return EmitBitcastU32ToF32(state, result_bits);
|
||||
}
|
||||
|
||||
uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.float_type, ret, state.glsl_std450, GlslTrunc, value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
|
||||
const auto bits = state.builder.AllocateId();
|
||||
const auto abs_bits = state.builder.AllocateId();
|
||||
const auto sign_bits = state.builder.AllocateId();
|
||||
const auto non_zero = state.builder.AllocateId();
|
||||
const auto subnormal = state.builder.AllocateId();
|
||||
const auto flush = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, sign_bits, bits, ConstantU32(state, 0x80000000u)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, non_zero, abs_bits, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, flush, non_zero, subnormal});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, flush, sign_bits, bits});
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, ret, selected});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero) {
|
||||
const auto fract = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
const auto abs_bits = state.builder.AllocateId();
|
||||
const auto large = state.builder.AllocateId();
|
||||
const auto finite = state.builder.AllocateId();
|
||||
const auto large_finite = state.builder.AllocateId();
|
||||
const auto reduced = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.float_type, fract, state.glsl_std450, GlslFract, src});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, src});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, large, abs_bits, ConstantU32(state, 0x4b000000u)});
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, finite, abs_bits, ConstantU32(state, 0x7f800000u)});
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, large_finite, large, finite});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.float_type, reduced, large_finite, ConstantF32(state, 0), fract});
|
||||
if (!preserve_signed_zero) {
|
||||
return reduced;
|
||||
}
|
||||
const auto zero = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpIEqual, state.bool_type, zero, abs_bits, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpSelect, state.float_type, ret, zero, src, reduced});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpFNegate, state.float_type, ret, value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.float_type, ret, state.glsl_std450, GlslFAbs, value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
|
||||
const auto unpacked = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, bits});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.float_type, ret, unpacked, 0});
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -5,6 +5,26 @@
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) {
|
||||
uint32_t ret = ConstantU32(state, 0);
|
||||
for (uint32_t i = 0; i < 8u; i++) {
|
||||
const auto mask = 0x0fu << (i * 4u);
|
||||
const auto masked = state.builder.AllocateId();
|
||||
const auto non_zero = state.builder.AllocateId();
|
||||
const auto expanded = state.builder.AllocateId();
|
||||
const auto combined = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero,
|
||||
ConstantU32(state, mask), ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded});
|
||||
ret = combined;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) {
|
||||
const auto result = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, result, value,
|
||||
@@ -407,20 +427,13 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)});
|
||||
return true;
|
||||
case IR::ValueOpcode::DppMoveU32: {
|
||||
const auto flags = inst.Flags<IR::DppMoveFlags>();
|
||||
IR::Operand operand;
|
||||
operand.dpp = true;
|
||||
operand.dpp_ctrl = flags.control;
|
||||
operand.dpp_row_mask = flags.row_mask;
|
||||
operand.dpp_bank_mask = flags.bank_mask;
|
||||
operand.dpp_fetch_inactive = flags.fetch_inactive;
|
||||
operand.dpp_bound_ctrl = flags.bound_control;
|
||||
const auto target = EmitDppTargetLane(state, operand.dpp_ctrl);
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
const auto flags = inst.Flags<IR::DppMoveFlags>();
|
||||
const auto target = EmitDppTargetLane(state, flags.control);
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
|
||||
target.lane});
|
||||
if (operand.dpp_fetch_inactive) {
|
||||
if (flags.fetch_inactive) {
|
||||
ctx.Define(inst, shuffled);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
uint32_t EmitTrueBool(EmitterState& state) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
|
||||
const auto quad_base = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto shift = state.builder.AllocateId();
|
||||
const auto selected0 = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
|
||||
bool left) {
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto lane_shifted = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
const auto valid = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
if (left) {
|
||||
state.builder.AddFunction(
|
||||
{OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)});
|
||||
}
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted});
|
||||
return {target, valid};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid,
|
||||
uint32_t amount) {
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto minus = state.builder.AllocateId();
|
||||
const auto plus = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
|
||||
const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
|
||||
const auto lane_mask = half_row ? 7u : 15u;
|
||||
const auto base = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto mirrored = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
if (control <= 0xffu) {
|
||||
return EmitDppQuadPermTargetLane(state, subid, control);
|
||||
}
|
||||
if (control >= 0x101u && control <= 0x10fu) {
|
||||
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true);
|
||||
}
|
||||
if (control >= 0x111u && control <= 0x11fu) {
|
||||
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false);
|
||||
}
|
||||
if (control >= 0x121u && control <= 0x12fu) {
|
||||
return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu);
|
||||
}
|
||||
if (control == 0x140u) {
|
||||
return EmitDppMirrorTargetLane(state, subid, false);
|
||||
}
|
||||
if (control == 0x141u) {
|
||||
return EmitDppMirrorTargetLane(state, subid, true);
|
||||
}
|
||||
return {subid, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
|
||||
if (state.subgroup_local_invocation_id_variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == kind) {
|
||||
return input.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
|
||||
return &input;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
|
||||
const auto variable = InputVariableForKind(state, kind);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)});
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t EmitLocalInvocationIndex(EmitterState& state) {
|
||||
const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, variable});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind,
|
||||
uint32_t component) {
|
||||
if ((component & 3u) != 3u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
|
||||
}
|
||||
|
||||
uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
|
||||
uint32_t component) {
|
||||
const auto count = VertexParameterComponentCount(state, input);
|
||||
const auto kind = VertexParameterScalarKind(state, input.location);
|
||||
if (component >= count) {
|
||||
return VertexInputDefaultComponentU32(state, kind, component);
|
||||
}
|
||||
|
||||
const auto scalar_type = VertexParameterScalarType(state, kind);
|
||||
uint32_t raw = state.builder.AllocateId();
|
||||
if (count == 1u) {
|
||||
state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
|
||||
} else {
|
||||
const auto pointer_type = VertexParameterScalarPointerType(state, kind);
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
|
||||
ConstantU32(state, component)});
|
||||
state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
|
||||
}
|
||||
|
||||
if (kind == VertexInputScalarKind::Uint) {
|
||||
return raw;
|
||||
}
|
||||
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
|
||||
return bits;
|
||||
}
|
||||
|
||||
uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
|
||||
const auto active_ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot,
|
||||
ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
|
||||
return EmitBallotLaneActiveBool(state, active_ballot, lane);
|
||||
}
|
||||
uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
|
||||
const auto low = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0});
|
||||
uint32_t mask = low;
|
||||
if (state.wave_size == 64u) {
|
||||
const auto high = state.builder.AllocateId();
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low});
|
||||
mask = selected;
|
||||
}
|
||||
|
||||
const auto lane_low = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
|
||||
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
const auto in_range = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)});
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range});
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -6,6 +6,23 @@
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
|
||||
uint32_t index = 0;
|
||||
for (uint32_t i = 0; i < component; i++) {
|
||||
index += (dmask >> i) & 1u;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
uint32_t ImageGatherComponent(uint32_t dmask) {
|
||||
switch (dmask) {
|
||||
case 0x2u: return 1;
|
||||
case 0x4u: return 2;
|
||||
case 0x8u: return 3;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, type, result, value});
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpIAdd, state.uint_type, ret, lhs, rhs});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t use_pc) {
|
||||
(void)use_pc;
|
||||
if (mem.resource >= state.program.info.buffers.size()) {
|
||||
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
|
||||
"buffer specialization is missing");
|
||||
}
|
||||
return state.program.info.buffers[mem.resource].packed_stride;
|
||||
}
|
||||
|
||||
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t use_pc) {
|
||||
(void)use_pc;
|
||||
if (mem.resource >= state.program.info.buffers.size()) {
|
||||
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
|
||||
"buffer specialization is missing");
|
||||
}
|
||||
return state.program.info.buffers[mem.resource].descriptor_format;
|
||||
}
|
||||
|
||||
void EmitStorageBufferOffsets(EmitterState& state) {
|
||||
for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) {
|
||||
const auto word =
|
||||
EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u);
|
||||
const auto shift = ConstantU32(state, (i % 4u) * 8u + 2u);
|
||||
state.storage_buffer_offsets[i] = EmitBinaryU32(
|
||||
state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
|
||||
ConstantU32(state, 0x3fu));
|
||||
}
|
||||
}
|
||||
|
||||
bool IsAddressMemoryKind(IR::ResourceKind kind) {
|
||||
return kind == IR::ResourceKind::ScalarAddress || kind == IR::ResourceKind::Flat ||
|
||||
kind == IR::ResourceKind::Global || kind == IR::ResourceKind::Scratch;
|
||||
}
|
||||
|
||||
DescriptorResourceBinding
|
||||
StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) {
|
||||
(void)use_pc;
|
||||
const auto binding =
|
||||
ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
|
||||
if (state.storage_buffer_variable == 0) {
|
||||
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
|
||||
"storage buffer descriptor array was not emitted");
|
||||
}
|
||||
return binding;
|
||||
}
|
||||
|
||||
uint32_t EmitStorageBufferIndex(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
|
||||
uint32_t use_pc) {
|
||||
if (IsAddressMemoryKind(mem.kind)) {
|
||||
return index;
|
||||
}
|
||||
const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
|
||||
return EmitAddU32(state, index, state.storage_buffer_offsets[binding.array_index]);
|
||||
}
|
||||
|
||||
uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t use_pc) {
|
||||
if (IsAddressMemoryKind(mem.kind)) {
|
||||
if (state.address_memory_variable == 0) {
|
||||
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
|
||||
mem.resource, "address memory binding was not emitted");
|
||||
}
|
||||
const auto binding =
|
||||
ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
|
||||
state.address_memory_variable,
|
||||
ConstantU32(state, binding.array_index)});
|
||||
return pointer;
|
||||
}
|
||||
const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
|
||||
state.storage_buffer_variable,
|
||||
ConstantU32(state, binding.array_index)});
|
||||
return pointer;
|
||||
}
|
||||
|
||||
uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t index, uint32_t use_pc) {
|
||||
index = EmitStorageBufferIndex(state, mem, index, use_pc);
|
||||
const auto object = EmitStorageBufferObjectPointer(state, mem, use_pc);
|
||||
const auto length = state.builder.AllocateId();
|
||||
const auto in_bounds = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0});
|
||||
state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
|
||||
return in_bounds;
|
||||
}
|
||||
|
||||
uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t index, uint32_t use_pc) {
|
||||
index = EmitStorageBufferIndex(state, mem, index, use_pc);
|
||||
if (IsAddressMemoryKind(mem.kind)) {
|
||||
if (state.address_memory_variable == 0) {
|
||||
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
|
||||
mem.resource, "address memory binding was not emitted");
|
||||
}
|
||||
const auto binding =
|
||||
ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable,
|
||||
ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
|
||||
return pointer;
|
||||
}
|
||||
const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.storage_buffer_variable,
|
||||
ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
|
||||
return pointer;
|
||||
}
|
||||
|
||||
uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, state.ptr_workgroup_uint, pointer, state.lds_variable, index});
|
||||
return pointer;
|
||||
}
|
||||
|
||||
uint32_t LdsDwordCount(const EmitterState& state) {
|
||||
return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u;
|
||||
}
|
||||
|
||||
uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) {
|
||||
const auto in_bounds = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, in_bounds, index, ConstantU32(state, LdsDwordCount(state))});
|
||||
return in_bounds;
|
||||
}
|
||||
|
||||
uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
|
||||
uint32_t use_pc) {
|
||||
if (mem.kind == IR::ResourceKind::Lds) {
|
||||
return EmitLdsElementPointer(state, index);
|
||||
}
|
||||
if (mem.kind == IR::ResourceKind::Gds) {
|
||||
return EmitGdsElementPointer(state, index);
|
||||
}
|
||||
return EmitStorageBufferElementPointer(state, mem, index, use_pc);
|
||||
}
|
||||
|
||||
uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, ret, value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, ret, value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, ret, value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
|
||||
uint32_t constant) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
|
||||
uint32_t false_value) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.float_type, ret, condition, true_value, false_value});
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool IsSignedFormatComponent(Format::ComponentType type) {
|
||||
return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm ||
|
||||
type == Format::ComponentType::Sscaled;
|
||||
}
|
||||
|
||||
uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw) {
|
||||
const auto unpacked = state.builder.AllocateId();
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, raw});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.float_type, value, unpacked, 0});
|
||||
return EmitTBufferBitcastF32ToU32(state, value);
|
||||
}
|
||||
|
||||
uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) {
|
||||
const auto mantissa_bits = bits == 11u ? 6u : 5u;
|
||||
const auto mantissa_mask = (1u << mantissa_bits) - 1u;
|
||||
const auto mantissa =
|
||||
EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask));
|
||||
const auto exponent = EmitBinaryU32(
|
||||
state, OpBitwiseAnd,
|
||||
EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)),
|
||||
ConstantU32(state, 0x1fu));
|
||||
|
||||
const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u));
|
||||
const auto exponent_bits =
|
||||
EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23));
|
||||
const auto mantissa_bits_32 =
|
||||
EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits));
|
||||
const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32);
|
||||
const auto normal = EmitTBufferBitcastU32ToF32(state, normal_bits);
|
||||
|
||||
const auto special_bits =
|
||||
EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32);
|
||||
const auto special = EmitTBufferBitcastU32ToF32(state, special_bits);
|
||||
|
||||
const auto mantissa_f32 = state.builder.AllocateId();
|
||||
const auto subnormal = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpConvertUToF, state.float_type, mantissa_f32, mantissa});
|
||||
state.builder.AddFunction(
|
||||
{OpFMul, state.float_type, subnormal, mantissa_f32,
|
||||
ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits)))});
|
||||
|
||||
const auto zero_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 0);
|
||||
const auto special_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 31);
|
||||
const auto finite = EmitTBufferSelectF32(state, zero_exp, subnormal, normal);
|
||||
const auto result = EmitTBufferSelectF32(state, special_exp, special, finite);
|
||||
return EmitTBufferBitcastF32ToU32(state, result);
|
||||
}
|
||||
|
||||
uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info,
|
||||
uint32_t component, uint32_t raw) {
|
||||
const auto bits = info.component_bits[component];
|
||||
switch (info.type) {
|
||||
case Format::ComponentType::Uint:
|
||||
case Format::ComponentType::Sint: return raw;
|
||||
case Format::ComponentType::Uscaled: {
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw});
|
||||
return EmitTBufferBitcastF32ToU32(state, value);
|
||||
}
|
||||
case Format::ComponentType::Sscaled: {
|
||||
const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw});
|
||||
return EmitTBufferBitcastF32ToU32(state, value);
|
||||
}
|
||||
case Format::ComponentType::Unorm: {
|
||||
const auto value = state.builder.AllocateId();
|
||||
const auto normalized = state.builder.AllocateId();
|
||||
const auto max_value = static_cast<float>((1u << bits) - 1u);
|
||||
state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw});
|
||||
state.builder.AddFunction(
|
||||
{OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
|
||||
return EmitTBufferBitcastF32ToU32(state, normalized);
|
||||
}
|
||||
case Format::ComponentType::Snorm: {
|
||||
const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
|
||||
const auto value = state.builder.AllocateId();
|
||||
const auto normalized = state.builder.AllocateId();
|
||||
const auto clamped = state.builder.AllocateId();
|
||||
const auto max_value = static_cast<float>((1u << (bits - 1u)) - 1u);
|
||||
state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw});
|
||||
state.builder.AddFunction(
|
||||
{OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
|
||||
state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450,
|
||||
GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
|
||||
return EmitTBufferBitcastF32ToU32(state, clamped);
|
||||
}
|
||||
case Format::ComponentType::Float:
|
||||
if (bits == 32u) {
|
||||
return raw;
|
||||
}
|
||||
if (bits == 16u) {
|
||||
return EmitHalfToF32Bits(state, raw);
|
||||
}
|
||||
return EmitUFloatToF32Bits(state, raw, bits);
|
||||
default: return raw;
|
||||
}
|
||||
}
|
||||
|
||||
void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
|
||||
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory;
|
||||
state.builder.AddFunction(
|
||||
{OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)});
|
||||
}
|
||||
|
||||
uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) {
|
||||
const auto length = state.builder.AllocateId();
|
||||
const auto in_bounds = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpArrayLength, state.uint_type, length, state.gds_variable, 0});
|
||||
state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
|
||||
return in_bounds;
|
||||
}
|
||||
|
||||
uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
|
||||
state.gds_variable, ConstantU32(state, 0), index});
|
||||
return pointer;
|
||||
}
|
||||
|
||||
uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
|
||||
if ((control & 0xc000u) == 0xc000u) {
|
||||
const uint32_t mask = control & 0x1fu;
|
||||
const uint32_t rotate = (control >> 5u) & 0x1fu;
|
||||
const uint32_t rotate_delta = (control & 0x400u) != 0u ? ((32u - rotate) & 0x1fu) : rotate;
|
||||
const auto lane = EmitAndConstant(state, subid, 31);
|
||||
const auto rotated_sum = EmitAddU32(state, lane, ConstantU32(state, rotate_delta));
|
||||
const auto rotated = EmitAndConstant(state, rotated_sum, 31);
|
||||
const auto kept = EmitAndConstant(state, lane, mask);
|
||||
const auto moved = EmitAndConstant(state, rotated, (~mask) & 31u);
|
||||
const auto combined = EmitOrU32(state, kept, moved);
|
||||
const auto base = EmitAndConstant(state, subid, 0xffffffe0u);
|
||||
return EmitOrU32(state, base, combined);
|
||||
}
|
||||
|
||||
if ((control & 0x8000u) != 0) {
|
||||
const auto lane2 = state.builder.AllocateId();
|
||||
const auto shift = state.builder.AllocateId();
|
||||
const auto perm0 = state.builder.AllocateId();
|
||||
const auto perm = state.builder.AllocateId();
|
||||
const auto base = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane2, subid, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, shift, lane2, ConstantU32(state, 1)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, perm0, ConstantU32(state, control), shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, perm, perm0, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xfffffffcu)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, perm});
|
||||
return target;
|
||||
}
|
||||
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto masked = state.builder.AllocateId();
|
||||
const auto ored = state.builder.AllocateId();
|
||||
const auto xored = state.builder.AllocateId();
|
||||
const auto base = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored});
|
||||
return target;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -1,839 +0,0 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width) {
|
||||
if (sel <= 3u) {
|
||||
offset = sel * 8u;
|
||||
width = 8u;
|
||||
return true;
|
||||
}
|
||||
if (sel == 4u || sel == 5u) {
|
||||
offset = sel == 5u ? 16u : 0u;
|
||||
width = 16u;
|
||||
return true;
|
||||
}
|
||||
if (sel == 6u) {
|
||||
offset = 0;
|
||||
width = 32u;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t EmitSdwaExtractU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
|
||||
if (operand.sdwa_sel == 6u && !operand.sdwa_sext) {
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t offset = 0;
|
||||
uint32_t width = 0;
|
||||
if (!SdwaSelectorOffsetWidth(operand.sdwa_sel, offset, width) || width == 32u) {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (operand.sdwa_sext) {
|
||||
const auto signed_value = state.builder.AllocateId();
|
||||
const auto extracted = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, signed_value, value});
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_value,
|
||||
ConstantU32(state, offset), ConstantU32(state, width)});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, extracted});
|
||||
return bits;
|
||||
}
|
||||
|
||||
const auto extracted = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, value,
|
||||
ConstantU32(state, offset), ConstantU32(state, width)});
|
||||
return extracted;
|
||||
}
|
||||
|
||||
uint32_t EmitTrueBool(EmitterState& state) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
|
||||
const auto quad_base = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto shift = state.builder.AllocateId();
|
||||
const auto selected0 = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
|
||||
bool left) {
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto lane_shifted = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
const auto valid = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
if (left) {
|
||||
state.builder.AddFunction(
|
||||
{OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)});
|
||||
}
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted});
|
||||
return {target, valid};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid,
|
||||
uint32_t amount) {
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto minus = state.builder.AllocateId();
|
||||
const auto plus = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
|
||||
const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
|
||||
const auto lane_mask = half_row ? 7u : 15u;
|
||||
const auto base = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto mirrored = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
if (control <= 0xffu) {
|
||||
return EmitDppQuadPermTargetLane(state, subid, control);
|
||||
}
|
||||
if (control >= 0x101u && control <= 0x10fu) {
|
||||
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true);
|
||||
}
|
||||
if (control >= 0x111u && control <= 0x11fu) {
|
||||
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false);
|
||||
}
|
||||
if (control >= 0x121u && control <= 0x12fu) {
|
||||
return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu);
|
||||
}
|
||||
if (control == 0x140u) {
|
||||
return EmitDppMirrorTargetLane(state, subid, false);
|
||||
}
|
||||
if (control == 0x141u) {
|
||||
return EmitDppMirrorTargetLane(state, subid, true);
|
||||
}
|
||||
return {subid, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
uint32_t EmitDppValueU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
|
||||
if (!operand.dpp) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const auto target = EmitDppTargetLane(state, operand.dpp_ctrl);
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), value, target.lane});
|
||||
if (operand.dpp_fetch_inactive) {
|
||||
return shuffled;
|
||||
}
|
||||
|
||||
const auto source_active = EmitLaneIndexActiveBool(state, target.lane);
|
||||
const auto can_fetch = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand) {
|
||||
uint32_t value = 0;
|
||||
switch (operand.kind) {
|
||||
case IR::OperandKind::ImmediateU32:
|
||||
case IR::OperandKind::PcRelativeU32:
|
||||
case IR::OperandKind::PcRelativeHighU32: value = ConstantU32(state, operand.imm); break;
|
||||
case IR::OperandKind::Register: {
|
||||
const auto pointer = PointerForRegister(state, operand.reg);
|
||||
if (pointer == 0) {
|
||||
value = ConstantU32(state, 0);
|
||||
break;
|
||||
}
|
||||
value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
break;
|
||||
}
|
||||
case IR::OperandKind::Null: value = ConstantU32(state, 0); break;
|
||||
default: value = ConstantU32(state, 0); break;
|
||||
}
|
||||
value = EmitDppValueU32(state, operand, value);
|
||||
return EmitSdwaExtractU32(state, operand, value);
|
||||
}
|
||||
|
||||
uint32_t EmitRegisterLoad(EmitterState& state, IR::Register reg) {
|
||||
if (IsInactiveWave32ExecHigh(state, reg)) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto pointer = PointerForRegister(state, reg);
|
||||
if (pointer == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t WaveMaskHighLoad(EmitterState& state, IR::RegisterFile file) {
|
||||
if (state.wave_size == 32u && IsMaskRegisterFile(file)) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitRegisterLoad(state, {file, 1});
|
||||
}
|
||||
|
||||
uint32_t EmitLaneMaskPartU32(EmitterState& state, uint32_t part) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
|
||||
if (part == 0) {
|
||||
const auto in_low = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, in_low, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_low, bit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
if (state.wave_size == 32u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_high, bit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitThreadMaskBelowPartU32(EmitterState& state, uint32_t part) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
const auto below = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
|
||||
state.builder.AddFunction({OpISub, state.uint_type, below, bit, ConstantU32(state, 1)});
|
||||
|
||||
if (part == 0) {
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below});
|
||||
return ret;
|
||||
}
|
||||
if (state.wave_size == 32u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_high, below, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskActiveBool(EmitterState& state, IR::RegisterFile file) {
|
||||
const auto low = EmitRegisterLoad(state, {file, 0});
|
||||
const auto condition = state.builder.AllocateId();
|
||||
if (state.per_invocation_masks) {
|
||||
const auto combined = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseOr, state.uint_type, combined, low, WaveMaskHighLoad(state, file)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, condition, combined, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
if (state.exact_subgroup_operations) {
|
||||
const auto high = WaveMaskHighLoad(state, file);
|
||||
const auto low_hit = state.builder.AllocateId();
|
||||
const auto high_hit = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, active, low_hit, high_hit});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, condition, active, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, condition, low, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
|
||||
const auto active = EmitMaskActiveBool(state, file);
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto cond = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, cond, active});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskSummaryZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
|
||||
if (state.per_invocation_masks) {
|
||||
return EmitMaskZeroBool(state, file, zero);
|
||||
}
|
||||
if (state.exact_subgroup_operations) {
|
||||
const auto low = EmitRegisterLoad(state, {file, 0});
|
||||
const auto high = WaveMaskHighLoad(state, file);
|
||||
const auto combined = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto inactive = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
|
||||
return inactive;
|
||||
}
|
||||
const auto active = EmitMaskActiveBool(state, file);
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto inactive = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
|
||||
return inactive;
|
||||
}
|
||||
|
||||
uint32_t EmitExecActiveBool(EmitterState& state) {
|
||||
return EmitMaskZeroBool(state, IR::RegisterFile::Exec, false);
|
||||
}
|
||||
|
||||
uint32_t EmitConditionBool(EmitterState& state, const IR::Operand& operand) {
|
||||
if (operand.kind == IR::OperandKind::Register) {
|
||||
if (operand.reg.file == IR::RegisterFile::Exec ||
|
||||
operand.reg.file == IR::RegisterFile::Vcc) {
|
||||
return EmitMaskActiveBool(state, operand.reg.file);
|
||||
}
|
||||
}
|
||||
const auto cond_value = EmitValueLoad(state, operand);
|
||||
const auto cond = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, cond, cond_value, ConstantU32(state, 0)});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
|
||||
if (state.subgroup_local_invocation_id_variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == kind) {
|
||||
return input.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
|
||||
return input.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
|
||||
return &input;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
|
||||
const auto variable = InputVariableForKind(state, kind);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)});
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t EmitLocalInvocationIndex(EmitterState& state) {
|
||||
const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, variable});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind,
|
||||
uint32_t component) {
|
||||
if ((component & 3u) != 3u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
|
||||
}
|
||||
|
||||
uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
|
||||
uint32_t component) {
|
||||
const auto count = VertexParameterComponentCount(state, input);
|
||||
const auto kind = VertexParameterScalarKind(state, input.location);
|
||||
if (component >= count) {
|
||||
return VertexInputDefaultComponentU32(state, kind, component);
|
||||
}
|
||||
|
||||
const auto scalar_type = VertexParameterScalarType(state, kind);
|
||||
uint32_t raw = state.builder.AllocateId();
|
||||
if (count == 1u) {
|
||||
state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
|
||||
} else {
|
||||
const auto pointer_type = VertexParameterScalarPointerType(state, kind);
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
|
||||
ConstantU32(state, component)});
|
||||
state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
|
||||
}
|
||||
|
||||
if (kind == VertexInputScalarKind::Uint) {
|
||||
return raw;
|
||||
}
|
||||
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
|
||||
return bits;
|
||||
}
|
||||
|
||||
void EmitLoadInputF32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto* input = InputBindingForParameter(state, inst.input_info.attr);
|
||||
if (input == nullptr || input->variable_id == 0) {
|
||||
EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.stage == ShaderType::Vertex) {
|
||||
EmitStoreU32(state, inst.dst,
|
||||
EmitVertexParameterComponentU32(state, *input, inst.input_info.chan & 3u));
|
||||
return;
|
||||
}
|
||||
|
||||
const auto input_value = state.builder.AllocateId();
|
||||
const auto component = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.vec4_float_type, input_value, input->variable_id});
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, component, input_value, inst.input_info.chan & 3u});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
EmitStoreU32(state, inst.dst, bits);
|
||||
}
|
||||
|
||||
uint32_t EmitSccBool(EmitterState& state, bool non_zero) {
|
||||
const auto value = EmitRegisterLoad(state, SccRegister());
|
||||
const auto cond = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{non_zero ? OpINotEqual : OpIEqual, state.bool_type, cond, value, ConstantU32(state, 0)});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitFloatLoad(EmitterState& state, const IR::Operand& operand) {
|
||||
const auto uint_value = EmitValueLoad(state, operand);
|
||||
auto float_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, float_value, uint_value});
|
||||
if (operand.absolute) {
|
||||
const auto abs_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, float_value});
|
||||
float_value = abs_value;
|
||||
}
|
||||
if (operand.negate) {
|
||||
const auto neg_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpFNegate, state.float_type, neg_value, float_value});
|
||||
float_value = neg_value;
|
||||
}
|
||||
return float_value;
|
||||
}
|
||||
|
||||
uint32_t ApplyResultModifiersF32(EmitterState& state, uint32_t value, const IR::Operand& dst) {
|
||||
uint32_t result = value;
|
||||
if (dst.omod != 0) {
|
||||
uint32_t bits = 0x3f000000u;
|
||||
switch (dst.omod) {
|
||||
case 1u: bits = 0x40000000u; break;
|
||||
case 2u: bits = 0x40800000u; break;
|
||||
default: break;
|
||||
}
|
||||
const auto scaled = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpFMul, state.float_type, scaled, result, ConstantF32(state, bits)});
|
||||
result = scaled;
|
||||
}
|
||||
if (!dst.clamp) {
|
||||
return result;
|
||||
}
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpExtInst, state.float_type, ret, state.glsl_std450, GlslFClamp,
|
||||
result, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitMixF32Load(EmitterState& state, const IR::Operand& operand) {
|
||||
auto value_operand = operand;
|
||||
value_operand.negate_hi = false;
|
||||
|
||||
if (operand.op_sel_hi) {
|
||||
auto raw = operand;
|
||||
raw.sdwa_sel = 6;
|
||||
raw.sdwa_sext = false;
|
||||
raw.op_sel = false;
|
||||
raw.op_sel_hi = false;
|
||||
raw.negate = false;
|
||||
raw.negate_hi = false;
|
||||
raw.absolute = false;
|
||||
raw.dpp = false;
|
||||
raw.dpp_fetch_inactive = false;
|
||||
raw.dpp_bound_ctrl = false;
|
||||
|
||||
const auto bits = EmitValueLoad(state, raw);
|
||||
const auto unpacked = state.builder.AllocateId();
|
||||
auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450,
|
||||
GlslUnpackHalf2x16, bits});
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, f32, unpacked, operand.op_sel ? 1u : 0u});
|
||||
if (operand.absolute) {
|
||||
const auto abs_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, f32});
|
||||
f32 = abs_value;
|
||||
}
|
||||
if (value_operand.negate) {
|
||||
const auto neg_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpFNegate, state.float_type, neg_value, f32});
|
||||
f32 = neg_value;
|
||||
}
|
||||
return f32;
|
||||
}
|
||||
|
||||
value_operand.op_sel = false;
|
||||
value_operand.op_sel_hi = false;
|
||||
return EmitFloatLoad(state, value_operand);
|
||||
}
|
||||
|
||||
IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset) {
|
||||
if (offset == 0 || operand.kind != IR::OperandKind::Register) {
|
||||
return operand;
|
||||
}
|
||||
auto ret = operand;
|
||||
ret.reg.index += offset;
|
||||
ret.sdwa_sel = 6;
|
||||
ret.omod = 0;
|
||||
ret.sdwa_sext = false;
|
||||
ret.op_sel = false;
|
||||
ret.op_sel_hi = false;
|
||||
ret.negate = false;
|
||||
ret.negate_hi = false;
|
||||
ret.absolute = false;
|
||||
ret.clamp = false;
|
||||
ret.dpp_ctrl = 0;
|
||||
ret.dpp_row_mask = 0xf;
|
||||
ret.dpp_bank_mask = 0xf;
|
||||
ret.dpp_fetch_inactive = false;
|
||||
ret.dpp_bound_ctrl = false;
|
||||
ret.dpp = false;
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitLaneMaskOperandActiveBool(EmitterState& state, const IR::Operand& operand) {
|
||||
const auto low = EmitValueLoad(state, operand);
|
||||
const auto active = state.builder.AllocateId();
|
||||
if (state.per_invocation_masks || state.exact_subgroup_operations) {
|
||||
uint32_t high = ConstantU32(state, 0);
|
||||
if (operand.kind == IR::OperandKind::Register &&
|
||||
(operand.reg.file == IR::RegisterFile::Scalar ||
|
||||
operand.reg.file == IR::RegisterFile::Vcc ||
|
||||
operand.reg.file == IR::RegisterFile::Exec)) {
|
||||
high = IsMaskRegisterFile(operand.reg.file)
|
||||
? WaveMaskHighLoad(state, operand.reg.file)
|
||||
: EmitValueLoad(state, OffsetRegisterOperand(operand, 1));
|
||||
}
|
||||
if (state.per_invocation_masks) {
|
||||
const auto combined = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
const auto low_hit = state.builder.AllocateId();
|
||||
const auto high_hit = state.builder.AllocateId();
|
||||
const auto hit = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, hit, low_hit, high_hit});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, active, low, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
|
||||
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane) {
|
||||
if (state.per_invocation_masks) {
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
|
||||
return EmitBallotLaneActiveBool(state, ballot, lane);
|
||||
}
|
||||
const auto lane_low = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
|
||||
|
||||
uint32_t mask = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
if (state.wave_size == 64u) {
|
||||
const auto high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, mask});
|
||||
mask = selected;
|
||||
}
|
||||
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, ret, hit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
|
||||
const auto active_ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot,
|
||||
ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
|
||||
return EmitBallotLaneActiveBool(state, active_ballot, lane);
|
||||
}
|
||||
uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
|
||||
const auto low = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0});
|
||||
uint32_t mask = low;
|
||||
if (state.wave_size == 64u) {
|
||||
const auto high = state.builder.AllocateId();
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low});
|
||||
mask = selected;
|
||||
}
|
||||
|
||||
const auto lane_low = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
|
||||
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
const auto in_range = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)});
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitSequentialValueLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
|
||||
if (offset == 0 || operand.kind == IR::OperandKind::Register) {
|
||||
return EmitValueLoad(state, OffsetRegisterOperand(operand, offset));
|
||||
}
|
||||
if (operand.kind == IR::OperandKind::ImmediateU32 && offset == 1) {
|
||||
return ConstantU32(state, operand.sext_64 ? 0xffffffffu : 0u);
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
|
||||
uint32_t EmitSequentialFloatLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
|
||||
const auto value = EmitSequentialValueLoad(state, operand, offset);
|
||||
const auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
|
||||
return f32;
|
||||
}
|
||||
|
||||
bool ImageAddressUsesA16(const IR::Instruction& inst) {
|
||||
return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagA16) != 0u;
|
||||
}
|
||||
|
||||
bool ImageAddressComponentUsesA16(const IR::Instruction& inst, uint32_t component) {
|
||||
if (!ImageAddressUsesA16(inst)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t cursor = 0;
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagOffset) != 0u) {
|
||||
if (component == cursor) {
|
||||
return false;
|
||||
}
|
||||
cursor++;
|
||||
}
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagBias) != 0u) {
|
||||
if (component == cursor) {
|
||||
return true;
|
||||
}
|
||||
cursor++;
|
||||
}
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u &&
|
||||
component == cursor) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component) {
|
||||
if (!ImageAddressUsesA16(inst)) {
|
||||
return component * 2u;
|
||||
}
|
||||
|
||||
uint32_t half_component = 0;
|
||||
for (uint32_t i = 0; i < component; i++) {
|
||||
const auto half_width = ImageAddressComponentUsesA16(inst, i) ? 1u : 2u;
|
||||
if (half_width == 2u && (half_component & 1u) != 0u) {
|
||||
half_component++;
|
||||
}
|
||||
half_component += half_width;
|
||||
}
|
||||
if (!ImageAddressComponentUsesA16(inst, component) && (half_component & 1u) != 0u) {
|
||||
half_component++;
|
||||
}
|
||||
return half_component;
|
||||
}
|
||||
|
||||
uint32_t ImageAddressPackedComponent(const IR::Instruction& inst, uint32_t component) {
|
||||
return ImageAddressHalfComponent(inst, component) / 2u;
|
||||
}
|
||||
|
||||
IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base,
|
||||
uint32_t component) {
|
||||
const auto packed_component = ImageAddressPackedComponent(inst, component);
|
||||
if (packed_component > 0 && inst.memory.image_nsa_dwords != 0) {
|
||||
const auto nsa_index = packed_component - 1u;
|
||||
const auto nsa_count = std::min<uint32_t>(inst.memory.image_nsa_dwords * 4u,
|
||||
Decoder::MaxImageNsaAddressComponents);
|
||||
if (nsa_index < nsa_count) {
|
||||
return MakeRegisterOperand(IR::RegisterFile::Vector,
|
||||
inst.memory.image_nsa_addr[nsa_index]);
|
||||
}
|
||||
}
|
||||
return OffsetRegisterOperand(base, packed_component);
|
||||
}
|
||||
|
||||
uint32_t EmitImageAddressValueLoad(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& base, uint32_t component) {
|
||||
const auto operand = ImageAddressOperand(inst, base, component);
|
||||
const auto packed_component = ImageAddressPackedComponent(inst, component);
|
||||
if (packed_component == 0 || operand.kind == IR::OperandKind::Register) {
|
||||
auto value = EmitValueLoad(state, operand);
|
||||
if (ImageAddressComponentUsesA16(inst, component)) {
|
||||
if ((ImageAddressHalfComponent(inst, component) & 1u) != 0u) {
|
||||
value = EmitShiftRightConstant(state, value, 16u);
|
||||
}
|
||||
value = EmitAndConstant(state, value, 0xffffu);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
|
||||
uint32_t EmitImageAddressFloatLoad(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& base, uint32_t component) {
|
||||
const auto value = EmitImageAddressValueLoad(state, inst, base, component);
|
||||
if (ImageAddressComponentUsesA16(inst, component)) {
|
||||
return EmitF16BitsToF32(state, value);
|
||||
}
|
||||
const auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
|
||||
return f32;
|
||||
}
|
||||
|
||||
uint32_t EmitZeroF32(EmitterState& state) {
|
||||
const auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32, ConstantU32(state, 0)});
|
||||
return f32;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
Reference in New Issue
Block a user