raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-29 22:36:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

graphics: refactoring #2

Browse Source
This commit is contained in:
nmzik
2026-07-21 07:20:55 +02:00
parent b51d4322cc
commit 0a5f78c727
137 changed files with 10230 additions and 10893 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAluOps.cpp
+1162 -1181
View File
File diff suppressed because it is too large Load Diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAnalysis.cpp
+46 -46
View File
@@ -60,12 +60,12 @@ void SetError(std::string* error, const char* message) {
}
}
void CollectRegister(std::vector<RegisterBinding>* registers, IR::Register reg) {
if (std::any_of(registers->begin(), registers->end(),
void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg) {
if (std::any_of(registers.begin(), registers.end(),
[reg](const RegisterBinding& binding) { return binding.reg == reg; })) {
return;
}
registers->push_back({reg, 0});
registers.push_back({reg, 0});
}
IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index) {
@@ -98,7 +98,7 @@ bool IsSccOperand(const IR::Operand& operand) {
bool IsCompareOpcode(IR::Opcode op);
void CollectMaskStateRegisters(std::vector<RegisterBinding>* registers) {
void CollectMaskStateRegisters(std::vector<RegisterBinding>& registers) {
CollectRegister(registers, {IR::RegisterFile::Exec, 0});
CollectRegister(registers, {IR::RegisterFile::Exec, 1});
CollectRegister(registers, {IR::RegisterFile::Vcc, 0});
@@ -106,7 +106,7 @@ void CollectMaskStateRegisters(std::vector<RegisterBinding>* registers) {
CollectRegister(registers, SccRegister());
}
void CollectSequentialRegisters(std::vector<RegisterBinding>* registers, const IR::Operand& base,
void CollectSequentialRegisters(std::vector<RegisterBinding>& registers, const IR::Operand& base,
uint32_t count) {
if (base.kind != IR::OperandKind::Register) {
return;
@@ -129,8 +129,8 @@ uint32_t MaxCollectedVectorRegisterEnd(const std::vector<RegisterBinding>& regis
}
void CollectMoveRelSourceRegisters(const IR::Program& program,
std::vector<RegisterBinding>* registers) {
const auto max_vector_end = MaxCollectedVectorRegisterEnd(*registers);
std::vector<RegisterBinding>& registers) {
const auto max_vector_end = MaxCollectedVectorRegisterEnd(registers);
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op != IR::Opcode::MoveRelSourceU32 ||
@@ -184,7 +184,7 @@ uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count) {
}
}
void CollectRegisters(const IR::Program& program, std::vector<RegisterBinding>* registers) {
void CollectRegisters(const IR::Program& program, std::vector<RegisterBinding>& registers) {
CollectMaskStateRegisters(registers);
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
@@ -267,16 +267,16 @@ bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind ki
});
}
void CopyProgramInputsAndOutputs(EmitterState* state, const IR::Program& program) {
void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
for (const auto& input: program.info.inputs) {
state->inputs.push_back(
state.inputs.push_back(
{input.kind, input.location, input.component_count, 0, input.debug_name});
}
for (const auto& output: program.info.outputs) {
if (HasOutput(state->outputs, output.kind, output.index)) {
if (HasOutput(state.outputs, output.kind, output.index)) {
continue;
}
state->outputs.push_back(
state.outputs.push_back(
{output.kind, output.index, output.location, 0, output.debug_name});
}
}
@@ -528,23 +528,23 @@ uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) {
return 0;
}
uint32_t ConstantU32(EmitterState* state, uint32_t value);
void EmitStoreU32(EmitterState* state, const IR::Operand& dst, uint32_t value);
uint32_t EmitSubgroupLocalInvocationId(EmitterState* state);
uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane);
uint32_t ConstantU32(EmitterState& state, uint32_t value);
void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value);
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane);
[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state,
IR::DescriptorBindingKind kind, uint32_t resource,
const char* reason) {
EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
" stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
state.program->shader_hash, static_cast<unsigned>(state.stage), resource,
state.program.shader_hash, static_cast<unsigned>(state.stage), resource,
static_cast<unsigned>(kind), reason);
std::abort();
}
DescriptorResourceBinding ResourceForDescriptor(const EmitterState& state,
IR::DescriptorBindingKind kind, uint32_t resource) {
const auto* descriptor = IR::FindBinding(state.program->bindings, kind);
const auto* descriptor = IR::FindBinding(state.program.bindings, kind);
if (descriptor == nullptr) {
ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated");
}
@@ -557,15 +557,15 @@ DescriptorResourceBinding ResourceForDescriptor(const EmitterState& state,
return {descriptor, static_cast<uint32_t>(found - descriptor->resources.begin())};
}
uint32_t DescriptorElementPointer(EmitterState* state, uint32_t result_ptr_type,
uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
uint32_t variable_id, uint32_t array_index,
IR::DescriptorBindingKind kind, uint32_t resource,
const char* variable_name) {
if (variable_id == 0) {
ExitDescriptorBindingFailure(*state, kind, resource, variable_name);
ExitDescriptorBindingFailure(state, kind, resource, variable_name);
}
const auto pointer = state->builder.AllocateId();
state->builder.AddFunction(
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
return pointer;
}
@@ -673,71 +673,71 @@ IR::DescriptorBindingKind StorageBindingKind(bool uint_image, ImageViewKind view
}
}
uint32_t LoadSampledImageDescriptor(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc,
uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
ImageViewKind view) {
(void)use_pc;
const bool integer = mem.kind == IR::ResourceKind::ImageUint;
const auto kind = SampledBindingKind(integer, view);
const auto binding = ResourceForDescriptor(*state, kind, mem.resource);
const auto& descriptors = state->sampled_images[SampledImageIndex(integer, view)];
const auto binding = ResourceForDescriptor(state, kind, mem.resource);
const auto& descriptors = state.sampled_images[SampledImageIndex(integer, view)];
const auto pointer = DescriptorElementPointer(
state, descriptors.pointer_type, descriptors.variable, binding.array_index, kind,
mem.resource, "sampled image descriptor array was not emitted");
const auto image = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, ImageViewImageType(*state, view, integer), image, pointer});
const auto image = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, ImageViewImageType(state, view, integer), image, pointer});
return image;
}
uint32_t LoadSamplerDescriptor(EmitterState* state, uint32_t sampler, uint32_t use_pc) {
uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t use_pc) {
(void)use_pc;
const auto binding =
ResourceForDescriptor(*state, IR::DescriptorBindingKind::Samplers, sampler);
ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
const auto pointer = DescriptorElementPointer(
state, state->ptr_uniform_sampler, state->sampler_variable, binding.array_index,
state, state.ptr_uniform_sampler, state.sampler_variable, binding.array_index,
IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
const auto sampler_id = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->sampler_type, sampler_id, pointer});
const auto sampler_id = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.sampler_type, sampler_id, pointer});
return sampler_id;
}
uint32_t MakeSampledImage(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc,
uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
ImageViewKind view) {
const auto image = LoadSampledImageDescriptor(state, mem, use_pc, view);
const auto sampler = LoadSamplerDescriptor(state, mem.sampler, use_pc);
if (image == 0 || sampler == 0) {
ExitDescriptorBindingFailure(
*state, SampledBindingKind(mem.kind == IR::ResourceKind::ImageUint, view), mem.resource,
state, SampledBindingKind(mem.kind == IR::ResourceKind::ImageUint, view), mem.resource,
"sampled image or sampler descriptor load failed");
}
const auto sampled_image = state->builder.AllocateId();
state->builder.AddFunction(
const auto sampled_image = state.builder.AllocateId();
state.builder.AddFunction(
{OpSampledImage,
ImageViewSampledImageType(*state, view, mem.kind == IR::ResourceKind::ImageUint),
ImageViewSampledImageType(state, view, mem.kind == IR::ResourceKind::ImageUint),
sampled_image, image, sampler});
return sampled_image;
}
uint32_t StorageImageDescriptorPointer(EmitterState* state, uint32_t resource, bool uint_image,
uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, bool uint_image,
uint32_t use_pc, ImageViewKind view) {
(void)use_pc;
const auto kind = StorageBindingKind(uint_image, view);
const auto binding = ResourceForDescriptor(*state, kind, resource);
const auto ptr_type = StorageImagePointerType(*state, uint_image, view);
const auto variable = StorageImageVariable(*state, uint_image, view);
const auto binding = ResourceForDescriptor(state, kind, resource);
const auto ptr_type = StorageImagePointerType(state, uint_image, view);
const auto variable = StorageImageVariable(state, uint_image, view);
return DescriptorElementPointer(state, ptr_type, variable, binding.array_index, kind, resource,
"storage image descriptor array was not emitted");
}
uint32_t LoadStorageImageDescriptor(EmitterState* state, uint32_t resource, bool uint_image,
uint32_t LoadStorageImageDescriptor(EmitterState& state, uint32_t resource, bool uint_image,
uint32_t use_pc, ImageViewKind view) {
const auto pointer = StorageImageDescriptorPointer(state, resource, uint_image, use_pc, view);
if (pointer == 0) {
ExitDescriptorBindingFailure(*state, StorageBindingKind(uint_image, view), resource,
ExitDescriptorBindingFailure(state, StorageBindingKind(uint_image, view), resource,
"storage image descriptor pointer creation failed");
}
const auto type = StorageImageType(*state, uint_image, view);
const auto image = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, type, image, pointer});
const auto type = StorageImageType(state, uint_image, view);
const auto image = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, type, image, pointer});
return image;
}
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterControlFlow.cpp
+212 -212
View File
@@ -2,8 +2,8 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
void ComputeReachableBlocks(EmitterState* state, const IR::Program& program) {
state->reachable_blocks.assign(program.blocks.size(), false);
void ComputeReachableBlocks(EmitterState& state, const IR::Program& program) {
state.reachable_blocks.assign(program.blocks.size(), false);
if (program.blocks.empty()) {
return;
}
@@ -13,38 +13,38 @@ void ComputeReachableBlocks(EmitterState* state, const IR::Program& program) {
while (!stack.empty()) {
const auto block_id = stack.back();
stack.pop_back();
if (block_id >= program.blocks.size() || state->reachable_blocks[block_id]) {
if (block_id >= program.blocks.size() || state.reachable_blocks[block_id]) {
continue;
}
state->reachable_blocks[block_id] = true;
state.reachable_blocks[block_id] = true;
for (auto succ: program.blocks[block_id].successors) {
if (succ < program.blocks.size() && !state->reachable_blocks[succ]) {
if (succ < program.blocks.size() && !state.reachable_blocks[succ]) {
stack.push_back(succ);
}
}
}
}
void AllocateBlockLabels(EmitterState* state, const IR::Program& program) {
void AllocateBlockLabels(EmitterState& state, const IR::Program& program) {
for (const auto& block: program.blocks) {
if (block.id < state->reachable_blocks.size() && state->reachable_blocks[block.id]) {
state->block_labels.emplace(block.id, state->builder.AllocateId());
if (block.id < state.reachable_blocks.size() && state.reachable_blocks[block.id]) {
state.block_labels.emplace(block.id, state.builder.AllocateId());
}
}
}
void AllocateDispatcherState(EmitterState* state, const IR::Program& program) {
state->dispatcher_fallback = program.dispatcher_fallback;
if (!state->dispatcher_fallback) {
void AllocateDispatcherState(EmitterState& state, const IR::Program& program) {
state.dispatcher_fallback = program.dispatcher_fallback;
if (!state.dispatcher_fallback) {
return;
}
state->dispatch_pc_variable = state->builder.AllocateId();
state->dispatch_header_label = state->builder.AllocateId();
state->dispatch_select_label = state->builder.AllocateId();
state->dispatch_default_label = state->builder.AllocateId();
state->dispatch_after_switch_label = state->builder.AllocateId();
state->dispatch_continue_label = state->builder.AllocateId();
state->dispatch_merge_label = state->builder.AllocateId();
state.dispatch_pc_variable = state.builder.AllocateId();
state.dispatch_header_label = state.builder.AllocateId();
state.dispatch_select_label = state.builder.AllocateId();
state.dispatch_default_label = state.builder.AllocateId();
state.dispatch_after_switch_label = state.builder.AllocateId();
state.dispatch_continue_label = state.builder.AllocateId();
state.dispatch_merge_label = state.builder.AllocateId();
}
uint32_t BlockLabel(const EmitterState& state, uint32_t block_id) {
@@ -52,7 +52,7 @@ uint32_t BlockLabel(const EmitterState& state, uint32_t block_id) {
return it == state.block_labels.end() ? 0 : it->second;
}
uint32_t EmitBranchCondition(EmitterState* state, CFG::BranchCondition condition) {
uint32_t EmitBranchCondition(EmitterState& state, CFG::BranchCondition condition) {
switch (condition) {
case CFG::BranchCondition::SccZero: return EmitSccBool(state, false);
case CFG::BranchCondition::SccNonZero: return EmitSccBool(state, true);
@@ -68,29 +68,29 @@ uint32_t EmitBranchCondition(EmitterState* state, CFG::BranchCondition condition
}
}
void EmitStructuredPrefix(EmitterState* state, const CFG::Terminator& term) {
void EmitStructuredPrefix(EmitterState& state, const CFG::Terminator& term) {
if (term.loop_header) {
const auto merge_label = BlockLabel(*state, term.merge_block);
const auto continue_label = BlockLabel(*state, term.continue_block);
const auto merge_label = BlockLabel(state, term.merge_block);
const auto continue_label = BlockLabel(state, term.continue_block);
if (merge_label != 0 && continue_label != 0) {
state->builder.AddFunction({OpLoopMerge, merge_label, continue_label, LoopControlNone});
state.builder.AddFunction({OpLoopMerge, merge_label, continue_label, LoopControlNone});
}
return;
}
if (term.kind == CFG::TerminatorKind::ConditionalBranch && term.merge_block != UINT32_MAX) {
const auto merge_label = BlockLabel(*state, term.merge_block);
const auto merge_label = BlockLabel(state, term.merge_block);
if (merge_label != 0) {
state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
}
}
}
void EmitReturn(EmitterState* state) {
void EmitReturn(EmitterState& state) {
EmitKillIfPixelValidMaskInactive(state);
state->builder.AddFunction({OpReturn});
state.builder.AddFunction({OpReturn});
}
void EmitTerminator(EmitterState* state, const CFG::Terminator& term) {
void EmitTerminator(EmitterState& state, const CFG::Terminator& term) {
const auto condition = term.kind == CFG::TerminatorKind::ConditionalBranch
? EmitBranchCondition(state, term.condition)
: 0;
@@ -98,19 +98,19 @@ void EmitTerminator(EmitterState* state, const CFG::Terminator& term) {
switch (term.kind) {
case CFG::TerminatorKind::Branch: {
const auto label = BlockLabel(*state, term.true_block);
const auto label = BlockLabel(state, term.true_block);
if (label != 0) {
state->builder.AddFunction({OpBranch, label});
state.builder.AddFunction({OpBranch, label});
} else {
EmitReturn(state);
}
break;
}
case CFG::TerminatorKind::ConditionalBranch: {
const auto true_label = BlockLabel(*state, term.true_block);
const auto false_label = BlockLabel(*state, term.false_block);
const auto true_label = BlockLabel(state, term.true_block);
const auto false_label = BlockLabel(state, term.false_block);
if (true_label != 0 && false_label != 0) {
state->builder.AddFunction(
state.builder.AddFunction(
{OpBranchConditional, condition, true_label, false_label});
} else {
EmitReturn(state);
@@ -122,10 +122,10 @@ void EmitTerminator(EmitterState* state, const CFG::Terminator& term) {
}
bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) {
if (state.program == nullptr || reg.file != IR::RegisterFile::Scalar) {
if (reg.file != IR::RegisterFile::Scalar) {
return false;
}
const auto& registers = state.program->bindings.user_data_registers;
const auto& registers = state.program.bindings.user_data_registers;
const auto found = std::lower_bound(registers.begin(), registers.end(), reg.index);
if (found != registers.end() && *found == reg.index) {
dword_index = static_cast<uint32_t>(found - registers.begin());
@@ -134,24 +134,24 @@ bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& d
return false;
}
uint32_t EmitVsharpDwordLoad(EmitterState* state, uint32_t dword_index) {
if (state->push_constant_variable != 0) {
const auto pointer = state->builder.AllocateId();
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpAccessChain, state->ptr_push_constant_uint, pointer,
state->push_constant_variable, ConstantU32(state, 0),
uint32_t EmitVsharpDwordLoad(EmitterState& state, uint32_t dword_index) {
if (state.push_constant_variable != 0) {
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_push_constant_uint, pointer,
state.push_constant_variable, ConstantU32(state, 0),
ConstantU32(state, dword_index / 4u),
ConstantU32(state, dword_index % 4u)});
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
}
if (state->vsharp_storage_variable != 0) {
const auto pointer = state->builder.AllocateId();
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpAccessChain, state->ptr_storage_buffer_uint, pointer,
state->vsharp_storage_variable, ConstantU32(state, 0),
if (state.vsharp_storage_variable != 0) {
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
state.vsharp_storage_variable, ConstantU32(state, 0),
ConstantU32(state, dword_index)});
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
}
return ConstantU32(state, 0);
@@ -170,52 +170,52 @@ uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg) {
return 0;
}
uint32_t EmitInitialRegisterValue(EmitterState* state, IR::Register reg) {
uint32_t EmitInitialRegisterValue(EmitterState& state, IR::Register reg) {
uint32_t dword_index = 0;
if (UserDataDwordIndex(*state, reg, dword_index)) {
if (UserDataDwordIndex(state, reg, dword_index)) {
return EmitVsharpDwordLoad(state, dword_index);
}
return ConstantU32(state, InitialRegisterValue(*state, reg));
return ConstantU32(state, InitialRegisterValue(state, reg));
}
void EmitRegisterVariables(EmitterState* state) {
if (state->needs_function_lds) {
state->builder.AddFunction(
{OpVariable, state->ptr_workgroup_array, state->lds_variable, StorageClassFunction});
void EmitRegisterVariables(EmitterState& state) {
if (state.needs_function_lds) {
state.builder.AddFunction(
{OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassFunction});
}
for (const auto& binding: state->registers) {
state->builder.AddFunction(
{OpVariable, state->ptr_func_uint, binding.pointer_id, StorageClassFunction});
for (const auto& binding: state.registers) {
state.builder.AddFunction(
{OpVariable, state.ptr_func_uint, binding.pointer_id, StorageClassFunction});
}
if (state->dispatcher_fallback) {
state->builder.AddFunction(
{OpVariable, state->ptr_func_uint, state->dispatch_pc_variable, StorageClassFunction});
if (state.dispatcher_fallback) {
state.builder.AddFunction(
{OpVariable, state.ptr_func_uint, state.dispatch_pc_variable, StorageClassFunction});
}
if (state->pixel_valid_mask_variable != 0) {
state->builder.AddFunction({OpVariable, state->ptr_func_uint,
state->pixel_valid_mask_variable, StorageClassFunction});
state->builder.AddFunction(
{OpStore, state->pixel_valid_mask_variable, ConstantU32(state, 1)});
if (state.pixel_valid_mask_variable != 0) {
state.builder.AddFunction({OpVariable, state.ptr_func_uint,
state.pixel_valid_mask_variable, StorageClassFunction});
state.builder.AddFunction(
{OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
}
for (const auto& binding: state->registers) {
state->builder.AddFunction(
for (const auto& binding: state.registers) {
state.builder.AddFunction(
{OpStore, binding.pointer_id, EmitInitialRegisterValue(state, binding.reg)});
}
}
void EmitComputeInputRegisters(EmitterState* state) {
if (state->stage != ShaderType::Compute || state->compute_input_info == nullptr) {
void EmitComputeInputRegisters(EmitterState& state) {
if (state.stage != ShaderType::Compute || state.compute_input_info == nullptr) {
return;
}
const auto* cs = state->compute_input_info;
const auto* cs = state.compute_input_info;
const auto thread_ids = cs->thread_ids_num > 0
? std::min<uint32_t>(static_cast<uint32_t>(cs->thread_ids_num), 3u)
: 0u;
for (uint32_t i = 0; i < thread_ids; i++) {
const auto pointer = PointerForRegister(*state, {IR::RegisterFile::Vector, i});
const auto pointer = PointerForRegister(state, {IR::RegisterFile::Vector, i});
if (pointer != 0) {
state->builder.AddFunction(
state.builder.AddFunction(
{OpStore, pointer,
EmitInputComponentU32(state, IR::StageInputKind::LocalInvocationId, i)});
}
@@ -227,10 +227,10 @@ void EmitComputeInputRegisters(EmitterState* state) {
continue;
}
const auto pointer = PointerForRegister(
*state,
state,
{IR::RegisterFile::Scalar, static_cast<uint32_t>(cs->workgroup_register) + reg_offset});
if (pointer != 0) {
state->builder.AddFunction(
state.builder.AddFunction(
{OpStore, pointer,
EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, i)});
}
@@ -239,7 +239,7 @@ void EmitComputeInputRegisters(EmitterState* state) {
if (cs->tg_size_en) {
const auto pointer = PointerForRegister(
*state,
state,
{IR::RegisterFile::Scalar, static_cast<uint32_t>(cs->workgroup_register) + reg_offset});
if (pointer != 0) {
const uint32_t wave_size = cs->wave_size != 0 ? cs->wave_size : 64u;
@@ -250,66 +250,66 @@ void EmitComputeInputRegisters(EmitterState* state) {
std::min<uint32_t>((total_threads + wave_size - 1u) / wave_size, 0x3fu);
const auto local_index = EmitLocalInvocationIndex(state);
const auto wave_id = state->builder.AllocateId();
const auto wave_bits = state->builder.AllocateId();
const auto is_first = state->builder.AllocateId();
const auto first_bit = state->builder.AllocateId();
const auto base = state->builder.AllocateId();
const auto packed = state->builder.AllocateId();
state->builder.AddFunction(
{OpUDiv, state->uint_type, wave_id, local_index, ConstantU32(state, wave_size)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, wave_bits, wave_id, ConstantU32(state, 20)});
state->builder.AddFunction(
{OpIEqual, state->bool_type, is_first, wave_id, ConstantU32(state, 0)});
state->builder.AddFunction({OpSelect, state->uint_type, first_bit, is_first,
const auto wave_id = state.builder.AllocateId();
const auto wave_bits = state.builder.AllocateId();
const auto is_first = state.builder.AllocateId();
const auto first_bit = state.builder.AllocateId();
const auto base = state.builder.AllocateId();
const auto packed = state.builder.AllocateId();
state.builder.AddFunction(
{OpUDiv, state.uint_type, wave_id, local_index, ConstantU32(state, wave_size)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, wave_bits, wave_id, ConstantU32(state, 20)});
state.builder.AddFunction(
{OpIEqual, state.bool_type, is_first, wave_id, ConstantU32(state, 0)});
state.builder.AddFunction({OpSelect, state.uint_type, first_bit, is_first,
ConstantU32(state, 0x80000000u), ConstantU32(state, 0)});
state->builder.AddFunction(
{OpBitwiseOr, state->uint_type, base, wave_bits, ConstantU32(state, waves)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, packed, base, first_bit});
state->builder.AddFunction({OpStore, pointer, packed});
state.builder.AddFunction(
{OpBitwiseOr, state.uint_type, base, wave_bits, ConstantU32(state, waves)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, packed, base, first_bit});
state.builder.AddFunction({OpStore, pointer, packed});
}
}
}
void EmitPixelFragCoordComponent(EmitterState* state, uint32_t component, uint32_t vgpr) {
const auto input = InputVariableForKind(*state, IR::StageInputKind::FragCoord);
const auto dst = PointerForRegister(*state, {IR::RegisterFile::Vector, vgpr});
void EmitPixelFragCoordComponent(EmitterState& state, uint32_t component, uint32_t vgpr) {
const auto input = InputVariableForKind(state, IR::StageInputKind::FragCoord);
const auto dst = PointerForRegister(state, {IR::RegisterFile::Vector, vgpr});
if (input == 0 || dst == 0) {
return;
}
const auto pointer = state->builder.AllocateId();
const auto value = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction(
{OpAccessChain, state->ptr_input_float, pointer, input, ConstantU32(state, component)});
state->builder.AddFunction({OpLoad, state->float_type, value, pointer});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, value});
state->builder.AddFunction({OpStore, dst, bits});
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, state.ptr_input_float, pointer, input, ConstantU32(state, component)});
state.builder.AddFunction({OpLoad, state.float_type, value, pointer});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
state.builder.AddFunction({OpStore, dst, bits});
}
void EmitPixelFrontFacingRegister(EmitterState* state, uint32_t vgpr) {
const auto input = InputVariableForKind(*state, IR::StageInputKind::FrontFacing);
const auto dst = PointerForRegister(*state, {IR::RegisterFile::Vector, vgpr});
void EmitPixelFrontFacingRegister(EmitterState& state, uint32_t vgpr) {
const auto input = InputVariableForKind(state, IR::StageInputKind::FrontFacing);
const auto dst = PointerForRegister(state, {IR::RegisterFile::Vector, vgpr});
if (input == 0 || dst == 0) {
return;
}
const auto value = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->bool_type, value, input});
state->builder.AddFunction(
{OpSelect, state->uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, dst, bits});
const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.bool_type, value, input});
state.builder.AddFunction(
{OpSelect, state.uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, dst, bits});
}
void EmitPixelInputRegisters(EmitterState* state) {
if (state->stage != ShaderType::Pixel || state->pixel_input_info == nullptr) {
void EmitPixelInputRegisters(EmitterState& state) {
if (state.stage != ShaderType::Pixel || state.pixel_input_info == nullptr) {
return;
}
const auto* ps = state->pixel_input_info;
const auto* ps = state.pixel_input_info;
uint32_t reg = ps->ps_system_input_base;
if (ps->HasPositionInput()) {
if (ps->ps_pos_x) {
@@ -334,37 +334,37 @@ void EmitPixelInputRegisters(EmitterState* state) {
}
}
uint32_t EmitInputScalarU32(EmitterState* state, IR::StageInputKind kind) {
const auto variable = InputVariableForKind(*state, kind);
uint32_t EmitInputScalarU32(EmitterState& state, IR::StageInputKind kind) {
const auto variable = InputVariableForKind(state, kind);
if (variable == 0) {
return ConstantU32(state, 0);
}
const auto raw = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->int_type, raw, variable});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, raw});
const auto raw = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.int_type, raw, variable});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
return bits;
}
void EmitVertexInputRegisters(EmitterState* state) {
if (state->stage != ShaderType::Vertex) {
void EmitVertexInputRegisters(EmitterState& state) {
if (state.stage != ShaderType::Vertex) {
return;
}
const auto vertex_index = PointerForRegister(*state, {IR::RegisterFile::Vector, 5});
const auto vertex_index = PointerForRegister(state, {IR::RegisterFile::Vector, 5});
if (vertex_index != 0) {
state->builder.AddFunction(
state.builder.AddFunction(
{OpStore, vertex_index, EmitInputScalarU32(state, IR::StageInputKind::VertexIndex)});
}
const auto instance_index = PointerForRegister(*state, {IR::RegisterFile::Vector, 8});
const auto instance_index = PointerForRegister(state, {IR::RegisterFile::Vector, 8});
if (instance_index != 0) {
state->builder.AddFunction({OpStore, instance_index,
state.builder.AddFunction({OpStore, instance_index,
EmitInputScalarU32(state, IR::StageInputKind::InstanceIndex)});
}
}
void EmitInstruction(EmitterState* state, const IR::Instruction& inst) {
void EmitInstruction(EmitterState& state, const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::ControlNop: EmitControlNop(state, inst); break;
case IR::Opcode::Waitcnt: EmitWaitcnt(state, inst); break;
@@ -846,7 +846,7 @@ void EmitInstruction(EmitterState* state, const IR::Instruction& inst) {
case IR::Opcode::ImageSample: EmitImageSample(state, inst); break;
case IR::Opcode::ImageGather4: EmitImageGather4(state, inst); break;
case IR::Opcode::Export:
if (ExportUsesPixelValidMask(*state, inst)) {
if (ExportUsesPixelValidMask(state, inst)) {
EmitUpdatePixelValidMask(state);
}
EmitGuardedByExec(state, [&]() { EmitExport(state, inst); });
@@ -855,16 +855,16 @@ void EmitInstruction(EmitterState* state, const IR::Instruction& inst) {
}
}
uint32_t DispatcherTargetValue(EmitterState* state, uint32_t block_id) {
return BlockLabel(*state, block_id) != 0 ? block_id : UINT32_MAX;
uint32_t DispatcherTargetValue(EmitterState& state, uint32_t block_id) {
return BlockLabel(state, block_id) != 0 ? block_id : UINT32_MAX;
}
void EmitDispatcherStoreTarget(EmitterState* state, uint32_t block_id) {
state->builder.AddFunction({OpStore, state->dispatch_pc_variable,
void EmitDispatcherStoreTarget(EmitterState& state, uint32_t block_id) {
state.builder.AddFunction({OpStore, state.dispatch_pc_variable,
ConstantU32(state, DispatcherTargetValue(state, block_id))});
}
void EmitDispatcherExit(EmitterState* state);
void EmitDispatcherExit(EmitterState& state);
bool RegisterForScalarCode(uint32_t code, IR::Register& reg) {
if (code < 106u) {
@@ -881,21 +881,21 @@ bool RegisterForScalarCode(uint32_t code, IR::Register& reg) {
}
}
bool EmitDispatcherLoadScalarCode(EmitterState* state, uint32_t code, uint32_t& value) {
bool EmitDispatcherLoadScalarCode(EmitterState& state, uint32_t code, uint32_t& value) {
IR::Register reg;
if (!RegisterForScalarCode(code, reg)) {
return false;
}
const auto pointer = PointerForRegister(*state, reg);
const auto pointer = PointerForRegister(state, reg);
if (pointer == 0) {
return false;
}
value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return true;
}
void EmitDispatcherStoreSelectorTarget(EmitterState* state, const CFG::Terminator& term) {
void EmitDispatcherStoreSelectorTarget(EmitterState& state, const CFG::Terminator& term) {
if (term.indirect_selector_code == UINT32_MAX || term.indirect_selector_values.empty() ||
term.indirect_selector_values.size() != term.indirect_selector_targets.size()) {
EmitDispatcherExit(state);
@@ -910,21 +910,21 @@ void EmitDispatcherStoreSelectorTarget(EmitterState* state, const CFG::Terminato
uint32_t selected = ConstantU32(state, UINT32_MAX);
for (uint32_t i = 0; i < term.indirect_selector_values.size(); i++) {
const auto match = state->builder.AllocateId();
state->builder.AddFunction({OpIEqual, state->bool_type, match, selector_value,
const auto match = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, state.bool_type, match, selector_value,
ConstantU32(state, term.indirect_selector_values[i])});
const auto next = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, next, match,
const auto next = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, next, match,
ConstantU32(state, DispatcherTargetValue(state, term.indirect_selector_targets[i])),
selected});
selected = next;
}
state->builder.AddFunction({OpStore, state->dispatch_pc_variable, selected});
state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label});
state.builder.AddFunction({OpStore, state.dispatch_pc_variable, selected});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
}
void EmitDispatcherStoreIndirectTarget(EmitterState* state, const CFG::Terminator& term) {
void EmitDispatcherStoreIndirectTarget(EmitterState& state, const CFG::Terminator& term) {
if (term.indirect_selector_code != UINT32_MAX) {
EmitDispatcherStoreSelectorTarget(state, term);
return;
@@ -936,55 +936,55 @@ void EmitDispatcherStoreIndirectTarget(EmitterState* state, const CFG::Terminato
}
const auto pointer =
PointerForRegister(*state, IR::Register {IR::RegisterFile::Scalar, term.indirect_pc_sgpr});
PointerForRegister(state, IR::Register {IR::RegisterFile::Scalar, term.indirect_pc_sgpr});
if (pointer == 0) {
EmitDispatcherExit(state);
return;
}
const auto pc_value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, pc_value, pointer});
const auto pc_value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, pc_value, pointer});
uint32_t selected = ConstantU32(state, UINT32_MAX);
const auto count = std::min(term.indirect_target_pcs.size(), term.indirect_targets.size());
for (uint32_t i = 0; i < count; i++) {
const auto match = state->builder.AllocateId();
state->builder.AddFunction({OpIEqual, state->bool_type, match, pc_value,
const auto match = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, state.bool_type, match, pc_value,
ConstantU32(state, term.indirect_target_pcs[i])});
const auto next = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, next, match,
const auto next = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, next, match,
ConstantU32(state, DispatcherTargetValue(state, term.indirect_targets[i])), selected});
selected = next;
}
state->builder.AddFunction({OpStore, state->dispatch_pc_variable, selected});
state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label});
state.builder.AddFunction({OpStore, state.dispatch_pc_variable, selected});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
}
void EmitDispatcherExit(EmitterState* state) {
state->builder.AddFunction(
{OpStore, state->dispatch_pc_variable, ConstantU32(state, UINT32_MAX)});
state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label});
void EmitDispatcherExit(EmitterState& state) {
state.builder.AddFunction(
{OpStore, state.dispatch_pc_variable, ConstantU32(state, UINT32_MAX)});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
}
void EmitDispatcherTerminator(EmitterState* state, const CFG::Terminator& term) {
void EmitDispatcherTerminator(EmitterState& state, const CFG::Terminator& term) {
switch (term.kind) {
case CFG::TerminatorKind::Branch:
if (BlockLabel(*state, term.true_block) == 0) {
if (BlockLabel(state, term.true_block) == 0) {
EmitDispatcherExit(state);
return;
}
EmitDispatcherStoreTarget(state, term.true_block);
state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
return;
case CFG::TerminatorKind::ConditionalBranch: {
const auto condition = EmitBranchCondition(state, term.condition);
const auto selected = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, selected, condition,
const auto selected = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, selected, condition,
ConstantU32(state, DispatcherTargetValue(state, term.true_block)),
ConstantU32(state, DispatcherTargetValue(state, term.false_block))});
state->builder.AddFunction({OpStore, state->dispatch_pc_variable, selected});
state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label});
state.builder.AddFunction({OpStore, state.dispatch_pc_variable, selected});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
return;
}
case CFG::TerminatorKind::IndirectBranch:
@@ -994,44 +994,44 @@ void EmitDispatcherTerminator(EmitterState* state, const CFG::Terminator& term)
}
}
void EmitDispatcherSwitch(EmitterState* state, const IR::Program& program) {
state->builder.AddFunction({OpLabel, state->dispatch_header_label});
const auto pc_value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, pc_value, state->dispatch_pc_variable});
const auto done = state->builder.AllocateId();
state->builder.AddFunction(
{OpIEqual, state->bool_type, done, pc_value, ConstantU32(state, UINT32_MAX)});
state->builder.AddFunction({OpLoopMerge, state->dispatch_merge_label,
state->dispatch_continue_label, LoopControlNone});
state->builder.AddFunction(
{OpBranchConditional, done, state->dispatch_merge_label, state->dispatch_select_label});
void EmitDispatcherSwitch(EmitterState& state, const IR::Program& program) {
state.builder.AddFunction({OpLabel, state.dispatch_header_label});
const auto pc_value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, pc_value, state.dispatch_pc_variable});
const auto done = state.builder.AllocateId();
state.builder.AddFunction(
{OpIEqual, state.bool_type, done, pc_value, ConstantU32(state, UINT32_MAX)});
state.builder.AddFunction({OpLoopMerge, state.dispatch_merge_label,
state.dispatch_continue_label, LoopControlNone});
state.builder.AddFunction(
{OpBranchConditional, done, state.dispatch_merge_label, state.dispatch_select_label});
state->builder.AddFunction({OpLabel, state->dispatch_select_label});
state->builder.AddFunction(
{OpSelectionMerge, state->dispatch_after_switch_label, SelectionControlNone});
std::vector<uint32_t> switch_words = {OpSwitch, pc_value, state->dispatch_default_label};
state.builder.AddFunction({OpLabel, state.dispatch_select_label});
state.builder.AddFunction(
{OpSelectionMerge, state.dispatch_after_switch_label, SelectionControlNone});
std::vector<uint32_t> switch_words = {OpSwitch, pc_value, state.dispatch_default_label};
for (const auto& block: program.blocks) {
if (block.id >= state->reachable_blocks.size() || !state->reachable_blocks[block.id]) {
if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) {
continue;
}
const auto label = BlockLabel(*state, block.id);
const auto label = BlockLabel(state, block.id);
if (label != 0) {
switch_words.push_back(block.id);
switch_words.push_back(label);
}
}
state->builder.AddFunction(switch_words);
state.builder.AddFunction(switch_words);
state->builder.AddFunction({OpLabel, state->dispatch_default_label});
state.builder.AddFunction({OpLabel, state.dispatch_default_label});
EmitDispatcherExit(state);
}
void EmitDispatcherBlocks(EmitterState* state, const IR::Program& program) {
void EmitDispatcherBlocks(EmitterState& state, const IR::Program& program) {
for (const auto& block: program.blocks) {
if (block.id >= state->reachable_blocks.size() || !state->reachable_blocks[block.id]) {
if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) {
continue;
}
state->builder.AddFunction({OpLabel, BlockLabel(*state, block.id)});
state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
for (const auto& inst: block.instructions) {
EmitInstruction(state, inst);
}
@@ -1039,57 +1039,57 @@ void EmitDispatcherBlocks(EmitterState* state, const IR::Program& program) {
}
}
void EmitDispatcherLoopTail(EmitterState* state) {
state->builder.AddFunction({OpLabel, state->dispatch_after_switch_label});
state->builder.AddFunction({OpBranch, state->dispatch_continue_label});
state->builder.AddFunction({OpLabel, state->dispatch_continue_label});
state->builder.AddFunction({OpBranch, state->dispatch_header_label});
state->builder.AddFunction({OpLabel, state->dispatch_merge_label});
void EmitDispatcherLoopTail(EmitterState& state) {
state.builder.AddFunction({OpLabel, state.dispatch_after_switch_label});
state.builder.AddFunction({OpBranch, state.dispatch_continue_label});
state.builder.AddFunction({OpLabel, state.dispatch_continue_label});
state.builder.AddFunction({OpBranch, state.dispatch_header_label});
state.builder.AddFunction({OpLabel, state.dispatch_merge_label});
EmitReturn(state);
}
void EmitDispatcherFunction(EmitterState* state, const IR::Program& program) {
state->builder.AddFunction({OpStore, state->dispatch_pc_variable, ConstantU32(state, 0)});
state->builder.AddFunction({OpBranch, state->dispatch_header_label});
void EmitDispatcherFunction(EmitterState& state, const IR::Program& program) {
state.builder.AddFunction({OpStore, state.dispatch_pc_variable, ConstantU32(state, 0)});
state.builder.AddFunction({OpBranch, state.dispatch_header_label});
EmitDispatcherSwitch(state, program);
EmitDispatcherBlocks(state, program);
EmitDispatcherLoopTail(state);
}
void EmitFunction(EmitterState* state, const IR::Program& program) {
state->builder.AddFunction(
{OpFunction, state->void_type, state->main_func, FunctionControlNone, state->func_type});
state->builder.AddFunction({OpLabel, state->entry_label});
void EmitFunction(EmitterState& state, const IR::Program& program) {
state.builder.AddFunction(
{OpFunction, state.void_type, state.main_func, FunctionControlNone, state.func_type});
state.builder.AddFunction({OpLabel, state.entry_label});
EmitRegisterVariables(state);
EmitComputeInputRegisters(state);
EmitPixelInputRegisters(state);
EmitVertexInputRegisters(state);
if (state->dispatcher_fallback) {
if (state.dispatcher_fallback) {
EmitDispatcherFunction(state, program);
state->builder.AddFunction({OpFunctionEnd});
state.builder.AddFunction({OpFunctionEnd});
return;
}
const auto entry_block_label = BlockLabel(*state, 0);
const auto entry_block_label = BlockLabel(state, 0);
if (entry_block_label != 0) {
state->builder.AddFunction({OpBranch, entry_block_label});
state.builder.AddFunction({OpBranch, entry_block_label});
} else {
EmitReturn(state);
}
for (const auto& block: program.blocks) {
if (block.id >= state->reachable_blocks.size() || !state->reachable_blocks[block.id]) {
if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) {
continue;
}
state->builder.AddFunction({OpLabel, BlockLabel(*state, block.id)});
state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
for (const auto& inst: block.instructions) {
EmitInstruction(state, inst);
}
EmitTerminator(state, block.terminator);
}
state->builder.AddFunction({OpFunctionEnd});
state.builder.AddFunction({OpFunctionEnd});
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterExport.cpp
+59 -59
View File
@@ -2,7 +2,7 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
uint32_t EmitExportComponentF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitExportComponentF32(EmitterState& state, const IR::Instruction& inst,
uint32_t component) {
const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
if (!enabled || component >= inst.src_count || component >= 4u) {
@@ -11,7 +11,7 @@ uint32_t EmitExportComponentF32(EmitterState* state, const IR::Instruction& inst
return EmitFloatLoad(state, inst.src[component]);
}
uint32_t EmitExportVec4F32(EmitterState* state, const IR::Instruction& inst) {
uint32_t EmitExportVec4F32(EmitterState& state, const IR::Instruction& inst) {
if (inst.export_info.compr) {
uint32_t components[4] = {
ConstantF32(state, 0u),
@@ -22,21 +22,21 @@ uint32_t EmitExportVec4F32(EmitterState* state, const IR::Instruction& inst) {
for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
pair_index++) {
const auto raw = EmitValueLoad(state, inst.src[pair_index]);
const auto unpacked = state->builder.AllocateId();
state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked,
state->glsl_std450, GlslUnpackHalf2x16, raw});
const auto unpacked = state.builder.AllocateId();
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
state.glsl_std450, GlslUnpackHalf2x16, raw});
for (uint32_t lane = 0; lane < 2u; lane++) {
const auto component = pair_index * 2u + lane;
if (((inst.export_info.en >> component) & 1u) == 0) {
continue;
}
components[component] = state->builder.AllocateId();
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, components[component], unpacked, lane});
components[component] = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, components[component], unpacked, lane});
}
}
const auto vec = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, vec,
const auto vec = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec,
components[0], components[1], components[2], components[3]});
return vec;
}
@@ -45,24 +45,24 @@ uint32_t EmitExportVec4F32(EmitterState* state, const IR::Instruction& inst) {
const auto y = EmitExportComponentF32(state, inst, 1);
const auto z = EmitExportComponentF32(state, inst, 2);
const auto w = EmitExportComponentF32(state, inst, 3);
const auto vec = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, vec, x, y, z, w});
const auto vec = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, x, y, z, w});
return vec;
}
uint32_t ApplyMrtExportMapping(EmitterState* state, const IR::Instruction& inst, uint32_t value) {
if (inst.export_info.kind != IR::ExportTargetKind::Mrt || state->pixel_input_info == nullptr ||
inst.export_info.index >= state->pixel_input_info->target_export_mapping.size()) {
uint32_t ApplyMrtExportMapping(EmitterState& state, const IR::Instruction& inst, uint32_t value) {
if (inst.export_info.kind != IR::ExportTargetKind::Mrt || state.pixel_input_info == nullptr ||
inst.export_info.index >= state.pixel_input_info->target_export_mapping.size()) {
return value;
}
const auto mapping = state->pixel_input_info->target_export_mapping[inst.export_info.index];
const auto mapping = state.pixel_input_info->target_export_mapping[inst.export_info.index];
if (mapping.IsIdentity()) {
return value;
}
const auto mapped = state->builder.AllocateId();
state->builder.AddFunction({OpVectorShuffle, state->vec4_float_type, mapped, value, value,
const auto mapped = state.builder.AllocateId();
state.builder.AddFunction({OpVectorShuffle, state.vec4_float_type, mapped, value, value,
mapping.Map(0), mapping.Map(1), mapping.Map(2), mapping.Map(3)});
return mapped;
}
@@ -76,25 +76,25 @@ bool ExportWritesData(const IR::Instruction& inst) {
}
}
void EmitMrtZExport(EmitterState* state, const IR::Instruction& inst) {
if ((inst.export_info.en & 0x1u) != 0 && state->depth_variable != 0) {
state->builder.AddFunction(
{OpStore, state->depth_variable, EmitExportComponentF32(state, inst, 0)});
void EmitMrtZExport(EmitterState& state, const IR::Instruction& inst) {
if ((inst.export_info.en & 0x1u) != 0 && state.depth_variable != 0) {
state.builder.AddFunction(
{OpStore, state.depth_variable, EmitExportComponentF32(state, inst, 0)});
}
if ((inst.export_info.en & 0x4u) != 0 && state->sample_mask_variable != 0) {
if ((inst.export_info.en & 0x4u) != 0 && state.sample_mask_variable != 0) {
const auto raw =
inst.src_count > 2u ? EmitValueLoad(state, inst.src[2]) : ConstantU32(state, 0);
const auto mask = state->builder.AllocateId();
const auto ptr = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->int_type, mask, raw});
state->builder.AddFunction({OpAccessChain, state->ptr_output_int, ptr,
state->sample_mask_variable, ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, ptr, mask});
const auto mask = state.builder.AllocateId();
const auto ptr = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.int_type, mask, raw});
state.builder.AddFunction({OpAccessChain, state.ptr_output_int, ptr,
state.sample_mask_variable, ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, ptr, mask});
}
}
void EmitExport(EmitterState* state, const IR::Instruction& inst) {
void EmitExport(EmitterState& state, const IR::Instruction& inst) {
if (inst.export_info.kind == IR::ExportTargetKind::Null ||
inst.export_info.kind == IR::ExportTargetKind::Primitive) {
return;
@@ -109,62 +109,62 @@ void EmitExport(EmitterState* state, const IR::Instruction& inst) {
return;
}
const auto variable = OutputVariableForExport(*state, inst.export_info);
const auto variable = OutputVariableForExport(state, inst.export_info);
if (variable == 0) {
return;
}
const auto value = ApplyMrtExportMapping(state, inst, EmitExportVec4F32(state, inst));
if (inst.export_info.kind == IR::ExportTargetKind::Position) {
const auto pointer = state->builder.AllocateId();
state->builder.AddFunction({OpAccessChain, state->ptr_output_vec4_float, pointer, variable,
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_output_vec4_float, pointer, variable,
ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, pointer, value});
state.builder.AddFunction({OpStore, pointer, value});
return;
}
state->builder.AddFunction({OpStore, variable, value});
state.builder.AddFunction({OpStore, variable, value});
}
bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst) {
return state.stage == ShaderType::Pixel && inst.export_info.vm && state.needs_pixel_valid_mask;
}
void EmitKillIfBoolFalse(EmitterState* state, uint32_t active) {
const auto kill_label = state->builder.AllocateId();
const auto merge_label = state->builder.AllocateId();
const auto inactive = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state->builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
state->builder.AddFunction({OpLabel, kill_label});
state->builder.AddFunction({OpKill});
state->builder.AddFunction({OpLabel, merge_label});
void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
const auto kill_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId();
const auto inactive = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
state.builder.AddFunction({OpLabel, kill_label});
state.builder.AddFunction({OpKill});
state.builder.AddFunction({OpLabel, merge_label});
}
void EmitUpdatePixelValidMask(EmitterState* state) {
if (state->pixel_valid_mask_variable == 0) {
void EmitUpdatePixelValidMask(EmitterState& state) {
if (state.pixel_valid_mask_variable == 0) {
return;
}
const auto active = EmitExecActiveBool(state);
const auto value = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, state->pixel_valid_mask_variable, value});
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
}
void EmitKillIfPixelValidMaskInactive(EmitterState* state) {
if (state->pixel_valid_mask_variable == 0) {
void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
if (state.pixel_valid_mask_variable == 0) {
return;
}
const auto mask_value = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction(
{OpLoad, state->uint_type, mask_value, state->pixel_valid_mask_variable});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, mask_value, ConstantU32(state, 0)});
const auto mask_value = state.builder.AllocateId();
const auto active = state.builder.AllocateId();
state.builder.AddFunction(
{OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)});
EmitKillIfBoolFalse(state, active);
}
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterFlowOps.cpp
+158 -158
View File
@@ -2,15 +2,15 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
void EmitMoveU32(EmitterState* state, const IR::Instruction& inst) {
void EmitMoveU32(EmitterState& state, const IR::Instruction& inst) {
const auto value = EmitValueLoad(state, inst.src[0]);
EmitStoreU32(state, inst.dst, value);
}
void EmitMoveF32Bits(EmitterState* state, const IR::Instruction& inst) {
void EmitMoveF32Bits(EmitterState& state, const IR::Instruction& inst) {
const auto value = EmitFloatLoad(state, inst.src[0]);
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->uint_type, bits, value});
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
EmitStoreU32(state, inst.dst, bits);
}
@@ -24,7 +24,7 @@ uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base) {
return std::min(limit, 256u);
}
void EmitMoveRelSourceU32(EmitterState* state, const IR::Instruction& inst) {
void EmitMoveRelSourceU32(EmitterState& state, const IR::Instruction& inst) {
if (inst.src[0].kind != IR::OperandKind::Register ||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
EmitMoveU32(state, inst);
@@ -34,20 +34,20 @@ void EmitMoveRelSourceU32(EmitterState* state, const IR::Instruction& inst) {
const auto base_reg = inst.src[0].reg;
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
auto selected = EmitRegisterLoad(state, base_reg);
const auto limit = MoveRelRegisterLimit(*state, base_reg);
const auto limit = MoveRelRegisterLimit(state, base_reg);
for (uint32_t reg = base_reg.index + 1u; reg < limit; reg++) {
const auto candidate = EmitRegisterLoad(state, {IR::RegisterFile::Vector, reg});
const auto match = state->builder.AllocateId();
const auto next = state->builder.AllocateId();
state->builder.AddFunction(
{OpIEqual, state->bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
state->builder.AddFunction({OpSelect, state->uint_type, next, match, candidate, selected});
const auto match = state.builder.AllocateId();
const auto next = state.builder.AllocateId();
state.builder.AddFunction(
{OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
state.builder.AddFunction({OpSelect, state.uint_type, next, match, candidate, selected});
selected = next;
}
EmitStoreU32(state, inst.dst, selected);
}
void EmitMoveRelDestU32(EmitterState* state, const IR::Instruction& inst) {
void EmitMoveRelDestU32(EmitterState& state, const IR::Instruction& inst) {
if (inst.dst.kind != IR::OperandKind::Register ||
inst.dst.reg.file != IR::RegisterFile::Vector) {
EmitMoveU32(state, inst);
@@ -58,27 +58,27 @@ void EmitMoveRelDestU32(EmitterState* state, const IR::Instruction& inst) {
const auto value = EmitValueLoad(state, inst.src[0]);
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
const auto exec_active = EmitExecActiveBool(state);
const auto limit = MoveRelRegisterLimit(*state, base_reg);
const auto limit = MoveRelRegisterLimit(state, base_reg);
for (uint32_t reg = base_reg.index; reg < limit; reg++) {
const auto pointer = PointerForRegister(*state, {IR::RegisterFile::Vector, reg});
const auto pointer = PointerForRegister(state, {IR::RegisterFile::Vector, reg});
if (pointer == 0) {
continue;
}
const auto old_value = state->builder.AllocateId();
const auto match = state->builder.AllocateId();
const auto write = state->builder.AllocateId();
const auto merged = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, old_value, pointer});
state->builder.AddFunction(
{OpIEqual, state->bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
state->builder.AddFunction({OpLogicalAnd, state->bool_type, write, exec_active, match});
state->builder.AddFunction({OpSelect, state->uint_type, merged, write, value, old_value});
state->builder.AddFunction({OpStore, pointer, merged});
const auto old_value = state.builder.AllocateId();
const auto match = state.builder.AllocateId();
const auto write = state.builder.AllocateId();
const auto merged = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
state.builder.AddFunction(
{OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
state.builder.AddFunction({OpLogicalAnd, state.bool_type, write, exec_active, match});
state.builder.AddFunction({OpSelect, state.uint_type, merged, write, value, old_value});
state.builder.AddFunction({OpStore, pointer, merged});
}
}
void EmitMoveU64(EmitterState* state, const IR::Instruction& inst) {
if (state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
void EmitMoveU64(EmitterState& state, const IR::Instruction& inst) {
if (state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
IsMaskRegisterFile(inst.dst.reg.file)) {
EmitPerInvocationMask(state, inst.dst, EmitLaneMaskOperandActiveBool(state, inst.src[0]));
return;
@@ -89,28 +89,28 @@ void EmitMoveU64(EmitterState* state, const IR::Instruction& inst) {
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high);
}
uint32_t EmitWqmLaneU32(EmitterState* state, uint32_t src) {
uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) {
uint32_t ret = ConstantU32(state, 0);
for (uint32_t i = 0; i < 8u; i++) {
const auto mask = 0x0fu << (i * 4u);
const auto masked = state->builder.AllocateId();
const auto non_zero = state->builder.AllocateId();
const auto expanded = state->builder.AllocateId();
const auto combined = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, masked, src, ConstantU32(state, mask)});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, non_zero, masked, ConstantU32(state, 0)});
state->builder.AddFunction({OpSelect, state->uint_type, expanded, non_zero,
const auto masked = state.builder.AllocateId();
const auto non_zero = state.builder.AllocateId();
const auto expanded = state.builder.AllocateId();
const auto combined = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)});
state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero,
ConstantU32(state, mask), ConstantU32(state, 0)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, ret, expanded});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded});
ret = combined;
}
return ret;
}
void EmitWqmB64(EmitterState* state, const IR::Instruction& inst) {
if (!state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
void EmitWqmB64(EmitterState& state, const IR::Instruction& inst) {
if (!state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
@@ -120,44 +120,44 @@ void EmitWqmB64(EmitterState* state, const IR::Instruction& inst) {
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high);
return;
}
const auto ballot = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup),
EmitLaneMaskOperandActiveBool(state, inst.src[0])});
const auto low = state->builder.AllocateId();
const auto high = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, low, ballot, 0});
state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, ballot, 1});
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
const auto wqm_low = EmitWqmLaneU32(state, low);
const auto wqm_high = EmitWqmLaneU32(state, high);
const auto lane = EmitSubgroupLocalInvocationId(state);
const auto high_half = state->builder.AllocateId();
const auto mask = state->builder.AllocateId();
const auto bit_index = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
const auto hit = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, high_half, lane, ConstantU32(state, 32)});
state->builder.AddFunction({OpSelect, state->uint_type, mask, high_half, wqm_high, wqm_low});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, bit_index, lane, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), bit_index});
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
state->builder.AddFunction({OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
if (state->per_invocation_masks) {
const auto high_half = state.builder.AllocateId();
const auto mask = state.builder.AllocateId();
const auto bit_index = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
const auto hit = state.builder.AllocateId();
const auto active = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, high_half, lane, ConstantU32(state, 32)});
state.builder.AddFunction({OpSelect, state.uint_type, mask, high_half, wqm_high, wqm_low});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, bit_index, lane, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_index});
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
if (state.per_invocation_masks) {
EmitPerInvocationMask(state, inst.dst, active);
} else {
const auto result = state->builder.AllocateId();
state->builder.AddFunction({OpSelect, state->uint_type, result, active,
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, state.uint_type, result, active,
ConstantU32(state, 1), ConstantU32(state, 0)});
EmitStoreU32(state, inst.dst, result);
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ConstantU32(state, 0));
}
}
void EmitSaveexecPerInvocation(EmitterState* state, const IR::Instruction& inst) {
void EmitSaveexecPerInvocation(EmitterState& state, const IR::Instruction& inst) {
const auto old_exec = EmitExecActiveBool(state);
const auto src = EmitLaneMaskOperandActiveBool(state, inst.src[0]);
EmitPerInvocationMask(state, inst.dst, old_exec);
@@ -179,8 +179,8 @@ void EmitSaveexecPerInvocation(EmitterState* state, const IR::Instruction& inst)
EmitStoreSccBool(state, result);
}
void EmitSaveexecB32(EmitterState* state, const IR::Instruction& inst) {
if (state->per_invocation_masks) {
void EmitSaveexecB32(EmitterState& state, const IR::Instruction& inst) {
if (state.per_invocation_masks) {
EmitSaveexecPerInvocation(state, inst);
return;
}
@@ -191,29 +191,29 @@ void EmitSaveexecB32(EmitterState* state, const IR::Instruction& inst) {
uint32_t rhs_low = src_low;
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
rhs_low = state->builder.AllocateId();
state->builder.AddFunction({OpNot, state->uint_type, rhs_low, src_low});
rhs_low = state.builder.AllocateId();
state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
}
const auto new_low = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, new_low, rhs_low, old_low});
const auto new_low = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, new_low, rhs_low, old_low});
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
if (state->wave_size == 32u) {
if (state.wave_size == 32u) {
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), ConstantU32(state, 0));
}
const auto cond = state->builder.AllocateId();
const auto scc = state->builder.AllocateId();
state->builder.AddFunction(
{OpINotEqual, state->bool_type, cond, new_low, ConstantU32(state, 0)});
state->builder.AddFunction(
{OpSelect, state->uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
const auto cond = state.builder.AllocateId();
const auto scc = state.builder.AllocateId();
state.builder.AddFunction(
{OpINotEqual, state.bool_type, cond, new_low, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
EmitStoreU32(state, SccOperand(), scc);
}
void EmitSaveexecB64(EmitterState* state, const IR::Instruction& inst) {
if (state->per_invocation_masks) {
void EmitSaveexecB64(EmitterState& state, const IR::Instruction& inst) {
if (state.per_invocation_masks) {
EmitSaveexecPerInvocation(state, inst);
return;
}
@@ -231,150 +231,150 @@ void EmitSaveexecB64(EmitterState* state, const IR::Instruction& inst) {
uint32_t rhs_high = src_high;
uint32_t opcode = OpBitwiseAnd;
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
rhs_low = state->builder.AllocateId();
rhs_high = state->builder.AllocateId();
state->builder.AddFunction({OpNot, state->uint_type, rhs_low, src_low});
state->builder.AddFunction({OpNot, state->uint_type, rhs_high, src_high});
rhs_low = state.builder.AllocateId();
rhs_high = state.builder.AllocateId();
state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
state.builder.AddFunction({OpNot, state.uint_type, rhs_high, src_high});
}
if (inst.saveexec_mode == IR::SaveexecMode::Orn2) {
lhs_low = state->builder.AllocateId();
lhs_high = state->builder.AllocateId();
state->builder.AddFunction({OpNot, state->uint_type, lhs_low, old_low});
state->builder.AddFunction({OpNot, state->uint_type, lhs_high, old_high});
lhs_low = state.builder.AllocateId();
lhs_high = state.builder.AllocateId();
state.builder.AddFunction({OpNot, state.uint_type, lhs_low, old_low});
state.builder.AddFunction({OpNot, state.uint_type, lhs_high, old_high});
opcode = OpBitwiseOr;
}
const auto new_low = state->builder.AllocateId();
const auto new_high = state->builder.AllocateId();
state->builder.AddFunction({opcode, state->uint_type, new_low, lhs_low, rhs_low});
state->builder.AddFunction({opcode, state->uint_type, new_high, lhs_high, rhs_high});
const auto new_low = state.builder.AllocateId();
const auto new_high = state.builder.AllocateId();
state.builder.AddFunction({opcode, state.uint_type, new_low, lhs_low, rhs_low});
state.builder.AddFunction({opcode, state.uint_type, new_high, lhs_high, rhs_high});
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), new_high);
const auto active_new_high = state->wave_size == 32u ? ConstantU32(state, 0) : new_high;
const auto mask = state->builder.AllocateId();
const auto cond = state->builder.AllocateId();
const auto scc = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseOr, state->uint_type, mask, new_low, active_new_high});
state->builder.AddFunction({OpINotEqual, state->bool_type, cond, mask, ConstantU32(state, 0)});
state->builder.AddFunction(
{OpSelect, state->uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
const auto active_new_high = state.wave_size == 32u ? ConstantU32(state, 0) : new_high;
const auto mask = state.builder.AllocateId();
const auto cond = state.builder.AllocateId();
const auto scc = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseOr, state.uint_type, mask, new_low, active_new_high});
state.builder.AddFunction({OpINotEqual, state.bool_type, cond, mask, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
EmitStoreU32(state, SccOperand(), scc);
}
void EmitReadFirstLaneU32(EmitterState* state, const IR::Instruction& inst) {
void EmitReadFirstLaneU32(EmitterState& state, const IR::Instruction& inst) {
const auto src = EmitValueLoad(state, inst.src[0]);
const auto active = EmitExecActiveBool(state);
const auto ballot = state->builder.AllocateId();
const auto first_lane = state->builder.AllocateId();
const auto first_value = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
const auto ballot = state.builder.AllocateId();
const auto first_lane = state.builder.AllocateId();
const auto first_value = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), active});
state->builder.AddFunction({OpGroupNonUniformBallotFindLSB, state->uint_type, first_lane,
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first_lane,
ConstantU32(state, ScopeSubgroup), ballot});
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, first_value,
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, first_value,
ConstantU32(state, ScopeSubgroup), src, first_lane});
EmitStoreU32(state, inst.dst, first_value);
}
uint32_t EmitLaneIndex(EmitterState* state, const IR::Operand& operand) {
const auto lane = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, lane, EmitValueLoad(state, operand),
uint32_t EmitLaneIndex(EmitterState& state, const IR::Operand& operand) {
const auto lane = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, EmitValueLoad(state, operand),
ConstantU32(state, 63)});
return lane;
}
void EmitReadLaneU32(EmitterState* state, const IR::Instruction& inst) {
void EmitReadLaneU32(EmitterState& state, const IR::Instruction& inst) {
const auto src = EmitValueLoad(state, inst.src[0]);
const auto lane = EmitLaneIndex(state, inst.src[1]);
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, value,
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value,
ConstantU32(state, ScopeSubgroup), src, lane});
EmitStoreU32(state, inst.dst, value);
}
void EmitWriteLaneU32(EmitterState* state, const IR::Instruction& inst) {
void EmitWriteLaneU32(EmitterState& state, const IR::Instruction& inst) {
const auto value = EmitValueLoad(state, inst.src[0]);
const auto lane = EmitLaneIndex(state, inst.src[1]);
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto hit = state->builder.AllocateId();
const auto hit = state.builder.AllocateId();
const auto pointer =
inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(*state, inst.dst.reg) : 0;
const auto old = pointer != 0 ? state->builder.AllocateId() : ConstantU32(state, 0);
const auto merged = state->builder.AllocateId();
inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(state, inst.dst.reg) : 0;
const auto old = pointer != 0 ? state.builder.AllocateId() : ConstantU32(state, 0);
const auto merged = state.builder.AllocateId();
if (pointer != 0) {
state->builder.AddFunction({OpLoad, state->uint_type, old, pointer});
state.builder.AddFunction({OpLoad, state.uint_type, old, pointer});
}
state->builder.AddFunction({OpIEqual, state->bool_type, hit, subid, lane});
state->builder.AddFunction({OpSelect, state->uint_type, merged, hit, value, old});
state.builder.AddFunction({OpIEqual, state.bool_type, hit, subid, lane});
state.builder.AddFunction({OpSelect, state.uint_type, merged, hit, value, old});
if (pointer != 0) {
// RDNA2 V_WRITELANE_B32 ignores EXEC, unlike ordinary VGPR destination writes.
state->builder.AddFunction({OpStore, pointer, merged});
state.builder.AddFunction({OpStore, pointer, merged});
}
}
void EmitPermlaneB32(EmitterState* state, const IR::Instruction& inst, bool x16) {
void EmitPermlaneB32(EmitterState& state, const IR::Instruction& inst, bool x16) {
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto row = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto lane8 = state->builder.AllocateId();
const auto shift = state->builder.AllocateId();
const auto upper = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
const auto index0 = state->builder.AllocateId();
const auto index1 = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
const auto shuffled = state->builder.AllocateId();
const auto row = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto lane8 = state.builder.AllocateId();
const auto shift = state.builder.AllocateId();
const auto upper = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
const auto index0 = state.builder.AllocateId();
const auto index1 = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
const auto shuffled = state.builder.AllocateId();
const auto value = EmitValueLoad(state, inst.src[0]);
const auto src1 = EmitValueLoad(state, inst.src[1]);
const auto src2 = EmitValueLoad(state, inst.src[2]);
uint32_t row_value = row;
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
if (x16) {
row_value = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseXor, state->uint_type, row_value, row, ConstantU32(state, 16)});
row_value = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
}
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane8, lane, ConstantU32(state, 7)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, shift, lane8, ConstantU32(state, 2)});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, upper, lane, ConstantU32(state, 8)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, upper, src2, src1});
state->builder.AddFunction({OpShiftRightLogical, state->uint_type, index0, selected, shift});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, index1, index0, ConstantU32(state, 15)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row_value, index1});
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, shuffled,
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
state.builder.AddFunction({OpSelect, state.uint_type, selected, upper, src2, src1});
state.builder.AddFunction({OpShiftRightLogical, state.uint_type, index0, selected, shift});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, index1, index0, ConstantU32(state, 15)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index1});
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
ConstantU32(state, ScopeSubgroup), value, target});
uint32_t ret = shuffled;
if (!inst.dst.op_sel) {
const auto source_active = EmitLaneIndexActiveBool(state, target);
ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, source_active, shuffled, ConstantU32(state, 0)});
ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, source_active, shuffled, ConstantU32(state, 0)});
}
EmitStoreU32(state, inst.dst, ret);
}
void EmitControlNop(EmitterState* state, const IR::Instruction& inst) {
void EmitControlNop(EmitterState& state, const IR::Instruction& inst) {
(void)state;
(void)inst;
}
void EmitWaitcnt(EmitterState* state, const IR::Instruction& inst) {
void EmitWaitcnt(EmitterState& state, const IR::Instruction& inst) {
(void)state;
(void)inst;
}
void EmitBarrier(EmitterState* state, const IR::Instruction& inst) {
void EmitBarrier(EmitterState& state, const IR::Instruction& inst) {
(void)inst;
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
state->builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)});
}
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageHelpers.cpp
+86 -86
View File
@@ -33,25 +33,25 @@ ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKi
return layout;
}
uint32_t EmitImageCoordF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout, ImageViewKind view) {
const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord);
const auto y = inst.memory.image_address_components > layout.coord + 1u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 1u)
: EmitZeroF32(state);
const auto coord = state->builder.AllocateId();
const auto coord = state.builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > layout.coord + 2u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 2u)
: EmitZeroF32(state);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_float_type, coord, x, y, z});
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, coord, x, y});
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
}
return coord;
}
uint32_t EmitImageLodF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero) ||
layout.lod == NoImageComponent || inst.memory.image_address_components <= layout.lod) {
@@ -60,7 +60,7 @@ uint32_t EmitImageLodF32(EmitterState* state, const IR::Instruction& inst,
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.lod);
}
uint32_t EmitImageDrefF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageDrefF32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (layout.dref == NoImageComponent || inst.memory.image_address_components <= layout.dref) {
return EmitZeroF32(state);
@@ -68,7 +68,7 @@ uint32_t EmitImageDrefF32(EmitterState* state, const IR::Instruction& inst,
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.dref);
}
uint32_t EmitImageBiasF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageBiasF32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (layout.bias == NoImageComponent || inst.memory.image_address_components <= layout.bias) {
return EmitZeroF32(state);
@@ -76,7 +76,7 @@ uint32_t EmitImageBiasF32(EmitterState* state, const IR::Instruction& inst,
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.bias);
}
uint32_t EmitImageGradientF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst,
uint32_t first_component) {
const auto x = inst.memory.image_address_components > first_component
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component)
@@ -84,37 +84,37 @@ uint32_t EmitImageGradientF32(EmitterState* state, const IR::Instruction& inst,
const auto y = inst.memory.image_address_components > first_component + 1u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 1u)
: EmitZeroF32(state);
const auto grad = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, grad, x, y});
const auto grad = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, grad, x, y});
return grad;
}
uint32_t EmitImagePackedOffset2I32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImagePackedOffset2I32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (layout.offset == NoImageComponent ||
inst.memory.image_address_components <= layout.offset) {
const auto zero = ConstantI32(state, 0);
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec2_int_type, ret, zero, zero});
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec2_int_type, ret, zero, zero});
return ret;
}
const auto packed_bits = EmitImageAddressValueLoad(state, inst, inst.src[0], layout.offset);
const auto packed_i32 = state->builder.AllocateId();
const auto offset_x = state->builder.AllocateId();
const auto offset_y = state->builder.AllocateId();
const auto offset = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->int_type, packed_i32, packed_bits});
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, offset_x, packed_i32,
const auto packed_i32 = state.builder.AllocateId();
const auto offset_x = state.builder.AllocateId();
const auto offset_y = state.builder.AllocateId();
const auto offset = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.int_type, packed_i32, packed_bits});
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_x, packed_i32,
ConstantI32(state, 0), ConstantI32(state, 6)});
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, offset_y, packed_i32,
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_y, packed_i32,
ConstantI32(state, 8), ConstantI32(state, 6)});
state->builder.AddFunction(
{OpCompositeConstruct, state->vec2_int_type, offset, offset_x, offset_y});
state.builder.AddFunction(
{OpCompositeConstruct, state.vec2_int_type, offset, offset_x, offset_y});
return offset;
}
uint32_t EmitImageOffsetCoordF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageOffsetCoordF32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout, uint32_t sampled_image,
uint32_t coord, ImageViewKind view) {
if (!HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
@@ -122,92 +122,92 @@ uint32_t EmitImageOffsetCoordF32(EmitterState* state, const IR::Instruction& ins
}
const auto offset_i32 = EmitImagePackedOffset2I32(state, inst, layout);
const auto offset_f32 = state->builder.AllocateId();
const auto image = state->builder.AllocateId();
const auto size_u32 = state->builder.AllocateId();
const auto offset_f32 = state.builder.AllocateId();
const auto image = state.builder.AllocateId();
const auto size_u32 = state.builder.AllocateId();
const auto size_xy_u32 =
ImageViewCoordinateComponents(view) == 3u ? state->builder.AllocateId() : size_u32;
const auto size_f32 = state->builder.AllocateId();
const auto normalized = state->builder.AllocateId();
const auto adjusted = state->builder.AllocateId();
state->builder.AddFunction({OpConvertSToF, state->vec2_float_type, offset_f32, offset_i32});
state->builder.AddFunction(
{OpImage, ImageViewImageType(*state, view, inst.memory.kind == IR::ResourceKind::ImageUint),
ImageViewCoordinateComponents(view) == 3u ? state.builder.AllocateId() : size_u32;
const auto size_f32 = state.builder.AllocateId();
const auto normalized = state.builder.AllocateId();
const auto adjusted = state.builder.AllocateId();
state.builder.AddFunction({OpConvertSToF, state.vec2_float_type, offset_f32, offset_i32});
state.builder.AddFunction(
{OpImage, ImageViewImageType(state, view, inst.memory.kind == IR::ResourceKind::ImageUint),
image, sampled_image});
state->builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(*state, view), size_u32,
state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(state, view), size_u32,
image, ConstantU32(state, 0)});
if (ImageViewCoordinateComponents(view) == 3u) {
const auto width = state->builder.AllocateId();
const auto height = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, width, size_u32, 0});
state->builder.AddFunction({OpCompositeExtract, state->uint_type, height, size_u32, 1});
state->builder.AddFunction(
{OpCompositeConstruct, state->vec2_uint_type, size_xy_u32, width, height});
const auto width = state.builder.AllocateId();
const auto height = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, width, size_u32, 0});
state.builder.AddFunction({OpCompositeExtract, state.uint_type, height, size_u32, 1});
state.builder.AddFunction(
{OpCompositeConstruct, state.vec2_uint_type, size_xy_u32, width, height});
}
state->builder.AddFunction({OpConvertUToF, state->vec2_float_type, size_f32, size_xy_u32});
state->builder.AddFunction({OpFDiv, state->vec2_float_type, normalized, offset_f32, size_f32});
state.builder.AddFunction({OpConvertUToF, state.vec2_float_type, size_f32, size_xy_u32});
state.builder.AddFunction({OpFDiv, state.vec2_float_type, normalized, offset_f32, size_f32});
if (ImageViewCoordinateComponents(view) == 2u) {
state->builder.AddFunction({OpFAdd, state->vec2_float_type, adjusted, coord, normalized});
state.builder.AddFunction({OpFAdd, state.vec2_float_type, adjusted, coord, normalized});
return adjusted;
}
const auto x = state->builder.AllocateId();
const auto y = state->builder.AllocateId();
const auto z = state->builder.AllocateId();
const auto normalized_x = state->builder.AllocateId();
const auto normalized_y = state->builder.AllocateId();
const auto adjusted_x = state->builder.AllocateId();
const auto adjusted_y = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->float_type, x, coord, 0});
state->builder.AddFunction({OpCompositeExtract, state->float_type, y, coord, 1});
state->builder.AddFunction({OpCompositeExtract, state->float_type, z, coord, 2});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, normalized_x, normalized, 0});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, normalized_y, normalized, 1});
state->builder.AddFunction({OpFAdd, state->float_type, adjusted_x, x, normalized_x});
state->builder.AddFunction({OpFAdd, state->float_type, adjusted_y, y, normalized_y});
state->builder.AddFunction(
{OpCompositeConstruct, state->vec3_float_type, adjusted, adjusted_x, adjusted_y, z});
const auto x = state.builder.AllocateId();
const auto y = state.builder.AllocateId();
const auto z = state.builder.AllocateId();
const auto normalized_x = state.builder.AllocateId();
const auto normalized_y = state.builder.AllocateId();
const auto adjusted_x = state.builder.AllocateId();
const auto adjusted_y = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.float_type, x, coord, 0});
state.builder.AddFunction({OpCompositeExtract, state.float_type, y, coord, 1});
state.builder.AddFunction({OpCompositeExtract, state.float_type, z, coord, 2});
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, normalized_x, normalized, 0});
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, normalized_y, normalized, 1});
state.builder.AddFunction({OpFAdd, state.float_type, adjusted_x, x, normalized_x});
state.builder.AddFunction({OpFAdd, state.float_type, adjusted_y, y, normalized_y});
state.builder.AddFunction(
{OpCompositeConstruct, state.vec3_float_type, adjusted, adjusted_x, adjusted_y, z});
return adjusted;
}
uint32_t EmitImageCoordU32(EmitterState* state, const IR::Instruction& inst, ImageViewKind view) {
uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst, ImageViewKind view) {
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[1], 0);
const auto y = inst.memory.image_address_components > 1u
? EmitImageAddressValueLoad(state, inst, inst.src[1], 1)
: ConstantU32(state, 0);
const auto coord = state->builder.AllocateId();
const auto coord = state.builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > 2u
? EmitImageAddressValueLoad(state, inst, inst.src[1], 2)
: ConstantU32(state, 0);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_uint_type, coord, x, y, z});
state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_uint_type, coord, x, y});
state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
}
return coord;
}
uint32_t EmitImageLoadCoordU32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst,
ImageViewKind view) {
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[0], 0);
const auto y = inst.memory.image_address_components > 1u
? EmitImageAddressValueLoad(state, inst, inst.src[0], 1)
: ConstantU32(state, 0);
const auto coord = state->builder.AllocateId();
const auto coord = state.builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > 2u
? EmitImageAddressValueLoad(state, inst, inst.src[0], 2)
: ConstantU32(state, 0);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_uint_type, coord, x, y, z});
state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_uint_type, coord, x, y});
state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
}
return coord;
}
uint32_t EmitImageMipLodU32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst,
const IR::Operand& address, ImageViewKind view) {
if (!inst.memory.image_has_mip || inst.memory.image_address_components == 0u) {
return ConstantU32(state, 0);
@@ -219,20 +219,20 @@ uint32_t EmitImageMipLodU32(EmitterState* state, const IR::Instruction& inst,
return EmitImageAddressValueLoad(state, inst, address, lod_component);
}
uint32_t EmitImageQueryCoordF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst,
ImageViewKind view) {
const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], 0);
const auto y = inst.memory.image_address_components > 1u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], 1)
: EmitZeroF32(state);
const auto coord = state->builder.AllocateId();
const auto coord = state.builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > 2u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], 2)
: EmitZeroF32(state);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_float_type, coord, x, y, z});
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, coord, x, y});
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
}
return coord;
}
@@ -245,7 +245,7 @@ uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
return index;
}
uint32_t EmitImageStoreComponentF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageStoreComponentF32(EmitterState& state, const IR::Instruction& inst,
uint32_t component) {
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
if (((dmask >> component) & 1u) == 0) {
@@ -265,14 +265,14 @@ uint32_t InverseStorageSwizzleComponent(uint32_t swizzle, uint32_t component) {
}
uint32_t StorageImageSwizzle(const EmitterState& state, const IR::Instruction& inst) {
if (state.program == nullptr || inst.memory.resource >= state.program->info.images.size()) {
if (inst.memory.resource >= state.program.info.images.size()) {
EXIT("storage image instruction has no specialized swizzle\n");
}
return state.program->info.images[inst.memory.resource].storage_swizzle;
return state.program.info.images[inst.memory.resource].storage_swizzle;
}
uint32_t EmitImageStoreTexelF32(EmitterState* state, const IR::Instruction& inst) {
const auto swizzle = StorageImageSwizzle(*state, inst);
uint32_t EmitImageStoreTexelF32(EmitterState& state, const IR::Instruction& inst) {
const auto swizzle = StorageImageSwizzle(state, inst);
const auto component_value = [&](uint32_t component) {
const auto source = InverseStorageSwizzleComponent(swizzle, component);
return source < 4u ? EmitImageStoreComponentF32(state, inst, source) : EmitZeroF32(state);
@@ -281,12 +281,12 @@ uint32_t EmitImageStoreTexelF32(EmitterState* state, const IR::Instruction& inst
const auto y = component_value(1);
const auto z = component_value(2);
const auto w = component_value(3);
const auto texel = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, texel, x, y, z, w});
const auto texel = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, texel, x, y, z, w});
return texel;
}
uint32_t EmitImageStoreComponentU32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageStoreComponentU32(EmitterState& state, const IR::Instruction& inst,
uint32_t component) {
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
if (((dmask >> component) & 1u) == 0) {
@@ -295,8 +295,8 @@ uint32_t EmitImageStoreComponentU32(EmitterState* state, const IR::Instruction&
return EmitSequentialValueLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
}
uint32_t EmitImageStoreTexelU32(EmitterState* state, const IR::Instruction& inst) {
const auto swizzle = StorageImageSwizzle(*state, inst);
uint32_t EmitImageStoreTexelU32(EmitterState& state, const IR::Instruction& inst) {
const auto swizzle = StorageImageSwizzle(state, inst);
const auto component_value = [&](uint32_t component) {
const auto source = InverseStorageSwizzleComponent(swizzle, component);
return source < 4u ? EmitImageStoreComponentU32(state, inst, source)
@@ -306,8 +306,8 @@ uint32_t EmitImageStoreTexelU32(EmitterState* state, const IR::Instruction& inst
const auto y = component_value(1);
const auto z = component_value(2);
const auto w = component_value(3);
const auto texel = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_uint_type, texel, x, y, z, w});
const auto texel = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, texel, x, y, z, w});
return texel;
}
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageOps.cpp
+106 -115
View File
@@ -4,54 +4,54 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
uint32_t ConstantVec2I32(EmitterState* state, int32_t x, int32_t y) {
const auto value = state->builder.AllocateId();
state->builder.AddType({OpConstantComposite, state->vec2_int_type, value, ConstantI32(state, x),
ConstantI32(state, y)});
uint32_t ConstantVec2I32(EmitterState& state, int32_t x, int32_t y) {
const auto value = state.builder.AllocateId();
state.builder.AddType({OpConstantComposite, state.vec2_int_type, value, ConstantI32(state, x),
ConstantI32(state, y)});
return value;
}
uint32_t ConstantImageGatherHorizontalOffsets(EmitterState* state) {
uint32_t ConstantImageGatherHorizontalOffsets(EmitterState& state) {
const auto offset0 = ConstantVec2I32(state, -1, 0);
const auto offset1 = ConstantVec2I32(state, 0, 0);
const auto offset2 = ConstantVec2I32(state, 1, 0);
const auto offset3 = ConstantVec2I32(state, 2, 0);
const auto type = state->builder.AllocateId();
state->builder.AddType({OpTypeArray, type, state->vec2_int_type, ConstantU32(state, 4)});
const auto value = state->builder.AllocateId();
state->builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3});
const auto type = state.builder.AllocateId();
state.builder.AddType({OpTypeArray, type, state.vec2_int_type, ConstantU32(state, 4)});
const auto value = state.builder.AllocateId();
state.builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3});
return value;
}
uint32_t LoadStorageImageDescriptorAtIndex(EmitterState* state, uint32_t resource,
uint32_t LoadStorageImageDescriptorAtIndex(EmitterState& state, uint32_t resource,
uint32_t array_index, bool uint_image,
ImageViewKind view) {
uint32_t pointer_type =
uint_image ? state->ptr_uniform_storage_image_uint : state->ptr_uniform_storage_image;
uint_image ? state.ptr_uniform_storage_image_uint : state.ptr_uniform_storage_image;
uint32_t variable =
uint_image ? state->storage_image_uint_variable : state->storage_image_variable;
uint32_t image_type = uint_image ? state->storage_image_uint_type : state->storage_image_type;
uint_image ? state.storage_image_uint_variable : state.storage_image_variable;
uint32_t image_type = uint_image ? state.storage_image_uint_type : state.storage_image_type;
IR::DescriptorBindingKind kind = uint_image ? IR::DescriptorBindingKind::StorageUint2D
: IR::DescriptorBindingKind::Storage2D;
switch (view) {
case ImageViewKind::Dim2D: break;
case ImageViewKind::Dim2DArray:
pointer_type = uint_image ? state->ptr_uniform_storage_image_uint_2d_array
: state->ptr_uniform_storage_image_2d_array;
variable = uint_image ? state->storage_image_uint_2d_array_variable
: state->storage_image_2d_array_variable;
image_type = uint_image ? state->storage_image_uint_2d_array_type
: state->storage_image_2d_array_type;
pointer_type = uint_image ? state.ptr_uniform_storage_image_uint_2d_array
: state.ptr_uniform_storage_image_2d_array;
variable = uint_image ? state.storage_image_uint_2d_array_variable
: state.storage_image_2d_array_variable;
image_type = uint_image ? state.storage_image_uint_2d_array_type
: state.storage_image_2d_array_type;
kind = uint_image ? IR::DescriptorBindingKind::StorageUint2DArray
: IR::DescriptorBindingKind::Storage2DArray;
break;
case ImageViewKind::Dim3D:
pointer_type = uint_image ? state->ptr_uniform_storage_image_uint_3d
: state->ptr_uniform_storage_image_3d;
variable = uint_image ? state->storage_image_uint_3d_variable
: state->storage_image_3d_variable;
pointer_type = uint_image ? state.ptr_uniform_storage_image_uint_3d
: state.ptr_uniform_storage_image_3d;
variable =
uint_image ? state.storage_image_uint_3d_variable : state.storage_image_3d_variable;
image_type =
uint_image ? state->storage_image_uint_3d_type : state->storage_image_3d_type;
uint_image ? state.storage_image_uint_3d_type : state.storage_image_3d_type;
kind = uint_image ? IR::DescriptorBindingKind::StorageUint3D
: IR::DescriptorBindingKind::Storage3D;
break;
@@ -59,50 +59,50 @@ uint32_t LoadStorageImageDescriptorAtIndex(EmitterState* state, uint32_t resourc
const auto pointer =
DescriptorElementPointer(state, pointer_type, variable, array_index, kind, resource,
"storage image descriptor array was not emitted");
const auto image = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, image_type, image, pointer});
const auto image = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, image_type, image, pointer});
return image;
}
} // namespace
uint32_t EmitImageGetResinfoComponent(EmitterState* state, uint32_t image, uint32_t size,
uint32_t EmitImageGetResinfoComponent(EmitterState& state, uint32_t image, uint32_t size,
uint32_t component_index, ImageViewKind view) {
switch (component_index) {
case 0: {
const auto width = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, width, size, 0});
const auto width = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, width, size, 0});
return width;
}
case 1: {
const auto height = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, height, size, 1});
const auto height = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, height, size, 1});
return height;
}
case 2:
if (ImageViewCoordinateComponents(view) == 3u) {
const auto depth = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, depth, size, 2});
const auto depth = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, depth, size, 2});
return depth;
}
return ConstantU32(state, 1);
case 3: {
const auto levels = state->builder.AllocateId();
state->builder.AddFunction({OpImageQueryLevels, state->uint_type, levels, image});
const auto levels = state.builder.AllocateId();
state.builder.AddFunction({OpImageQueryLevels, state.uint_type, levels, image});
return levels;
}
default: return ConstantU32(state, 0);
}
}
void EmitImageGetResinfo(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
void EmitImageGetResinfo(EmitterState& state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
const auto mip_level = EmitValueLoad(state, inst.src[0]);
const auto size = state->builder.AllocateId();
state->builder.AddFunction(
{OpImageQuerySizeLod, ImageViewSizeType(*state, view), size, image, mip_level});
const auto size = state.builder.AllocateId();
state.builder.AddFunction(
{OpImageQuerySizeLod, ImageViewSizeType(state, view), size, image, mip_level});
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
uint32_t dst_index = 0;
@@ -116,13 +116,13 @@ void EmitImageGetResinfo(EmitterState* state, const IR::Instruction& inst) {
}
}
void EmitImageGetLod(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
void EmitImageGetLod(EmitterState& state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
const auto lod = state->builder.AllocateId();
state->builder.AddFunction({OpImageQueryLod, state->vec2_float_type, lod, sampled_image,
EmitImageQueryCoordF32(state, inst, view)});
const auto lod = state.builder.AllocateId();
state.builder.AddFunction({OpImageQueryLod, state.vec2_float_type, lod, sampled_image,
EmitImageQueryCoordF32(state, inst, view)});
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
uint32_t dst_index = 0;
@@ -130,25 +130,25 @@ void EmitImageGetLod(EmitterState* state, const IR::Instruction& inst) {
if (((dmask >> component_index) & 1u) == 0) {
continue;
}
const auto component = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, component, lod, component_index});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
const auto component = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, component, lod, component_index});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
}
}
void EmitImageLoad(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
void EmitImageLoad(EmitterState& state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
const auto color = state->builder.AllocateId();
state->builder.AddFunction(
{OpImageFetch, integer ? state->vec4_uint_type : state->vec4_float_type, color, image,
EmitImageLoadCoordU32(state, inst, view), ImageOperandsLodMask,
EmitImageMipLodU32(state, inst, inst.src[0], view)});
const auto color = state.builder.AllocateId();
state.builder.AddFunction({OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
color, image, EmitImageLoadCoordU32(state, inst, view),
ImageOperandsLodMask,
EmitImageMipLodU32(state, inst, inst.src[0], view)});
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
uint32_t dst_index = 0;
@@ -156,23 +156,22 @@ void EmitImageLoad(EmitterState* state, const IR::Instruction& inst) {
if (((dmask >> component_index) & 1u) == 0) {
continue;
}
const auto component = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract,
integer ? state->uint_type : state->float_type, component,
color, component_index});
const auto bits = integer ? component : state->builder.AllocateId();
const auto component = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
component, color, component_index});
const auto bits = integer ? component : state.builder.AllocateId();
if (!integer) {
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
}
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
}
}
void EmitImageStore(EmitterState* state, const IR::Instruction& inst) {
void EmitImageStore(EmitterState& state, const IR::Instruction& inst) {
const auto uint_image = inst.memory.kind == IR::ResourceKind::StorageImageUint;
const auto view = StorageImageViewKind(*state, inst.memory, uint_image, inst.pc);
const auto view = StorageImageViewKind(state, inst.memory, uint_image, inst.pc);
const auto binding = ResourceForDescriptor(
*state,
state,
uint_image
? (view == ImageViewKind::Dim2DArray ? IR::DescriptorBindingKind::StorageUint2DArray
: view == ImageViewKind::Dim3D ? IR::DescriptorBindingKind::StorageUint3D
@@ -184,16 +183,16 @@ void EmitImageStore(EmitterState* state, const IR::Instruction& inst) {
const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource,
binding.array_index, uint_image, view);
state->builder.AddFunction(
state.builder.AddFunction(
{OpImageWrite, image, EmitImageCoordU32(state, inst, view),
uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)});
}
void EmitImageSampleResult(EmitterState* state, const IR::Instruction& inst, uint32_t sample,
void EmitImageSampleResult(EmitterState& state, const IR::Instruction& inst, uint32_t sample,
bool dref, bool integer) {
if (dref) {
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->uint_type, bits, sample});
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.uint_type, bits, sample});
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
}
@@ -206,13 +205,12 @@ void EmitImageSampleResult(EmitterState* state, const IR::Instruction& inst, uin
if (((dmask >> component_index) & 1u) == 0) {
continue;
}
const auto component = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract,
integer ? state->uint_type : state->float_type, component,
sample, component_index});
const auto bits = integer ? component : state->builder.AllocateId();
const auto component = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
component, sample, component_index});
const auto bits = integer ? component : state.builder.AllocateId();
if (!integer) {
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
}
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
}
@@ -231,12 +229,9 @@ bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instructio
return state.stage != ShaderType::Pixel;
}
void AddImageSampleOperands(EmitterState* state, const IR::Instruction& inst,
void AddImageSampleOperands(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout, bool explicit_lod,
std::vector<uint32_t>* words) {
if (words == nullptr) {
return;
}
std::vector<uint32_t>& words) {
uint32_t mask = 0;
std::vector<uint32_t> operands;
@@ -255,8 +250,8 @@ void AddImageSampleOperands(EmitterState* state, const IR::Instruction& inst,
operands.push_back(EmitImageBiasF32(state, inst, layout));
}
if (mask != 0) {
words->push_back(mask);
words->insert(words->end(), operands.begin(), operands.end());
words.push_back(mask);
words.insert(words.end(), operands.begin(), operands.end());
}
}
@@ -270,20 +265,20 @@ uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& ins
return dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod;
}
void EmitImageSample(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
void EmitImageSample(EmitterState& state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
const auto layout = MakeImageSampleLayout(inst, view);
const auto base_coord = EmitImageCoordF32(state, inst, layout, view);
const auto sample = state->builder.AllocateId();
const auto sample = state.builder.AllocateId();
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
const auto result_type = dref ? state->float_type
: integer ? state->vec4_uint_type
: state->vec4_float_type;
const auto explicit_lod = ImageSampleNeedsExplicitLod(*state, inst);
const auto opcode = ImageSampleOpcode(*state, inst);
const auto result_type = dref ? state.float_type
: integer ? state.vec4_uint_type
: state.vec4_float_type;
const auto explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
const auto opcode = ImageSampleOpcode(state, inst);
const auto coord =
EmitImageOffsetCoordF32(state, inst, layout, sampled_image, base_coord, view);
@@ -291,8 +286,8 @@ void EmitImageSample(EmitterState* state, const IR::Instruction& inst) {
if (dref) {
words.push_back(EmitImageDrefF32(state, inst, layout));
}
AddImageSampleOperands(state, inst, layout, explicit_lod, &words);
state->builder.AddFunction(words);
AddImageSampleOperands(state, inst, layout, explicit_lod, words);
state.builder.AddFunction(words);
EmitImageSampleResult(state, inst, sample, dref, integer);
}
@@ -305,35 +300,32 @@ uint32_t ImageGatherComponent(uint32_t dmask) {
}
}
void AddImageGatherOperands(EmitterState* state, const IR::Instruction& inst,
const ImageSampleLayout& layout, std::vector<uint32_t>* words) {
if (words == nullptr) {
return;
}
void AddImageGatherOperands(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout, std::vector<uint32_t>& words) {
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagGatherHorizontal)) {
words->push_back(ImageOperandsConstOffsetsMask);
words->push_back(ConstantImageGatherHorizontalOffsets(state));
words.push_back(ImageOperandsConstOffsetsMask);
words.push_back(ConstantImageGatherHorizontalOffsets(state));
return;
}
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
words->push_back(ImageOperandsOffsetMask);
words->push_back(EmitImagePackedOffset2I32(state, inst, layout));
words.push_back(ImageOperandsOffsetMask);
words.push_back(EmitImagePackedOffset2I32(state, inst, layout));
}
}
void EmitImageGather4(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
void EmitImageGather4(EmitterState& state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
const auto layout = MakeImageSampleLayout(inst, view);
const auto coord = EmitImageCoordF32(state, inst, layout, view);
const auto texels = state->builder.AllocateId();
const auto texels = state.builder.AllocateId();
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
const auto result_type = integer ? state->vec4_uint_type : state->vec4_float_type;
const auto result_type = integer ? state.vec4_uint_type : state.vec4_float_type;
std::vector<uint32_t> words =
dref ? std::vector<uint32_t> {OpImageDrefGather,
state->vec4_float_type,
state.vec4_float_type,
texels,
sampled_image,
coord,
@@ -342,17 +334,16 @@ void EmitImageGather4(EmitterState* state, const IR::Instruction& inst) {
OpImageGather, result_type,
texels, sampled_image,
coord, ConstantU32(state, ImageGatherComponent(inst.memory.dmask))};
AddImageGatherOperands(state, inst, layout, &words);
state->builder.AddFunction(words);
AddImageGatherOperands(state, inst, layout, words);
state.builder.AddFunction(words);
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
const auto component = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract,
integer ? state->uint_type : state->float_type, component,
texels, i});
const auto bits = integer ? component : state->builder.AllocateId();
const auto component = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
component, texels, i});
const auto bits = integer ? component : state.builder.AllocateId();
if (!integer) {
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
}
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
}
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h
+410 -407
View File
File diff suppressed because it is too large Load Diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterMemory.cpp
+499 -499
View File
File diff suppressed because it is too large Load Diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterModule.cpp
+525 -525
View File
File diff suppressed because it is too large Load Diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterValues.cpp
+332 -332
View File
@@ -21,7 +21,7 @@ bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width) {
return false;
}
uint32_t EmitSdwaExtractU32(EmitterState* state, const IR::Operand& operand, uint32_t value) {
uint32_t EmitSdwaExtractU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
if (operand.sdwa_sel == 6u && !operand.sdwa_sext) {
return value;
}
@@ -33,117 +33,117 @@ uint32_t EmitSdwaExtractU32(EmitterState* state, const IR::Operand& operand, uin
}
if (operand.sdwa_sext) {
const auto signed_value = state->builder.AllocateId();
const auto extracted = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->int_type, signed_value, value});
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, extracted, signed_value,
const auto signed_value = state.builder.AllocateId();
const auto extracted = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.int_type, signed_value, value});
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_value,
ConstantU32(state, offset), ConstantU32(state, width)});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, extracted});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, extracted});
return bits;
}
const auto extracted = state->builder.AllocateId();
state->builder.AddFunction({OpBitFieldUExtract, state->uint_type, extracted, value,
const auto extracted = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, value,
ConstantU32(state, offset), ConstantU32(state, width)});
return extracted;
}
uint32_t EmitTrueBool(EmitterState* state) {
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpINotEqual, state->bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
uint32_t EmitTrueBool(EmitterState& state) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
return ret;
}
DppTargetLane EmitDppQuadPermTargetLane(EmitterState* state, uint32_t subid, uint32_t control) {
const auto quad_base = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto shift = state->builder.AllocateId();
const auto selected0 = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 3)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, shift, lane, ConstantU32(state, 1)});
state->builder.AddFunction(
{OpShiftRightLogical, state->uint_type, selected0, ConstantU32(state, control), shift});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, selected, selected0, ConstantU32(state, 3)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, quad_base, selected});
DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
const auto quad_base = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto shift = state.builder.AllocateId();
const auto selected0 = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)});
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected});
return {target, EmitTrueBool(state)};
}
DppTargetLane EmitDppRowShiftTargetLane(EmitterState* state, uint32_t subid, uint32_t amount,
DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
bool left) {
const auto row = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto lane_shifted = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
const auto valid = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
const auto row = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto lane_shifted = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
const auto valid = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
if (left) {
state->builder.AddFunction(
{OpIAdd, state->uint_type, lane_shifted, lane, ConstantU32(state, amount)});
state->builder.AddFunction(
{OpULessThan, state->bool_type, valid, lane, ConstantU32(state, 16u - amount)});
state.builder.AddFunction(
{OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
state.builder.AddFunction(
{OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)});
} else {
state->builder.AddFunction(
{OpISub, state->uint_type, lane_shifted, lane, ConstantU32(state, amount)});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, valid, lane, ConstantU32(state, amount)});
state.builder.AddFunction(
{OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)});
}
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row, lane_shifted});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted});
return {target, valid};
}
DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState* state, uint32_t subid,
DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid,
uint32_t amount) {
const auto row = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto in_high = state->builder.AllocateId();
const auto minus = state->builder.AllocateId();
const auto plus = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, amount)});
state->builder.AddFunction({OpISub, state->uint_type, minus, lane, ConstantU32(state, amount)});
state->builder.AddFunction(
{OpIAdd, state->uint_type, plus, lane, ConstantU32(state, 16u - amount)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, minus, plus});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row, selected});
const auto row = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto in_high = state.builder.AllocateId();
const auto minus = state.builder.AllocateId();
const auto plus = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)});
state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)});
state.builder.AddFunction(
{OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)});
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected});
return {target, EmitTrueBool(state)};
}
DppTargetLane EmitDppMirrorTargetLane(EmitterState* state, uint32_t subid, bool half_row) {
DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
const auto lane_mask = half_row ? 7u : 15u;
const auto base = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto mirrored = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, base, subid, ConstantU32(state, base_mask)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, lane_mask)});
state->builder.AddFunction(
{OpISub, state->uint_type, mirrored, ConstantU32(state, lane_mask), lane});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, base, mirrored});
const auto base = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto mirrored = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)});
state.builder.AddFunction(
{OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored});
return {target, EmitTrueBool(state)};
}
DppTargetLane EmitDppTargetLane(EmitterState* state, uint32_t control) {
DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
const auto subid = EmitSubgroupLocalInvocationId(state);
if (control <= 0xffu) {
return EmitDppQuadPermTargetLane(state, subid, control);
@@ -166,42 +166,42 @@ DppTargetLane EmitDppTargetLane(EmitterState* state, uint32_t control) {
return {subid, EmitTrueBool(state)};
}
uint32_t EmitDppValueU32(EmitterState* state, const IR::Operand& operand, uint32_t value) {
uint32_t EmitDppValueU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
if (!operand.dpp) {
return value;
}
const auto target = EmitDppTargetLane(state, operand.dpp_ctrl);
const auto shuffled = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, shuffled,
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
ConstantU32(state, ScopeSubgroup), value, target.lane});
if (operand.dpp_fetch_inactive) {
return shuffled;
}
const auto source_active = EmitLaneIndexActiveBool(state, target.lane);
const auto can_fetch = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpLogicalAnd, state->bool_type, can_fetch, target.valid, source_active});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)});
const auto can_fetch = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitValueLoad(EmitterState* state, const IR::Operand& operand) {
uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand) {
uint32_t value = 0;
switch (operand.kind) {
case IR::OperandKind::ImmediateU32:
case IR::OperandKind::PcRelativeU32: value = ConstantU32(state, operand.imm); break;
case IR::OperandKind::Register: {
const auto pointer = PointerForRegister(*state, operand.reg);
const auto pointer = PointerForRegister(state, operand.reg);
if (pointer == 0) {
value = ConstantU32(state, 0);
break;
}
value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
break;
}
case IR::OperandKind::Null: value = ConstantU32(state, 0); break;
@@ -211,160 +211,160 @@ uint32_t EmitValueLoad(EmitterState* state, const IR::Operand& operand) {
return EmitSdwaExtractU32(state, operand, value);
}
uint32_t EmitRegisterLoad(EmitterState* state, IR::Register reg) {
if (IsInactiveWave32ExecHigh(*state, reg)) {
uint32_t EmitRegisterLoad(EmitterState& state, IR::Register reg) {
if (IsInactiveWave32ExecHigh(state, reg)) {
return ConstantU32(state, 0);
}
const auto pointer = PointerForRegister(*state, reg);
const auto pointer = PointerForRegister(state, reg);
if (pointer == 0) {
return ConstantU32(state, 0);
}
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
}
uint32_t WaveMaskHighLoad(EmitterState* state, IR::RegisterFile file) {
if (state->wave_size == 32u && IsMaskRegisterFile(file)) {
uint32_t WaveMaskHighLoad(EmitterState& state, IR::RegisterFile file) {
if (state.wave_size == 32u && IsMaskRegisterFile(file)) {
return ConstantU32(state, 0);
}
return EmitRegisterLoad(state, {file, 1});
}
uint32_t EmitLaneMaskPartU32(EmitterState* state, uint32_t part) {
uint32_t EmitLaneMaskPartU32(EmitterState& state, uint32_t part) {
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto lane = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane});
const auto lane = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
if (part == 0) {
const auto in_low = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpULessThan, state->bool_type, in_low, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_low, bit, ConstantU32(state, 0)});
const auto in_low = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpULessThan, state.bool_type, in_low, subid, ConstantU32(state, 32)});
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, in_low, bit, ConstantU32(state, 0)});
return ret;
}
if (state->wave_size == 32u) {
if (state.wave_size == 32u) {
return ConstantU32(state, 0);
}
const auto in_high = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_high, bit, ConstantU32(state, 0)});
const auto in_high = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, in_high, bit, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitThreadMaskBelowPartU32(EmitterState* state, uint32_t part) {
uint32_t EmitThreadMaskBelowPartU32(EmitterState& state, uint32_t part) {
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto lane = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
const auto below = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane});
state->builder.AddFunction({OpISub, state->uint_type, below, bit, ConstantU32(state, 1)});
const auto lane = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
const auto below = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
state.builder.AddFunction({OpISub, state.uint_type, below, bit, ConstantU32(state, 1)});
if (part == 0) {
const auto in_high = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below});
const auto in_high = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below});
return ret;
}
if (state->wave_size == 32u) {
if (state.wave_size == 32u) {
return ConstantU32(state, 0);
}
const auto in_high = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_high, below, ConstantU32(state, 0)});
const auto in_high = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, in_high, below, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitMaskActiveBool(EmitterState* state, IR::RegisterFile file) {
uint32_t EmitMaskActiveBool(EmitterState& state, IR::RegisterFile file) {
const auto low = EmitRegisterLoad(state, {file, 0});
const auto condition = state->builder.AllocateId();
if (state->per_invocation_masks) {
const auto combined = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseOr, state->uint_type, combined, low, WaveMaskHighLoad(state, file)});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, condition, combined, ConstantU32(state, 0)});
const auto condition = state.builder.AllocateId();
if (state.per_invocation_masks) {
const auto combined = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseOr, state.uint_type, combined, low, WaveMaskHighLoad(state, file)});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, condition, combined, ConstantU32(state, 0)});
return condition;
}
if (state->exact_subgroup_operations) {
if (state.exact_subgroup_operations) {
const auto high = WaveMaskHighLoad(state, file);
const auto low_hit = state->builder.AllocateId();
const auto high_hit = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, active, low_hit, high_hit});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, condition, active, ConstantU32(state, 0)});
const auto low_hit = state.builder.AllocateId();
const auto high_hit = state.builder.AllocateId();
const auto active = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, active, low_hit, high_hit});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, condition, active, ConstantU32(state, 0)});
return condition;
}
state->builder.AddFunction(
{OpINotEqual, state->bool_type, condition, low, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, condition, low, ConstantU32(state, 0)});
return condition;
}
uint32_t EmitMaskZeroBool(EmitterState* state, IR::RegisterFile file, bool zero) {
uint32_t EmitMaskZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
const auto active = EmitMaskActiveBool(state, file);
if (!zero) {
return active;
}
const auto cond = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, cond, active});
const auto cond = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalNot, state.bool_type, cond, active});
return cond;
}
uint32_t EmitMaskSummaryZeroBool(EmitterState* state, IR::RegisterFile file, bool zero) {
if (state->per_invocation_masks) {
uint32_t EmitMaskSummaryZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
if (state.per_invocation_masks) {
return EmitMaskZeroBool(state, file, zero);
}
if (state->exact_subgroup_operations) {
if (state.exact_subgroup_operations) {
const auto low = EmitRegisterLoad(state, {file, 0});
const auto high = WaveMaskHighLoad(state, file);
const auto combined = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, low, high});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, combined, ConstantU32(state, 0)});
const auto combined = state.builder.AllocateId();
const auto active = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
if (!zero) {
return active;
}
const auto inactive = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
const auto inactive = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
return inactive;
}
const auto active = EmitMaskActiveBool(state, file);
if (!zero) {
return active;
}
const auto inactive = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
const auto inactive = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
return inactive;
}
uint32_t EmitExecActiveBool(EmitterState* state) {
uint32_t EmitExecActiveBool(EmitterState& state) {
return EmitMaskZeroBool(state, IR::RegisterFile::Exec, false);
}
uint32_t EmitConditionBool(EmitterState* state, const IR::Operand& operand) {
uint32_t EmitConditionBool(EmitterState& state, const IR::Operand& operand) {
if (operand.kind == IR::OperandKind::Register) {
if (operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc) {
@@ -372,19 +372,19 @@ uint32_t EmitConditionBool(EmitterState* state, const IR::Operand& operand) {
}
}
const auto cond_value = EmitValueLoad(state, operand);
const auto cond = state->builder.AllocateId();
state->builder.AddFunction(
{OpINotEqual, state->bool_type, cond, cond_value, ConstantU32(state, 0)});
const auto cond = state.builder.AllocateId();
state.builder.AddFunction(
{OpINotEqual, state.bool_type, cond, cond_value, ConstantU32(state, 0)});
return cond;
}
uint32_t EmitSubgroupLocalInvocationId(EmitterState* state) {
if (state->subgroup_local_invocation_id_variable == 0) {
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
if (state.subgroup_local_invocation_id_variable == 0) {
return ConstantU32(state, 0);
}
const auto value = state->builder.AllocateId();
state->builder.AddFunction(
{OpLoad, state->uint_type, value, state->subgroup_local_invocation_id_variable});
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable});
return value;
}
@@ -415,30 +415,30 @@ const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t
return nullptr;
}
uint32_t EmitInputComponentU32(EmitterState* state, IR::StageInputKind kind, uint32_t component) {
const auto variable = InputVariableForKind(*state, kind);
uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
const auto variable = InputVariableForKind(state, kind);
if (variable == 0) {
return ConstantU32(state, 0);
}
const auto pointer = state->builder.AllocateId();
const auto value = state->builder.AllocateId();
state->builder.AddFunction(
{OpAccessChain, state->ptr_input_uint, pointer, variable, ConstantU32(state, component)});
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)});
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
}
uint32_t EmitLocalInvocationIndex(EmitterState* state) {
const auto variable = InputVariableForKind(*state, IR::StageInputKind::LocalInvocationIndex);
uint32_t EmitLocalInvocationIndex(EmitterState& state) {
const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex);
if (variable == 0) {
return ConstantU32(state, 0);
}
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, variable});
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, value, variable});
return value;
}
uint32_t VertexInputDefaultComponentU32(EmitterState* state, VertexInputScalarKind kind,
uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind,
uint32_t component) {
if ((component & 3u) != 3u) {
return ConstantU32(state, 0);
@@ -446,105 +446,105 @@ uint32_t VertexInputDefaultComponentU32(EmitterState* state, VertexInputScalarKi
return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
}
uint32_t EmitVertexParameterComponentU32(EmitterState* state, const InputBinding& input,
uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
uint32_t component) {
const auto count = VertexParameterComponentCount(*state, input);
const auto kind = VertexParameterScalarKind(*state, input.location);
const auto count = VertexParameterComponentCount(state, input);
const auto kind = VertexParameterScalarKind(state, input.location);
if (component >= count) {
return VertexInputDefaultComponentU32(state, kind, component);
}
const auto scalar_type = VertexParameterScalarType(*state, kind);
uint32_t raw = state->builder.AllocateId();
const auto scalar_type = VertexParameterScalarType(state, kind);
uint32_t raw = state.builder.AllocateId();
if (count == 1u) {
state->builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
} else {
const auto pointer_type = VertexParameterScalarPointerType(*state, kind);
const auto pointer = state->builder.AllocateId();
state->builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
const auto pointer_type = VertexParameterScalarPointerType(state, kind);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
ConstantU32(state, component)});
state->builder.AddFunction({OpLoad, scalar_type, raw, pointer});
state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
}
if (kind == VertexInputScalarKind::Uint) {
return raw;
}
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->uint_type, bits, raw});
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
return bits;
}
void EmitLoadInputF32(EmitterState* state, const IR::Instruction& inst) {
const auto* input = InputBindingForParameter(*state, inst.input_info.attr);
void EmitLoadInputF32(EmitterState& state, const IR::Instruction& inst) {
const auto* input = InputBindingForParameter(state, inst.input_info.attr);
if (input == nullptr || input->variable_id == 0) {
EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
return;
}
if (state->stage == ShaderType::Vertex) {
if (state.stage == ShaderType::Vertex) {
EmitStoreU32(state, inst.dst,
EmitVertexParameterComponentU32(state, *input, inst.input_info.chan & 3u));
return;
}
const auto input_value = state->builder.AllocateId();
const auto component = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->vec4_float_type, input_value, input->variable_id});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, component, input_value, inst.input_info.chan & 3u});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
const auto input_value = state.builder.AllocateId();
const auto component = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.vec4_float_type, input_value, input->variable_id});
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, component, input_value, inst.input_info.chan & 3u});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
EmitStoreU32(state, inst.dst, bits);
}
uint32_t EmitSccBool(EmitterState* state, bool non_zero) {
uint32_t EmitSccBool(EmitterState& state, bool non_zero) {
const auto value = EmitRegisterLoad(state, SccRegister());
const auto cond = state->builder.AllocateId();
state->builder.AddFunction(
{non_zero ? OpINotEqual : OpIEqual, state->bool_type, cond, value, ConstantU32(state, 0)});
const auto cond = state.builder.AllocateId();
state.builder.AddFunction(
{non_zero ? OpINotEqual : OpIEqual, state.bool_type, cond, value, ConstantU32(state, 0)});
return cond;
}
uint32_t EmitFloatLoad(EmitterState* state, const IR::Operand& operand) {
uint32_t EmitFloatLoad(EmitterState& state, const IR::Operand& operand) {
const auto uint_value = EmitValueLoad(state, operand);
auto float_value = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, float_value, uint_value});
auto float_value = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, float_value, uint_value});
if (operand.absolute) {
const auto abs_value = state->builder.AllocateId();
state->builder.AddFunction(
{OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, float_value});
const auto abs_value = state.builder.AllocateId();
state.builder.AddFunction(
{OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, float_value});
float_value = abs_value;
}
if (operand.negate) {
const auto neg_value = state->builder.AllocateId();
state->builder.AddFunction({OpFNegate, state->float_type, neg_value, float_value});
const auto neg_value = state.builder.AllocateId();
state.builder.AddFunction({OpFNegate, state.float_type, neg_value, float_value});
float_value = neg_value;
}
return float_value;
}
uint32_t ApplyResultModifiersF32(EmitterState* state, uint32_t value, const IR::Operand& dst) {
uint32_t ApplyResultModifiersF32(EmitterState& state, uint32_t value, const IR::Operand& dst) {
uint32_t result = value;
if (dst.omod != 0) {
const uint32_t bits = dst.omod == 1u ? 0x40000000u
: dst.omod == 2u ? 0x40800000u
: 0x3f000000u;
const auto scaled = state->builder.AllocateId();
state->builder.AddFunction(
{OpFMul, state->float_type, scaled, result, ConstantF32(state, bits)});
const auto scaled = state.builder.AllocateId();
state.builder.AddFunction(
{OpFMul, state.float_type, scaled, result, ConstantF32(state, bits)});
result = scaled;
}
if (!dst.clamp) {
return result;
}
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpExtInst, state->float_type, ret, state->glsl_std450, GlslFClamp,
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpExtInst, state.float_type, ret, state.glsl_std450, GlslFClamp,
result, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
return ret;
}
uint32_t EmitMixF32Load(EmitterState* state, const IR::Operand& operand) {
uint32_t EmitMixF32Load(EmitterState& state, const IR::Operand& operand) {
auto value_operand = operand;
value_operand.negate_hi = false;
@@ -562,21 +562,21 @@ uint32_t EmitMixF32Load(EmitterState* state, const IR::Operand& operand) {
raw.dpp_bound_ctrl = false;
const auto bits = EmitValueLoad(state, raw);
const auto unpacked = state->builder.AllocateId();
auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450,
const auto unpacked = state.builder.AllocateId();
auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450,
GlslUnpackHalf2x16, bits});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, f32, unpacked, operand.op_sel ? 1u : 0u});
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, f32, unpacked, operand.op_sel ? 1u : 0u});
if (operand.absolute) {
const auto abs_value = state->builder.AllocateId();
state->builder.AddFunction(
{OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, f32});
const auto abs_value = state.builder.AllocateId();
state.builder.AddFunction(
{OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, f32});
f32 = abs_value;
}
if (value_operand.negate) {
const auto neg_value = state->builder.AllocateId();
state->builder.AddFunction({OpFNegate, state->float_type, neg_value, f32});
const auto neg_value = state.builder.AllocateId();
state.builder.AddFunction({OpFNegate, state.float_type, neg_value, f32});
f32 = neg_value;
}
return f32;
@@ -611,10 +611,10 @@ IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset) {
return ret;
}
uint32_t EmitLaneMaskOperandActiveBool(EmitterState* state, const IR::Operand& operand) {
uint32_t EmitLaneMaskOperandActiveBool(EmitterState& state, const IR::Operand& operand) {
const auto low = EmitValueLoad(state, operand);
const auto active = state->builder.AllocateId();
if (state->per_invocation_masks || state->exact_subgroup_operations) {
const auto active = state.builder.AllocateId();
if (state.per_invocation_masks || state.exact_subgroup_operations) {
uint32_t high = ConstantU32(state, 0);
if (operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Scalar ||
@@ -624,102 +624,102 @@ uint32_t EmitLaneMaskOperandActiveBool(EmitterState* state, const IR::Operand& o
? WaveMaskHighLoad(state, operand.reg.file)
: EmitValueLoad(state, OffsetRegisterOperand(operand, 1));
}
if (state->per_invocation_masks) {
const auto combined = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, low, high});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, combined, ConstantU32(state, 0)});
if (state.per_invocation_masks) {
const auto combined = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
return active;
}
const auto low_hit = state->builder.AllocateId();
const auto high_hit = state->builder.AllocateId();
const auto hit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, hit, low_hit, high_hit});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
const auto low_hit = state.builder.AllocateId();
const auto high_hit = state.builder.AllocateId();
const auto hit = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, hit, low_hit, high_hit});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
return active;
}
state->builder.AddFunction({OpINotEqual, state->bool_type, active, low, ConstantU32(state, 0)});
state.builder.AddFunction({OpINotEqual, state.bool_type, active, low, ConstantU32(state, 0)});
return active;
}
uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane) {
if (state->per_invocation_masks) {
const auto ballot = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane) {
if (state.per_invocation_masks) {
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
return EmitBallotLaneActiveBool(state, ballot, lane);
}
const auto lane_low = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane_low, lane, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane_low});
const auto lane_low = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
uint32_t mask = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
if (state->wave_size == 64u) {
if (state.wave_size == 64u) {
const auto high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
const auto in_high = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, 32)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, high, mask});
const auto in_high = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, mask});
mask = selected;
}
const auto hit = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
state->builder.AddFunction({OpINotEqual, state->bool_type, ret, hit, ConstantU32(state, 0)});
const auto hit = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
state.builder.AddFunction({OpINotEqual, state.bool_type, ret, hit, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitSubgroupLaneActiveBool(EmitterState* state, uint32_t lane) {
const auto active_ballot = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, active_ballot,
uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
const auto active_ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot,
ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
return EmitBallotLaneActiveBool(state, active_ballot, lane);
}
uint32_t EmitBallotLaneActiveBool(EmitterState* state, uint32_t active_ballot, uint32_t lane) {
const auto low = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, low, active_ballot, 0});
uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
const auto low = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0});
uint32_t mask = low;
if (state->wave_size == 64u) {
const auto high = state->builder.AllocateId();
const auto in_high = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, active_ballot, 1});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, 32)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, high, low});
if (state.wave_size == 64u) {
const auto high = state.builder.AllocateId();
const auto in_high = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1});
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low});
mask = selected;
}
const auto lane_low = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane_low, lane, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane_low});
const auto lane_low = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
const auto hit = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
const auto in_range = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
state->builder.AddFunction({OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
state->builder.AddFunction(
{OpULessThan, state->bool_type, in_range, lane, ConstantU32(state, state->wave_size)});
state->builder.AddFunction({OpLogicalAnd, state->bool_type, ret, active, in_range});
const auto hit = state.builder.AllocateId();
const auto active = state.builder.AllocateId();
const auto in_range = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)});
state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range});
return ret;
}
uint32_t EmitSequentialValueLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset) {
uint32_t EmitSequentialValueLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
if (offset == 0 || operand.kind == IR::OperandKind::Register) {
return EmitValueLoad(state, OffsetRegisterOperand(operand, offset));
}
@@ -729,10 +729,10 @@ uint32_t EmitSequentialValueLoad(EmitterState* state, const IR::Operand& operand
return ConstantU32(state, 0);
}
uint32_t EmitSequentialFloatLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset) {
uint32_t EmitSequentialFloatLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
const auto value = EmitSequentialValueLoad(state, operand, offset);
const auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, f32, value});
const auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
return f32;
}
@@ -803,7 +803,7 @@ IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand&
return OffsetRegisterOperand(base, packed_component);
}
uint32_t EmitImageAddressValueLoad(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageAddressValueLoad(EmitterState& state, const IR::Instruction& inst,
const IR::Operand& base, uint32_t component) {
const auto operand = ImageAddressOperand(inst, base, component);
const auto packed_component = ImageAddressPackedComponent(inst, component);
@@ -820,20 +820,20 @@ uint32_t EmitImageAddressValueLoad(EmitterState* state, const IR::Instruction& i
return ConstantU32(state, 0);
}
uint32_t EmitImageAddressFloatLoad(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageAddressFloatLoad(EmitterState& state, const IR::Instruction& inst,
const IR::Operand& base, uint32_t component) {
const auto value = EmitImageAddressValueLoad(state, inst, base, component);
if (ImageAddressComponentUsesA16(inst, component)) {
return EmitF16BitsToF32(state, value);
}
const auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, f32, value});
const auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
return f32;
}
uint32_t EmitZeroF32(EmitterState* state) {
const auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, f32, ConstantU32(state, 0)});
uint32_t EmitZeroF32(EmitterState& state) {
const auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, f32, ConstantU32(state, 0)});
return f32;
}