mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-20 19:07:01 +02:00
nmzik
754 lines
30 KiB
C++
754 lines
30 KiB
C++
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
|
|
|
#include "common/assert.h"
|
|
|
|
#include <algorithm>
|
|
#include <bit>
|
|
#include <functional>
|
|
#include <optional>
|
|
#include <unordered_set>
|
|
|
|
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
|
namespace {
|
|
|
|
void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
|
|
const auto kill_label = state.builder.AllocateId();
|
|
const auto merge_label = state.builder.AllocateId();
|
|
const auto inactive = state.builder.AllocateId();
|
|
state.builder.AddFunction({spv::OpLogicalNot, TypeBool(state), inactive, active});
|
|
state.builder.AddFunction({spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
|
|
state.builder.AddFunction({spv::OpBranchConditional, inactive, kill_label, merge_label});
|
|
EmitLabel(state, kill_label);
|
|
state.builder.AddFunction({spv::OpKill});
|
|
EmitLabel(state, merge_label);
|
|
}
|
|
|
|
void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
|
|
if (state.pixel_valid_mask_variable == 0) {
|
|
return;
|
|
}
|
|
|
|
const auto mask_value = state.builder.AllocateId();
|
|
const auto active = state.builder.AllocateId();
|
|
state.builder.AddFunction(
|
|
{spv::OpLoad, TypeU32(state), mask_value, state.pixel_valid_mask_variable});
|
|
state.builder.AddFunction(
|
|
{spv::OpINotEqual, TypeBool(state), active, mask_value, ConstantU32(state, 0)});
|
|
EmitKillIfBoolFalse(state, active);
|
|
}
|
|
|
|
uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
|
|
const auto value_type = ctx.TypeId(type);
|
|
return value_type == 0 ? 0 : TypePointer(ctx.state, spv::StorageClassFunction, value_type);
|
|
}
|
|
|
|
struct DeferredPhiPatch {
|
|
DeferredPhi phi;
|
|
const IR::Inst* instruction = nullptr;
|
|
uint32_t half = 0;
|
|
};
|
|
|
|
struct StructuredFunctionState {
|
|
std::unordered_map<const IR::Block*, uint32_t> block_exit_labels;
|
|
std::vector<DeferredPhiPatch> deferred_phis;
|
|
};
|
|
|
|
struct DispatcherFunctionState {
|
|
std::array<std::unordered_map<const IR::Inst*, uint32_t>, 2> spills;
|
|
uint32_t header_label = 0;
|
|
uint32_t select_label = 0;
|
|
uint32_t after_switch_label = 0;
|
|
uint32_t continue_label = 0;
|
|
uint32_t merge_label = 0;
|
|
};
|
|
|
|
void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
|
|
const IR::Block* from, const IR::Block* to) {
|
|
if (to == nullptr) {
|
|
return;
|
|
}
|
|
for (const auto& phi: *to) {
|
|
if (phi.GetOpcode() != IR::ValueOpcode::Phi) {
|
|
break;
|
|
}
|
|
for (size_t index = 0; index < phi.NumArgs(); index++) {
|
|
if (phi.PhiBlock(index) == from) {
|
|
ctx.state.builder.AddFunction(
|
|
{spv::OpStore, dispatcher.spills[ctx.half].at(&phi), ctx.Def(phi.Arg(index))});
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const IR::Block* TargetBlock(const IR::Program& program, uint32_t id) {
|
|
const auto found = std::ranges::find_if(
|
|
program.block_info, [&](const IR::BlockInfo& info) { return info.id == id; });
|
|
if (found == program.block_info.end()) {
|
|
return nullptr;
|
|
}
|
|
return program.blocks[static_cast<size_t>(found - program.block_info.begin())];
|
|
}
|
|
|
|
void EmitReturn(ValueEmitContext& ctx) {
|
|
EmitKillIfPixelValidMaskInactive(ctx.state);
|
|
ctx.state.builder.AddFunction({spv::OpReturn});
|
|
}
|
|
|
|
uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
|
|
// Scalar-instruction conditions already test the full wave's raw register values.
|
|
if (ctx.other_half == nullptr ||
|
|
info.terminator.condition == CFG::BranchCondition::ScalarInstruction ||
|
|
info.terminator.condition == CFG::BranchCondition::GotoVariable) {
|
|
return ctx.Def(info.condition);
|
|
}
|
|
const auto ballot = ctx.Ballot(info.condition);
|
|
const auto low = ctx.state.builder.AllocateId();
|
|
const auto high = ctx.state.builder.AllocateId();
|
|
const auto result = ctx.state.builder.AllocateId();
|
|
ctx.state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0});
|
|
ctx.state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1});
|
|
const auto kind = info.terminator.condition;
|
|
const bool zero = kind == CFG::BranchCondition::ExecZero ||
|
|
kind == CFG::BranchCondition::VccZero ||
|
|
kind == CFG::BranchCondition::SccZero;
|
|
const auto combined =
|
|
EmitBinaryU32(ctx.state, zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, low, high);
|
|
ctx.state.builder.AddFunction({zero ? spv::OpIEqual : spv::OpINotEqual, TypeBool(ctx.state),
|
|
result, combined, ConstantU32(ctx.state, zero ? ~0u : 0u)});
|
|
return result;
|
|
}
|
|
|
|
void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
|
|
const IR::BlockInfo& info) {
|
|
const auto& program = ctx.state.program;
|
|
const auto& term = info.terminator;
|
|
const auto emit_merge = [&]() {
|
|
if (term.loop_header) {
|
|
const auto* merge = TargetBlock(program, term.merge_block);
|
|
const auto* cont = TargetBlock(program, term.continue_block);
|
|
if (merge != nullptr && cont != nullptr) {
|
|
ctx.state.builder.AddFunction({spv::OpLoopMerge, ctx.Label(merge), ctx.Label(cont),
|
|
spv::LoopControlMaskNone});
|
|
}
|
|
} else if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
|
|
term.merge_block != UINT32_MAX) {
|
|
if (const auto* merge = TargetBlock(program, term.merge_block); merge != nullptr) {
|
|
ctx.state.builder.AddFunction(
|
|
{spv::OpSelectionMerge, ctx.Label(merge), spv::SelectionControlMaskNone});
|
|
}
|
|
}
|
|
};
|
|
|
|
switch (term.kind) {
|
|
case CFG::TerminatorKind::Branch: {
|
|
const auto* target = TargetBlock(program, term.true_block);
|
|
if (target == nullptr) {
|
|
EmitReturn(ctx);
|
|
return;
|
|
}
|
|
emit_merge();
|
|
ctx.state.builder.AddFunction({spv::OpBranch, ctx.Label(target)});
|
|
return;
|
|
}
|
|
case CFG::TerminatorKind::ConditionalBranch: {
|
|
const auto* true_block = TargetBlock(program, term.true_block);
|
|
const auto* false_block = TargetBlock(program, term.false_block);
|
|
if (true_block == nullptr || false_block == nullptr || info.condition.IsEmpty()) {
|
|
EmitReturn(ctx);
|
|
return;
|
|
}
|
|
const auto condition = BranchCondition(ctx, info);
|
|
emit_merge();
|
|
ctx.state.builder.AddFunction({spv::OpBranchConditional, condition,
|
|
ctx.Label(true_block), ctx.Label(false_block)});
|
|
return;
|
|
}
|
|
default: EmitReturn(ctx); return;
|
|
}
|
|
}
|
|
|
|
void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
|
|
const IR::Block* from, uint32_t target) {
|
|
const auto* block = TargetBlock(ctx.state.program, target);
|
|
if (block != nullptr) {
|
|
StoreDispatcherPhiEdge(ctx, dispatcher, from, block);
|
|
if (ctx.other_half != nullptr) {
|
|
StoreDispatcherPhiEdge(*ctx.other_half, dispatcher, from, block);
|
|
}
|
|
}
|
|
}
|
|
|
|
uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
|
|
const IR::Block* block, const IR::BlockInfo& info) {
|
|
const auto& term = info.terminator;
|
|
switch (term.kind) {
|
|
case CFG::TerminatorKind::Branch:
|
|
EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
|
|
return ConstantU32(ctx.state, term.true_block);
|
|
case CFG::TerminatorKind::ConditionalBranch: {
|
|
EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
|
|
EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
|
|
const auto selected = ctx.state.builder.AllocateId();
|
|
ctx.state.builder.AddFunction({spv::OpSelect, TypeU32(ctx.state), selected,
|
|
BranchCondition(ctx, info),
|
|
ConstantU32(ctx.state, term.true_block),
|
|
ConstantU32(ctx.state, term.false_block)});
|
|
return selected;
|
|
}
|
|
case CFG::TerminatorKind::IndirectBranch: {
|
|
for (const auto target: term.indirect_targets) {
|
|
EmitDispatcherTarget(ctx, dispatcher, block, target);
|
|
}
|
|
uint32_t selected = ConstantU32(ctx.state, UINT32_MAX);
|
|
if (!info.indirect_target.IsEmpty()) {
|
|
const auto selector = ctx.Def(info.indirect_target);
|
|
const auto& values = term.indirect_selector_code != UINT32_MAX
|
|
? term.indirect_selector_values
|
|
: term.indirect_target_pcs;
|
|
const auto& targets = term.indirect_selector_code != UINT32_MAX
|
|
? term.indirect_selector_targets
|
|
: term.indirect_targets;
|
|
for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
|
|
const auto match = ctx.state.builder.AllocateId();
|
|
const auto next = ctx.state.builder.AllocateId();
|
|
ctx.state.builder.AddFunction({spv::OpIEqual, TypeBool(ctx.state), match,
|
|
selector,
|
|
ConstantU32(ctx.state, values[index])});
|
|
ctx.state.builder.AddFunction({spv::OpSelect, TypeU32(ctx.state), next, match,
|
|
ConstantU32(ctx.state, targets[index]),
|
|
selected});
|
|
selected = next;
|
|
}
|
|
}
|
|
return selected;
|
|
}
|
|
default: return ConstantU32(ctx.state, UINT32_MAX);
|
|
}
|
|
}
|
|
|
|
void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
|
|
if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) ||
|
|
EmitValueImage(ctx, inst)) {
|
|
return;
|
|
}
|
|
ctx.Fail(inst, "has no direct SPIR-V emitter");
|
|
}
|
|
|
|
void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured,
|
|
const IR::Inst& inst) {
|
|
if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
|
|
const auto type = ctx.TypeId(inst.GetType());
|
|
if (type == 0 || inst.NumArgs() == 0) {
|
|
ctx.Fail(inst, "has no native SPIR-V representation");
|
|
}
|
|
for (size_t index = 0; index < inst.NumArgs(); index++) {
|
|
const auto* predecessor = inst.PhiBlock(index);
|
|
if (predecessor == nullptr || !ctx.state.labels.contains(predecessor)) {
|
|
ctx.Fail(inst, "has a predecessor outside the structured function");
|
|
}
|
|
}
|
|
structured.deferred_phis.push_back(
|
|
{ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst,
|
|
ctx.half});
|
|
return;
|
|
}
|
|
EmitDirectInstruction(ctx, inst);
|
|
}
|
|
|
|
void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
|
|
const IR::Inst& inst) {
|
|
if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
|
|
const auto type = ctx.TypeId(inst.GetType());
|
|
if (type == 0) {
|
|
ctx.Fail(inst, "cannot be loaded by the dispatcher");
|
|
}
|
|
ctx.state.builder.AddFunction(
|
|
{spv::OpLoad, type, ctx.Result(inst), dispatcher.spills[ctx.half].at(&inst)});
|
|
return;
|
|
}
|
|
EmitDirectInstruction(ctx, inst);
|
|
if (const auto found = dispatcher.spills[ctx.half].find(&inst);
|
|
found != dispatcher.spills[ctx.half].end()) {
|
|
ctx.state.builder.AddFunction(
|
|
{spv::OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
|
|
}
|
|
}
|
|
|
|
template <typename EmitInstruction>
|
|
void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& emit_instruction) {
|
|
ctx.state.current_block = block;
|
|
EmitLabel(ctx.state, ctx.Label(block));
|
|
bool emitted_non_phi = false;
|
|
for (const auto& inst: *block) {
|
|
if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
|
|
if (emitted_non_phi) {
|
|
ctx.Fail(inst, "appears after a non-Phi instruction");
|
|
}
|
|
} else {
|
|
emitted_non_phi = true;
|
|
}
|
|
for (uint32_t half = 0; half < ctx.state.lane_count; half++) {
|
|
auto& lane = half == 0 ? ctx : *ctx.other_half;
|
|
ctx.state.lane_half = half;
|
|
if (half == 0 || (inst.GetOpcode() != IR::ValueOpcode::Barrier &&
|
|
inst.GetOpcode() != IR::ValueOpcode::MeshAllocate)) {
|
|
emit_instruction(lane, inst);
|
|
}
|
|
}
|
|
ctx.state.lane_half = 0;
|
|
}
|
|
}
|
|
|
|
void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structured) {
|
|
for (const auto& deferred: structured.deferred_phis) {
|
|
auto& lane = deferred.half == 0 ? ctx : *ctx.other_half;
|
|
for (size_t index = 0; index < deferred.instruction->NumArgs(); index++) {
|
|
const auto* predecessor = deferred.instruction->PhiBlock(index);
|
|
const auto found = structured.block_exit_labels.find(predecessor);
|
|
if (found == structured.block_exit_labels.end()) {
|
|
ctx.Fail(*deferred.instruction, "has a predecessor that was not emitted");
|
|
}
|
|
ctx.state.builder.PatchDeferredPhi(
|
|
deferred.phi, index, lane.Def(deferred.instruction->Arg(index)), found->second);
|
|
}
|
|
}
|
|
}
|
|
|
|
void EmitStructuredFunction(ValueEmitContext& ctx) {
|
|
const auto& program = ctx.state.program;
|
|
StructuredFunctionState structured;
|
|
ctx.state.builder.AddFunction({spv::OpBranch, ctx.Label(program.blocks.front())});
|
|
for (size_t index = 0; index < program.blocks.size(); index++) {
|
|
const auto* block = program.blocks[index];
|
|
EmitBlock(ctx, block, [&](ValueEmitContext& lane, const IR::Inst& inst) {
|
|
EmitStructuredInstruction(lane, structured, inst);
|
|
});
|
|
structured.block_exit_labels.emplace(block, ctx.state.current_label);
|
|
EmitStructuredTerminator(ctx, block, program.block_info[index]);
|
|
}
|
|
PatchStructuredPhis(ctx, structured);
|
|
}
|
|
|
|
void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) {
|
|
auto& state = ctx.state;
|
|
const auto* entry = state.program.blocks.front();
|
|
state.builder.AddFunction({spv::OpBranch, ctx.Label(entry)});
|
|
EmitBlock(ctx, entry, [&](ValueEmitContext& lane, const IR::Inst& inst) {
|
|
EmitDispatcherInstruction(lane, dispatcher, inst);
|
|
});
|
|
const auto initial_pc =
|
|
EmitDispatcherNextPc(ctx, dispatcher, entry, state.program.block_info.front());
|
|
const auto initial_parent = state.current_label;
|
|
state.builder.AddFunction({spv::OpBranch, dispatcher.header_label});
|
|
|
|
EmitLabel(state, dispatcher.header_label);
|
|
const auto pc = state.builder.AllocateId();
|
|
const auto next_pc = state.builder.AllocateId();
|
|
state.builder.AddFunction({spv::OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc,
|
|
dispatcher.continue_label});
|
|
const auto done = state.builder.AllocateId();
|
|
state.builder.AddFunction(
|
|
{spv::OpIEqual, TypeBool(state), done, pc, ConstantU32(ctx.state, UINT32_MAX)});
|
|
state.builder.AddFunction({spv::OpLoopMerge, dispatcher.merge_label, dispatcher.continue_label,
|
|
spv::LoopControlMaskNone});
|
|
state.builder.AddFunction(
|
|
{spv::OpBranchConditional, done, dispatcher.merge_label, dispatcher.select_label});
|
|
|
|
EmitLabel(state, dispatcher.select_label);
|
|
state.builder.AddFunction(
|
|
{spv::OpSelectionMerge, dispatcher.after_switch_label, spv::SelectionControlMaskNone});
|
|
std::vector<uint32_t> words {spv::OpSwitch, pc, dispatcher.after_switch_label};
|
|
for (size_t index = 1; index < state.program.blocks.size(); index++) {
|
|
words.push_back(state.program.block_info[index].id);
|
|
words.push_back(ctx.Label(state.program.blocks[index]));
|
|
}
|
|
state.builder.AddFunction(words);
|
|
std::vector<uint32_t> next_pc_words {spv::OpPhi, TypeU32(state), next_pc,
|
|
ConstantU32(state, UINT32_MAX), dispatcher.select_label};
|
|
|
|
for (size_t index = 1; index < state.program.blocks.size(); index++) {
|
|
EmitBlock(ctx, state.program.blocks[index],
|
|
[&](ValueEmitContext& lane, const IR::Inst& inst) {
|
|
EmitDispatcherInstruction(lane, dispatcher, inst);
|
|
});
|
|
const auto selected = EmitDispatcherNextPc(ctx, dispatcher, state.program.blocks[index],
|
|
state.program.block_info[index]);
|
|
next_pc_words.push_back(selected);
|
|
next_pc_words.push_back(state.current_label);
|
|
state.builder.AddFunction({spv::OpBranch, dispatcher.after_switch_label});
|
|
}
|
|
EmitLabel(state, dispatcher.after_switch_label);
|
|
state.builder.AddFunction(next_pc_words);
|
|
state.builder.AddFunction({spv::OpBranch, dispatcher.continue_label});
|
|
EmitLabel(state, dispatcher.continue_label);
|
|
state.builder.AddFunction({spv::OpBranch, dispatcher.header_label});
|
|
EmitLabel(state, dispatcher.merge_label);
|
|
EmitReturn(ctx);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
uint32_t ValueEmitContext::TypeId(IR::Type type) const {
|
|
switch (type) {
|
|
case IR::Type::U1: return TypeBool(state);
|
|
case IR::Type::U8:
|
|
case IR::Type::U16:
|
|
case IR::Type::U32:
|
|
case IR::Type::F16: return TypeU32(state);
|
|
case IR::Type::U64: return TypeU64(state);
|
|
case IR::Type::U32x2: return TypeU32Pair(state);
|
|
case IR::Type::F32: return TypeF32(state);
|
|
case IR::Type::U32x3: return TypeU32Vector(state, 3);
|
|
case IR::Type::U32x4: return TypeU32Vector(state, 4);
|
|
case IR::Type::F32x2: return TypeF32Vector(state, 2);
|
|
default: return 0;
|
|
}
|
|
}
|
|
|
|
uint32_t ValueEmitContext::Def(IR::Value value) {
|
|
value = value.Resolve();
|
|
if (value.IsImmediate()) {
|
|
switch (value.GetType()) {
|
|
case IR::Type::U1: return ConstantBool(state, value.U1());
|
|
case IR::Type::U8: return ConstantU32(state, value.U8());
|
|
case IR::Type::U16: return ConstantU32(state, value.U16());
|
|
case IR::Type::U32: return ConstantU32(state, value.U32());
|
|
case IR::Type::U64: return ConstantU64(state, value.U64());
|
|
case IR::Type::F16: return ConstantU32(state, value.F16Bits());
|
|
case IR::Type::F32:
|
|
return ConstantF32(state, std::bit_cast<uint32_t>(value.F32Value()));
|
|
default: break;
|
|
}
|
|
}
|
|
const auto* inst = value.ResolveInstruction();
|
|
if (inst == nullptr) {
|
|
Fail("direct SPIR-V emitter received a non-value argument");
|
|
}
|
|
if (dispatcher_spills != nullptr && state.current_block != nullptr &&
|
|
inst->Parent() != state.current_block) {
|
|
if (const auto found = dispatcher_spills->find(inst); found != dispatcher_spills->end()) {
|
|
if (const auto loaded = dispatcher_block_loads.find(inst);
|
|
loaded != dispatcher_block_loads.end() &&
|
|
loaded->second.first == state.current_label) {
|
|
return loaded->second.second;
|
|
}
|
|
const auto id = state.builder.AllocateId();
|
|
state.builder.AddFunction({spv::OpLoad, TypeId(inst->GetType()), id, found->second});
|
|
dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id});
|
|
return id;
|
|
}
|
|
}
|
|
return Result(*inst);
|
|
}
|
|
|
|
uint32_t ValueEmitContext::Arg(const IR::Inst& inst, size_t index) {
|
|
return Def(inst.Arg(index));
|
|
}
|
|
|
|
uint32_t ValueEmitContext::HalfArg(const IR::Inst& inst, size_t index, uint32_t lane_half) {
|
|
return lane_half == half ? Arg(inst, index) : other_half->Arg(inst, index);
|
|
}
|
|
|
|
uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
|
|
const auto ballot_type = TypeU32Vector(state, 4);
|
|
const auto scope = ConstantU32(state, spv::ScopeSubgroup);
|
|
const auto low = state.builder.AllocateId();
|
|
state.builder.AddFunction(
|
|
{spv::OpGroupNonUniformBallot, ballot_type, low, scope,
|
|
other_half == nullptr || half == 0 ? Def(predicate) : other_half->Def(predicate)});
|
|
if (other_half == nullptr) {
|
|
return low;
|
|
}
|
|
const auto high = state.builder.AllocateId();
|
|
const auto low_word = state.builder.AllocateId();
|
|
const auto high_word = state.builder.AllocateId();
|
|
const auto ballot = state.builder.AllocateId();
|
|
state.builder.AddFunction({spv::OpGroupNonUniformBallot, ballot_type, high, scope,
|
|
half == 1 ? Def(predicate) : other_half->Def(predicate)});
|
|
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low_word, low, 0});
|
|
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high_word, high, 0});
|
|
state.builder.AddFunction({spv::OpCompositeConstruct, ballot_type, ballot, low_word, high_word,
|
|
ConstantU32(state, 0), ConstantU32(state, 0)});
|
|
return ballot;
|
|
}
|
|
|
|
uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
|
|
if (other_half == nullptr) {
|
|
const auto result = state.builder.AllocateId();
|
|
state.builder.AddFunction({spv::OpGroupNonUniformBallotFindLSB, TypeU32(state), result,
|
|
ConstantU32(state, spv::ScopeSubgroup), ballot});
|
|
return result;
|
|
}
|
|
const auto low = state.builder.AllocateId();
|
|
const auto high = state.builder.AllocateId();
|
|
const auto low_first = state.builder.AllocateId();
|
|
const auto high_first = state.builder.AllocateId();
|
|
const auto low_active = state.builder.AllocateId();
|
|
const auto result = state.builder.AllocateId();
|
|
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, ballot, 0});
|
|
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high, ballot, 1});
|
|
state.builder.AddFunction(
|
|
{spv::OpExtInst, TypeU32(state), low_first, GlslStd450(state), GLSLstd450FindILsb, low});
|
|
state.builder.AddFunction(
|
|
{spv::OpExtInst, TypeU32(state), high_first, GlslStd450(state), GLSLstd450FindILsb, high});
|
|
state.builder.AddFunction(
|
|
{spv::OpINotEqual, TypeBool(state), low_active, low, ConstantU32(state, 0)});
|
|
state.builder.AddFunction({spv::OpSelect, TypeU32(state), result, low_active, low_first,
|
|
EmitAddU32(state, high_first, ConstantU32(state, 32))});
|
|
return result;
|
|
}
|
|
|
|
uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) {
|
|
const auto type = TypeId(inst.Arg(index).GetType());
|
|
const auto scope = ConstantU32(state, spv::ScopeSubgroup);
|
|
const auto low = state.builder.AllocateId();
|
|
if (other_half == nullptr) {
|
|
state.builder.AddFunction(
|
|
{spv::OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), lane});
|
|
return low;
|
|
}
|
|
const auto physical_lane =
|
|
EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 31));
|
|
const auto high = state.builder.AllocateId();
|
|
const auto in_high = state.builder.AllocateId();
|
|
const auto value = state.builder.AllocateId();
|
|
state.builder.AddFunction(
|
|
{spv::OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0), physical_lane});
|
|
state.builder.AddFunction(
|
|
{spv::OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1), physical_lane});
|
|
state.builder.AddFunction(
|
|
{spv::OpINotEqual, TypeBool(state), in_high,
|
|
EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 32)),
|
|
ConstantU32(state, 0)});
|
|
state.builder.AddFunction({spv::OpSelect, type, value, in_high, high, low});
|
|
return value;
|
|
}
|
|
|
|
uint32_t ValueEmitContext::Result(const IR::Inst& inst) {
|
|
if (const auto found = definitions.find(&inst); found != definitions.end()) {
|
|
return found->second;
|
|
}
|
|
const auto id = state.builder.AllocateId();
|
|
definitions.emplace(&inst, id);
|
|
return id;
|
|
}
|
|
|
|
uint32_t ValueEmitContext::Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
|
|
std::initializer_list<uint32_t> args) {
|
|
const auto type_id = TypeId(type);
|
|
const auto result = Result(inst);
|
|
std::vector<uint32_t> words {opcode, type_id, result};
|
|
words.insert(words.end(), args.begin(), args.end());
|
|
state.builder.AddFunction(words);
|
|
return result;
|
|
}
|
|
|
|
uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
|
|
if (const auto found = definitions.find(&inst); found != definitions.end()) {
|
|
if (found->second != value) {
|
|
state.builder.AddFunction(
|
|
{spv::OpCopyObject, TypeId(inst.GetType()), found->second, value});
|
|
}
|
|
return found->second;
|
|
}
|
|
definitions.emplace(&inst, value);
|
|
return value;
|
|
}
|
|
|
|
uint32_t ValueEmitContext::ResourceIndex(IR::Value value, IR::ValueOpcode opcode) {
|
|
const auto* inst = value.ResolveInstruction();
|
|
if (inst == nullptr || inst->GetOpcode() != opcode) {
|
|
Fail("typed resource handle has the wrong producer");
|
|
}
|
|
return inst->Flags<uint32_t>();
|
|
}
|
|
|
|
const IR::Inst* ValueEmitContext::ImageAddress(IR::Value value) {
|
|
const auto* inst = value.ResolveInstruction();
|
|
if (inst == nullptr || inst->GetOpcode() != IR::ValueOpcode::MakeImageAddress) {
|
|
Fail("typed image address was not constructed by MakeImageAddress");
|
|
}
|
|
return inst;
|
|
}
|
|
|
|
const IR::MemoryInfo& ValueEmitContext::Memory(const IR::Inst& inst) const {
|
|
return state.program.memory_info.at(inst.Flags<IR::MemoryFlags>().index);
|
|
}
|
|
|
|
const IR::ExportInfo& ValueEmitContext::Export(const IR::Inst& inst) const {
|
|
return state.program.export_info.at(inst.Flags<IR::ExportFlags>().index);
|
|
}
|
|
|
|
uint32_t ValueEmitContext::Label(const IR::Block* block) const {
|
|
return state.labels.at(block);
|
|
}
|
|
|
|
[[noreturn]] void ValueEmitContext::Fail(const char* reason) const {
|
|
EXIT("SPIR-V emission failed: hash=0x%016" PRIx64 " stage=%u reason=%s\n",
|
|
state.program.shader_hash, static_cast<unsigned>(state.program.stage), reason);
|
|
std::abort();
|
|
}
|
|
|
|
[[noreturn]] void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) const {
|
|
EXIT("SPIR-V emission failed: hash=0x%016" PRIx64 " stage=%u opcode=%s reason=%s\n",
|
|
state.program.shader_hash, static_cast<unsigned>(state.program.stage),
|
|
IR::ValueOpcodeName(inst.GetOpcode()), reason);
|
|
std::abort();
|
|
}
|
|
|
|
void EmitProgram(EmitterState& state) {
|
|
const auto& program = state.program;
|
|
ValueEmitContext ctx(state);
|
|
ValueEmitContext high(state);
|
|
if (state.lane_count == 2) {
|
|
ctx.other_half = &high;
|
|
high.other_half = &ctx;
|
|
high.half = 1;
|
|
}
|
|
std::optional<DispatcherFunctionState> dispatcher;
|
|
if (state.program.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
|
|
state.pixel_valid_mask_variable = state.builder.AllocateId();
|
|
state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
|
|
}
|
|
for (const auto* block: program.blocks) {
|
|
const auto label = state.builder.AllocateId();
|
|
state.labels.emplace(block, label);
|
|
}
|
|
if (state.program.dispatcher_fallback) {
|
|
auto& dispatch = dispatcher.emplace();
|
|
for (const auto* block: program.blocks) {
|
|
for (const auto& inst: *block) {
|
|
if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
|
|
continue;
|
|
}
|
|
if (SpillPointerType(ctx, inst.GetType()) == 0) {
|
|
ctx.Fail(inst, "cannot be stored by the dispatcher");
|
|
break;
|
|
}
|
|
dispatch.spills[0].emplace(&inst, state.builder.AllocateId());
|
|
}
|
|
}
|
|
const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) {
|
|
value = value.Resolve();
|
|
const auto* definition = value.TryInstruction();
|
|
if (definition == nullptr || definition->Parent() == consumer ||
|
|
definition->Parent() == program.blocks.front()) {
|
|
return;
|
|
}
|
|
if (SpillPointerType(ctx, definition->GetType()) == 0) {
|
|
ctx.Fail(*definition, "cannot be stored by the dispatcher");
|
|
return;
|
|
}
|
|
if (!dispatch.spills[0].contains(definition)) {
|
|
dispatch.spills[0].emplace(definition, state.builder.AllocateId());
|
|
}
|
|
};
|
|
for (const auto* block: program.blocks) {
|
|
for (const auto& inst: *block) {
|
|
for (size_t index = 0; index < inst.NumArgs(); index++) {
|
|
const auto* consumer =
|
|
inst.GetOpcode() == IR::ValueOpcode::Phi ? inst.PhiBlock(index) : block;
|
|
mark_cross_block(inst.Arg(index), consumer);
|
|
}
|
|
}
|
|
}
|
|
for (size_t index = 0; index < program.blocks.size(); index++) {
|
|
mark_cross_block(program.block_info[index].condition, program.blocks[index]);
|
|
mark_cross_block(program.block_info[index].indirect_target, program.blocks[index]);
|
|
}
|
|
dispatch.header_label = state.builder.AllocateId();
|
|
dispatch.select_label = state.builder.AllocateId();
|
|
dispatch.after_switch_label = state.builder.AllocateId();
|
|
dispatch.continue_label = state.builder.AllocateId();
|
|
dispatch.merge_label = state.builder.AllocateId();
|
|
ctx.dispatcher_spills = &dispatch.spills[0];
|
|
if (state.lane_count == 2) {
|
|
for (const auto& [inst, id]: dispatch.spills[0]) {
|
|
dispatch.spills[1].emplace(inst, state.builder.AllocateId());
|
|
}
|
|
high.dispatcher_spills = &dispatch.spills[1];
|
|
}
|
|
}
|
|
DefineGetBdaPointer(state);
|
|
for (const auto* block: program.blocks) {
|
|
if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
|
|
return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
|
|
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
|
|
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
|
|
})) {
|
|
ctx.scratch_u32_variable = state.builder.AllocateId();
|
|
if (state.lane_count == 2) {
|
|
high.scratch_u32_variable = state.builder.AllocateId();
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
state.builder.AddFunction({spv::OpFunction, TypeVoid(state),
|
|
state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func,
|
|
spv::FunctionControlMaskNone, TypeFunction(state)});
|
|
EmitLabel(state, state.entry_label);
|
|
if (state.requirements.function_lds) {
|
|
state.builder.AddFunction(
|
|
{spv::OpVariable,
|
|
TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsDwordCount(state)),
|
|
state.lds_variable, spv::StorageClassFunction});
|
|
}
|
|
if (state.requirements.function_scratch) {
|
|
for (uint32_t half = 0; half < state.lane_count; half++) {
|
|
state.builder.AddFunction({spv::OpVariable,
|
|
TypeU32ArrayPointer(state, spv::StorageClassFunction,
|
|
state.program.scratch_dwords),
|
|
state.scratch_variable[half], spv::StorageClassFunction});
|
|
}
|
|
}
|
|
if (state.pixel_valid_mask_variable != 0) {
|
|
state.builder.AddFunction({spv::OpVariable,
|
|
TypePointer(state, spv::StorageClassFunction, TypeU32(state)),
|
|
state.pixel_valid_mask_variable, spv::StorageClassFunction});
|
|
}
|
|
for (uint32_t half = 0; half < state.lane_count; half++) {
|
|
auto& lane = half == 0 ? ctx : high;
|
|
if (state.program.dispatcher_fallback) {
|
|
for (const auto* block: program.blocks) {
|
|
for (const auto& inst: *block) {
|
|
if (const auto found = dispatcher->spills[half].find(&inst);
|
|
found != dispatcher->spills[half].end()) {
|
|
state.builder.AddFunction({spv::OpVariable,
|
|
SpillPointerType(lane, inst.GetType()),
|
|
found->second, spv::StorageClassFunction});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (lane.scratch_u32_variable != 0) {
|
|
state.builder.AddFunction(
|
|
{spv::OpVariable, TypePointer(state, spv::StorageClassFunction, TypeU32(state)),
|
|
lane.scratch_u32_variable, spv::StorageClassFunction});
|
|
}
|
|
}
|
|
if (state.gds_variable != 0) {
|
|
state.gds_length = state.builder.AllocateId();
|
|
state.builder.AddFunction(
|
|
{spv::OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0});
|
|
}
|
|
if (state.pixel_valid_mask_variable != 0) {
|
|
state.builder.AddFunction(
|
|
{spv::OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
|
|
}
|
|
EmitMemoryOffsets(state);
|
|
if (program.blocks.empty()) {
|
|
EmitReturn(ctx);
|
|
} else if (state.program.dispatcher_fallback) {
|
|
EmitDispatcherFunction(ctx, *dispatcher);
|
|
} else {
|
|
EmitStructuredFunction(ctx);
|
|
}
|
|
state.builder.AddFunction({spv::OpFunctionEnd});
|
|
if (state.program.stage == ShaderType::Mesh) {
|
|
EmitMeshEntryPoint(state);
|
|
}
|
|
}
|
|
|
|
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|