gpu: compile RDNA2 tessellation through native LS HS and TES stages

This commit is contained in:
nmzik
2026-09-13 07:45:08 +04:00
parent 6d1ba5870a
commit 7516068662
26 changed files with 1035 additions and 161 deletions
@@ -47,6 +47,9 @@ const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Compute: return "CS";
case ShaderType::Vertex: return "VS";
case ShaderType::Local: return "LS";
case ShaderType::TessellationControl: return "HS";
case ShaderType::TessellationEvaluation: return "TES";
case ShaderType::Mesh: return "MS";
case ShaderType::Pixel: return "PS";
default: return "unknown";
@@ -211,6 +214,8 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded
const ShaderVertexInputInfo* input_info,
uint32_t user_data_base, uint32_t user_data_count,
uint32_t wave_size) {
const uint32_t vertex_index_reg = input_info->logical_stage == ShaderType::Local ? 2u : 5u;
const uint32_t instance_index_reg = input_info->logical_stage == ShaderType::Local ? 5u : 8u;
EmbeddedFetchData data;
data.loads.reserve(input_info->resources_num);
int32_t vertex_offset_candidate = -1;
@@ -251,16 +256,17 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded
// corresponding direct-draw offsets before fetching.
const bool vertex_index_accumulator =
IsDecodedVgpr(inst.dst) &&
(inst.dst.reg == 0 || (user_data_base == 8 && inst.dst.reg == 5));
(inst.dst.reg == 0 || (user_data_base == 8 && inst.dst.reg == vertex_index_reg));
const bool instance_index_accumulator =
IsDecodedVgpr(inst.dst) && (inst.dst.reg == (user_data_base == 8 ? 8u : 3u));
IsDecodedVgpr(inst.dst) &&
(inst.dst.reg == (user_data_base == 8 ? instance_index_reg : 3u));
uint32_t sad_zero = 0;
const bool index_offset_add =
(vertex_index_accumulator || instance_index_accumulator) &&
IsDecodedSgpr(inst.src0) &&
(vertex_index_accumulator || instance_index_accumulator) && IsDecodedSgpr(inst.src0) &&
((inst.opcode == Decoder::Opcode::V_ADD_I32 && IsDecodedVgpr(inst.src1) &&
inst.src1.reg == inst.dst.reg) ||
(user_data_base == 8 && (inst.dst.reg == 5 || inst.dst.reg == 8) &&
(user_data_base == 8 &&
(inst.dst.reg == vertex_index_reg || inst.dst.reg == instance_index_reg) &&
inst.opcode == Decoder::Opcode::V_SAD_U32 && IsDecodedVgpr(inst.src2) &&
inst.src2.reg == inst.dst.reg &&
TryDecodedOperandConstant(sgprs, inst.src1, sad_zero) && sad_zero == 0));
@@ -391,8 +397,8 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded
ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg);
}
if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgpr_is_index.size() &&
IsDecodedVgpr(inst.src0) && inst.src0.reg == 8 &&
IsDecodedVgpr(inst.src1) && inst.src1.reg == 5) {
IsDecodedVgpr(inst.src0) && inst.src0.reg == instance_index_reg &&
IsDecodedVgpr(inst.src1) && inst.src1.reg == vertex_index_reg) {
vgpr_is_index[inst.dst.reg] = true;
}
} else if (IsEmbeddedFetchAttribPropagationAlu(inst)) {
@@ -459,9 +465,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded
return data;
}
Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<const uint32_t> back,
std::vector<uint32_t>& joined_code) {
EXIT_IF(back.empty());
Decoder::Program DecodeFrontProgram(std::span<const uint32_t> front) {
Decoder::Program result;
uint32_t front_words = 0;
while (front_words < front.size()) {
@@ -477,7 +481,16 @@ Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<c
}
EXIT_IF(result.instructions.empty() ||
result.instructions.back().opcode != Decoder::Opcode::S_SETPC_B64);
joined_code.assign(front.begin(), front.begin() + front_words);
result.code = front.first(front_words);
return result;
}
Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<const uint32_t> back,
std::vector<uint32_t>& joined_code) {
EXIT_IF(back.empty());
auto result = DecodeFrontProgram(front);
const auto front_words = static_cast<uint32_t>(result.code.size());
joined_code.assign(result.code.begin(), result.code.end());
joined_code.insert(joined_code.end(), back.begin(), back.end());
// The merged-stage ABI passes the back shader in s[6:7]. Give that handoff an
// ordinary CFG edge, retaining both bodies in one register and LDS lifetime.
@@ -500,12 +513,224 @@ Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<c
} // namespace
void AnalyzeTessellationPrograms(std::span<const uint32_t> local, std::span<const uint32_t> control,
ShaderTessellationInputInfo& info) {
using namespace Decoder;
const auto constant = [](const Operand& operand, uint32_t value) {
return (operand.kind == OperandKind::IntegerInlineConstant ||
operand.kind == OperandKind::LiteralConstant) &&
operand.value == value;
};
auto decoded = DecodeFrontProgram(local);
uint32_t local_address = UINT32_MAX;
for (const auto& inst: decoded.instructions) {
if (inst.opcode == Opcode::V_MUL_U32_U24 && inst.src1.kind == OperandKind::Vgpr &&
inst.src1.reg == 3u &&
(inst.src0.kind == OperandKind::IntegerInlineConstant ||
inst.src0.kind == OperandKind::LiteralConstant)) {
EXIT_IF(info.ls_stride != 0u && info.ls_stride != inst.src0.value);
info.ls_stride = inst.src0.value;
local_address = inst.dst.reg;
}
if (inst.family == Family::DS) {
EXIT_NOT_IMPLEMENTED(inst.src0.kind != OperandKind::Vgpr ||
inst.src0.reg != local_address);
}
}
decoded = {};
DecodeProgram(control, decoded);
uint32_t control_point = UINT32_MAX;
uint32_t control_address = UINT32_MAX;
for (const auto& inst: decoded.instructions) {
if (inst.opcode == Opcode::V_BFE_U32 && inst.src0.kind == OperandKind::Vgpr &&
inst.src0.reg == 1u && constant(inst.src1, 8u) && constant(inst.src2, 5u)) {
control_point = inst.dst.reg;
}
if (inst.opcode == Opcode::V_LSHL_ADD_U32 && inst.src0.kind == OperandKind::Vgpr &&
inst.src0.reg == control_point &&
(inst.src1.kind == OperandKind::IntegerInlineConstant ||
inst.src1.kind == OperandKind::LiteralConstant) &&
inst.src1.value < 32u) {
info.hs_stride = 1u << inst.src1.value;
control_address = inst.dst.reg;
}
}
EXIT_NOT_IMPLEMENTED(local_address == UINT32_MAX || control_address == UINT32_MAX ||
info.ls_stride == 0u || info.hs_stride == 0u ||
(info.ls_stride & 3u) != 0u || (info.hs_stride & 3u) != 0u);
LOGF("Tessellation interface: input_cp=%u output_cp=%u ls_stride=%u hs_stride=%u\n",
info.input_control_points, info.output_control_points, info.ls_stride, info.hs_stride);
}
namespace {
const IR::Inst* TessellationBufferBase(const IR::Inst& inst) {
if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::None || inst.NumArgs() <= 3u)
return nullptr;
const auto* base = inst.Arg(3).Resolve().TryInstruction();
return base != nullptr && base->GetOpcode() == IR::ValueOpcode::TessellationBase ? base
: nullptr;
}
IR::Value ActiveAddress(IR::Block& block, IR::Block::iterator before, IR::Value value,
IR::Value predicate, std::unordered_map<IR::Inst*, IR::Value>& resolved) {
using namespace IR;
value = value.Resolve();
auto* source = value.TryInstruction();
if (source == nullptr) return value;
if (const auto found = resolved.find(source); found != resolved.end()) return found->second;
if (source->GetOpcode() == ValueOpcode::SelectU32) {
return source->Arg(0).Resolve() == predicate.Resolve()
? ActiveAddress(block, before, source->Arg(1), predicate, resolved)
: value;
}
if (source->GetOpcode() != ValueOpcode::IAdd32 && source->GetOpcode() != ValueOpcode::ISub32)
return value;
const auto lhs = ActiveAddress(block, before, source->Arg(0), predicate, resolved);
const auto rhs = ActiveAddress(block, before, source->Arg(1), predicate, resolved);
if (lhs != source->Arg(0).Resolve() || rhs != source->Arg(1).Resolve()) {
auto copy = block.PrependNewInst(before, source->GetOpcode(), {lhs, rhs},
source->Flags<uint64_t>());
value = Value(&*copy);
}
resolved.emplace(source, value);
return value;
}
void LowerTessellationMemory(IR::Program& program, const CompileOptions& options) {
using namespace IR;
if (options.stage != ShaderType::Local && options.stage != ShaderType::TessellationControl &&
options.stage != ShaderType::TessellationEvaluation) {
return;
}
const auto& tess = options.input_info.vertex->tess;
// Ring addresses can reuse data VGPRs. Their inactive values are irrelevant to
// a store guarded by the same EXEC predicate, but must remain intact elsewhere.
for (auto* block: program.blocks) {
for (auto it = block->begin(); it != block->end(); ++it) {
if (TessellationBufferBase(*it) == nullptr) continue;
std::unordered_map<Inst*, Value> resolved;
it->SetArg(
2, ActiveAddress(*block, it, it->Arg(2), it->Arg(it->NumArgs() - 1u), resolved));
}
}
ConstantPropagationPass(program.blocks);
uint32_t reads = 0, writes = 0, factors = 0;
for (auto* block: program.blocks) {
for (auto it = block->begin(); it != block->end(); ++it) {
auto& inst = *it;
const auto shared = SharedAccessOf(inst.GetOpcode());
const auto buffer = BufferAccessOf(inst.GetOpcode());
if (shared == SharedAccess::None && buffer == BufferAccess::None) {
continue;
}
const auto& memory = program.memory_info.at(inst.Flags<MemoryFlags>().index);
bool write = false;
uint32_t components = 0;
TessellationAttribute kind;
Value address, predicate;
if (shared != SharedAccess::None && memory.kind == ResourceKind::Lds) {
write = shared == SharedAccess::Write;
EXIT_NOT_IMPLEMENTED(memory.data_bits != 32u ||
(options.stage == ShaderType::Local
? !write
: options.stage != ShaderType::TessellationControl ||
shared != SharedAccess::Read));
kind = write ? TessellationAttribute::LocalOutput
: TessellationAttribute::ControlInput;
components = SharedComponentCount(inst.GetOpcode());
address = inst.Arg(0);
predicate = inst.Arg(inst.NumArgs() - 1u);
} else if (buffer != BufferAccess::None && memory.kind == ResourceKind::Buffer) {
const auto* base = TessellationBufferBase(inst);
if (base == nullptr) {
continue;
}
EXIT_NOT_IMPLEMENTED(memory.data_bits != 32u || memory.formatted || memory.idxen ||
!memory.offen || buffer == BufferAccess::Atomic);
write = buffer == BufferAccess::Write;
components = BufferComponentCount(inst.GetOpcode());
address = inst.Arg(2).Resolve();
predicate = inst.Arg(inst.NumArgs() - 1u);
if (base->Arg(0).U32() == 1u) {
EXIT_NOT_IMPLEMENTED(!write ||
options.stage != ShaderType::TessellationControl);
kind = TessellationAttribute::Factor;
factors += components;
} else {
kind = write ? TessellationAttribute::ControlOutput
: TessellationAttribute::EvaluationInput;
EXIT_NOT_IMPLEMENTED(write
? options.stage != ShaderType::TessellationControl
: options.stage != ShaderType::TessellationEvaluation);
if (address.IsImmediate() &&
address.U32() >= tess.hs_stride * tess.output_control_points) {
EXIT_NOT_IMPLEMENTED(!write);
kind = TessellationAttribute::PatchOutput;
}
}
} else {
continue;
}
const auto emit = [&](ValueOpcode opcode, std::initializer_list<Value> args) {
return Value(&*block->PrependNewInst(it, opcode, args));
};
std::array<Value, 4> values;
for (uint32_t component = 0; component < components; component++) {
const auto offset = memory.offset + 4u * component;
const auto byte_address =
offset == 0u ? address : emit(ValueOpcode::IAdd32, {address, Value(offset)});
if (write) {
Value data = shared != SharedAccess::None ? inst.Arg(component + 1u)
: components == 1u
? inst.Arg(4)
: emit(components == 2u ? ValueOpcode::CompositeExtractU32x2
: components == 3u ? ValueOpcode::CompositeExtractU32x3
: ValueOpcode::CompositeExtractU32x4,
{inst.Arg(4), Value(component)});
emit(ValueOpcode::SetTessellationAttribute,
{Value(static_cast<uint32_t>(kind)), byte_address, data, predicate});
writes++;
} else {
values[component] =
emit(ValueOpcode::GetTessellationAttribute,
{Value(static_cast<uint32_t>(kind)), byte_address, predicate});
reads++;
}
}
if (!write) {
Value replacement = values[0];
if (components == 2u)
replacement =
emit(ValueOpcode::CompositeConstructU32x2, {values[0], values[1]});
if (components == 3u)
replacement = emit(ValueOpcode::CompositeConstructU32x3,
{values[0], values[1], values[2]});
if (components == 4u)
replacement = emit(ValueOpcode::CompositeConstructU32x4,
{values[0], values[1], values[2], values[3]});
inst.ReplaceUsesWith(replacement);
}
inst.Invalidate();
}
}
ConstantPropagationPass(program.blocks);
RemoveIdentities(program.blocks);
EliminateDeadCode(program.blocks);
LOGF("%s tessellation lowering: reads=%u writes=%u factors=%u\n", StageName(options.stage),
reads, writes, factors);
}
} // namespace
TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOptions& options) {
if (code.empty()) {
EXIT("shader recompiler input is empty\n");
}
if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex &&
options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh) {
options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh &&
options.stage != ShaderType::Local && options.stage != ShaderType::TessellationControl &&
options.stage != ShaderType::TessellationEvaluation) {
EXIT("shader recompiler received unsupported stage %u\n",
static_cast<unsigned>(options.stage));
}
@@ -525,6 +750,12 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
std::vector<uint32_t> joined_code;
if (!options.back_code.empty()) {
decoded = DecodeFusedProgram(code, options.back_code, joined_code);
} else if (options.stage == ShaderType::Local) {
decoded = DecodeFrontProgram(code);
// The separately compiled hull half runs in the next Vulkan stage.
auto& handoff = decoded.instructions.back();
handoff.opcode = Decoder::Opcode::S_ENDPGM;
handoff.src_count = 0;
} else {
Decoder::DecodeProgram(code, decoded);
}
@@ -568,8 +799,8 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
}
EmbeddedFetchData embedded_fetch;
if (options.stage == ShaderType::Vertex && options.input_info.vertex != nullptr &&
options.input_info.vertex->fetch_embedded) {
if ((options.stage == ShaderType::Vertex || options.stage == ShaderType::Local) &&
options.input_info.vertex != nullptr && options.input_info.vertex->fetch_embedded) {
embedded_fetch = DetectEmbeddedVertexFetch(
decoded, options.input_info.vertex, options.user_data_base,
static_cast<uint32_t>(options.user_data.size()), options.wave_size);
@@ -608,6 +839,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
IR::RemoveIdentities(ir.blocks);
IR::EliminateDeadCode(ir.blocks);
}
LowerTessellationMemory(ir, options);
IR::BuildSrtPlan(ir);
IR::EliminateDeadCode(ir.blocks);
IR::TrackResources(ir);
@@ -37,6 +37,9 @@ struct CompileResult {
IR::Program program;
};
void AnalyzeTessellationPrograms(std::span<const uint32_t> local, std::span<const uint32_t> control,
ShaderTessellationInputInfo& info);
[[nodiscard]] TranslateResult TranslateProgram(std::span<const uint32_t> code,
const CompileOptions& options);
[[nodiscard]] CompileResult CompileProgram(TranslateResult translated,
@@ -277,7 +277,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
break;
}
case IR::ValueOpcode::LaneId:
requirements.subgroup_local_invocation_id = true;
requirements.subgroup_local_invocation_id |=
program.stage != ShaderType::TessellationControl;
break;
case IR::ValueOpcode::ImageQueryLod: requirements.compute_derivatives = true; break;
case IR::ValueOpcode::ImageGatherRaw:
@@ -306,8 +307,10 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
using namespace Emitter;
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
program.stage != ShaderType::Pixel && program.stage != ShaderType::Mesh) {
Fail(program, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
program.stage != ShaderType::Pixel && program.stage != ShaderType::Mesh &&
program.stage != ShaderType::Local && program.stage != ShaderType::TessellationControl &&
program.stage != ShaderType::TessellationEvaluation) {
Fail(program, "binary SPIR-V emitter received an unsupported shader stage");
}
if (!program.srt_plan_complete || !program.resource_tracking_complete ||
!program.shader_info_complete || !program.binding_layout_complete) {
@@ -250,7 +250,10 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
spv::ExecutionModel ExecutionModelForStage(ShaderType stage) {
switch (stage) {
case ShaderType::Local:
case ShaderType::Vertex: return spv::ExecutionModelVertex;
case ShaderType::TessellationControl: return spv::ExecutionModelTessellationControl;
case ShaderType::TessellationEvaluation: return spv::ExecutionModelTessellationEvaluation;
case ShaderType::Mesh: return spv::ExecutionModelMeshEXT; // MeshEXT
case ShaderType::Pixel: return spv::ExecutionModelFragment;
default: return spv::ExecutionModelGLCompute;
@@ -51,6 +51,7 @@ uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t c
return bits;
}
if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex ||
kind == IR::StageInputKind::InvocationId || kind == IR::StageInputKind::PrimitiveId ||
kind == IR::StageInputKind::Layer || kind == IR::StageInputKind::SampleId) {
const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
@@ -58,7 +59,7 @@ uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t c
state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
return bits;
}
if (kind == IR::StageInputKind::FragCoord) {
if (kind == IR::StageInputKind::FragCoord || kind == IR::StageInputKind::TessCoord) {
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
@@ -138,7 +139,7 @@ uint32_t EmitAttribute(EmitterState& state, uint32_t attr, uint32_t chan) {
if (input == nullptr || input->variable_id == 0) {
return ConstantU32(state, 0);
}
if (state.program.stage == ShaderType::Vertex) {
if (state.program.stage == ShaderType::Vertex || state.program.stage == ShaderType::Local) {
return EmitVertexParameterComponentU32(state, *input, chan & 3u);
}
const auto load_per_vertex = [&](uint32_t vertex) {
@@ -517,11 +518,95 @@ uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) {
void EmitVoid(ValueEmitContext&) {}
void EmitBarrier(EmitterState& state) {
const auto semantics =
spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
const auto tessellation = state.program.stage == ShaderType::TessellationControl;
const auto memory_scope = tessellation ? spv::ScopeInvocation : spv::ScopeWorkgroup;
const auto semantics = tessellation ? spv::MemorySemanticsMaskNone
: spv::MemorySemanticsAcquireReleaseMask |
spv::MemorySemanticsWorkgroupMemoryMask;
state.builder.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
ConstantU32(state, spv::ScopeWorkgroup),
ConstantU32(state, semantics));
ConstantU32(state, memory_scope), ConstantU32(state, semantics));
}
uint32_t EmitLaneId(EmitterState& state) {
return state.program.stage == ShaderType::TessellationControl
? EmitBuiltinU32(state, IR::StageInputKind::InvocationId, 0)
: EmitSubgroupLocalInvocationId(state);
}
namespace {
uint32_t TessellationPointer(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
using Attribute = IR::TessellationAttribute;
const auto kind = static_cast<Attribute>(inst.Arg(0).U32());
const auto& tess = state.input_info.vertex->tess;
const auto variable = state.tess_variables.at(static_cast<uint32_t>(kind));
EXIT_IF(variable == 0);
const auto input = kind == Attribute::ControlInput || kind == Attribute::EvaluationInput;
const auto storage = input ? spv::StorageClassInput : spv::StorageClassOutput;
const auto pointer = state.builder.AllocateId();
if (kind == Attribute::Factor) {
EXIT_NOT_IMPLEMENTED(!inst.Arg(1).IsImmediate());
const auto index = inst.Arg(1).U32() / 4u;
const auto outer = index < 3u;
EXIT_NOT_IMPLEMENTED(index >= 4u);
state.builder.AddFunction(spv::OpAccessChain, TypePointer(state, storage, TypeF32(state)),
pointer, outer ? variable : state.tess_inner_variable,
ConstantU32(state, outer ? index : index - 3u));
return pointer;
}
auto address = ctx.Arg(inst, 1);
if (kind == Attribute::PatchOutput) {
address =
EmitBinaryU32(state, spv::OpISub, address, ConstantU32(state, state.tess_patch_base));
}
const bool local = kind == Attribute::LocalOutput || kind == Attribute::ControlInput;
const auto stride = local ? tess.ls_stride : tess.hs_stride;
const auto offset = kind == Attribute::PatchOutput ? address
: EmitBinaryU32(state, spv::OpUMod, address,
ConstantU32(state, stride));
const auto attribute =
EmitBinaryU32(state, spv::OpShiftRightLogical, offset, ConstantU32(state, 4));
const auto component =
EmitBinaryU32(state, spv::OpBitwiseAnd,
EmitBinaryU32(state, spv::OpShiftRightLogical, offset, ConstantU32(state, 2)),
ConstantU32(state, 3));
const auto type = TypePointer(state, storage, TypeU32(state));
if (kind == Attribute::LocalOutput || kind == Attribute::PatchOutput) {
state.builder.AddFunction(spv::OpAccessChain, type, pointer, variable, attribute,
component);
} else {
const auto vertex =
kind == Attribute::ControlOutput
? EmitBuiltinU32(state, IR::StageInputKind::InvocationId, 0)
: EmitBinaryU32(state, spv::OpUDiv, address, ConstantU32(state, stride));
state.builder.AddFunction(spv::OpAccessChain, type, pointer, variable, vertex, attribute,
component);
}
return pointer;
}
} // namespace
uint32_t EmitGetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, 2), [&] {
const auto value = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), value,
TessellationPointer(ctx, inst));
return value;
});
}
void EmitSetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
EmitIfCondition(ctx.state, ctx.Arg(inst, 3), [&] {
auto value = ctx.Arg(inst, 2);
if (inst.Arg(0).U32() == static_cast<uint32_t>(IR::TessellationAttribute::Factor)) {
const auto floating = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(spv::OpBitcast, TypeF32(ctx.state), floating, value);
value = floating;
}
ctx.state.builder.AddFunction(spv::OpStore, TessellationPointer(ctx, inst), value);
});
}
uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -155,6 +155,8 @@ inline constexpr auto EmitInstPrefetch = EmitVoid;
void EmitBarrier(EmitterState& state);
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitGetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitSetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitGetUserData(EmitterState& state, IR::ScalarReg reg);
uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index);
uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst);
@@ -165,7 +167,7 @@ inline constexpr auto EmitUndefU64 = EmitUndefU1;
uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitWqmU64(EmitterState& state, uint32_t value);
inline constexpr auto EmitLaneId = EmitSubgroupLocalInvocationId;
uint32_t EmitLaneId(EmitterState& state);
uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate);
uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -271,6 +273,7 @@ inline constexpr auto EmitImageAtomicOr32 = EmitImage;
inline constexpr auto EmitImageAtomicXor32 = EmitImage;
void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitPhi = EmitUnreachable;
inline constexpr auto EmitTessellationBase = EmitUnreachable;
inline constexpr auto EmitGetThreadBitScalarRegister = EmitUnreachable;
inline constexpr auto EmitSetThreadBitScalarRegister = EmitUnreachable;
inline constexpr auto EmitGetScalarMaskTag = EmitUnreachable;
@@ -80,6 +80,10 @@ struct EmitterState {
Builder builder;
const IR::Program& program;
ShaderStageInputInfo input_info;
std::array<uint32_t, 6> tess_variables {};
uint32_t tess_inner_variable = 0;
uint32_t tess_patch_base = 0;
const SpirvRequirements requirements;
uint32_t lane_count = 1;
uint32_t lane_half = 0;
@@ -290,7 +290,8 @@ uint32_t GlslStd450(EmitterState& state) {
}
VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) {
if (state.program.stage != ShaderType::Vertex || location >= ShaderVertexInputInfo::RES_MAX ||
if ((state.program.stage != ShaderType::Vertex && state.program.stage != ShaderType::Local) ||
location >= ShaderVertexInputInfo::RES_MAX ||
location >= static_cast<uint32_t>(state.input_info.vertex->resources_num)) {
return VertexInputScalarKind::Float;
}
@@ -347,6 +348,9 @@ uint32_t DefineInterfaceVariable(EmitterState& state, uint32_t type, spv::Storag
uint32_t BuiltInForInput(IR::StageInputKind kind) {
switch (kind) {
case IR::StageInputKind::VertexIndex: return spv::BuiltInVertexIndex;
case IR::StageInputKind::InvocationId: return spv::BuiltInInvocationId;
case IR::StageInputKind::PrimitiveId: return spv::BuiltInPrimitiveId;
case IR::StageInputKind::TessCoord: return spv::BuiltInTessCoord;
case IR::StageInputKind::InstanceIndex: return spv::BuiltInInstanceIndex;
case IR::StageInputKind::FragCoord: return spv::BuiltInFragCoord;
case IR::StageInputKind::FrontFacing: return spv::BuiltInFrontFacing;
@@ -387,6 +391,8 @@ void DefineInputs(EmitterState& state) {
uint32_t type = TypeU32(state);
switch (input.kind) {
case IR::StageInputKind::VertexIndex:
case IR::StageInputKind::InvocationId:
case IR::StageInputKind::PrimitiveId:
case IR::StageInputKind::InstanceIndex:
case IR::StageInputKind::Layer:
case IR::StageInputKind::SampleId: type = TypeI32(state); break;
@@ -394,11 +400,13 @@ void DefineInputs(EmitterState& state) {
case IR::StageInputKind::LocalInvocationId:
case IR::StageInputKind::GlobalInvocationId: type = TypeU32Vector(state, 3); break;
case IR::StageInputKind::FragCoord: type = TypeF32Vector(state, 4); break;
case IR::StageInputKind::TessCoord:
case IR::StageInputKind::BaryCoordSmooth:
case IR::StageInputKind::BaryCoordNoPerspective: type = TypeF32Vector(state, 3); break;
case IR::StageInputKind::FrontFacing: type = TypeBool(state); break;
case IR::StageInputKind::Parameter:
if (state.program.stage == ShaderType::Vertex) {
if (state.program.stage == ShaderType::Vertex ||
state.program.stage == ShaderType::Local) {
type = VertexParameterScalarType(
state, VertexParameterScalarKind(state, input.location));
const auto components = VertexParameterComponentCount(input);
@@ -536,6 +544,70 @@ void DefineOutputs(EmitterState& state) {
}
}
void DefineTessellationInterfaces(EmitterState& state) {
using Attribute = IR::TessellationAttribute;
std::array<bool, 6> used {};
uint32_t patch_begin = UINT32_MAX, patch_end = 0;
for (const auto* block: state.program.blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() != IR::ValueOpcode::GetTessellationAttribute &&
inst.GetOpcode() != IR::ValueOpcode::SetTessellationAttribute)
continue;
const auto kind = inst.Arg(0).U32();
used.at(kind) = true;
if (kind == static_cast<uint32_t>(Attribute::PatchOutput)) {
EXIT_NOT_IMPLEMENTED(!inst.Arg(1).IsImmediate());
patch_begin = std::min(patch_begin, inst.Arg(1).U32());
patch_end = std::max(patch_end, inst.Arg(1).U32() + 4u);
}
}
}
if (std::ranges::none_of(used, [](bool value) { return value; })) return;
const auto& tess = state.input_info.vertex->tess;
const auto array = [&](uint32_t type, uint32_t count) {
return state.builder.Type(spv::OpTypeArray, type, ConstantU32(state, count));
};
for (uint32_t index = 0; index < used.size(); index++) {
if (!used[index]) continue;
const auto kind = static_cast<Attribute>(index);
const bool input = kind == Attribute::ControlInput || kind == Attribute::EvaluationInput;
const auto storage = input ? spv::StorageClassInput : spv::StorageClassOutput;
uint32_t type;
if (kind == Attribute::Factor) {
type = array(TypeF32(state), 4u);
} else if (kind == Attribute::PatchOutput) {
state.tess_patch_base = patch_begin;
type = array(TypeU32Vector(state, 4), (patch_end - patch_begin + 15u) / 16u);
} else {
const bool local = kind == Attribute::LocalOutput || kind == Attribute::ControlInput;
const auto stride = local ? tess.ls_stride : tess.hs_stride;
type = array(TypeU32Vector(state, 4), (stride + 15u) / 16u);
if (kind != Attribute::LocalOutput) {
type = array(type, local ? tess.input_control_points : tess.output_control_points);
}
}
auto& variable = state.tess_variables[index];
variable = DefineInterfaceVariable(state, type, storage, "tess_attributes");
if (kind == Attribute::Factor) {
state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBuiltIn,
spv::BuiltInTessLevelOuter);
state.tess_inner_variable =
DefineInterfaceVariable(state, array(TypeF32(state), 2u), storage, "tess_inner");
state.builder.AddAnnotation(spv::OpDecorate, state.tess_inner_variable,
spv::DecorationBuiltIn, spv::BuiltInTessLevelInner);
state.builder.AddAnnotation(spv::OpDecorate, state.tess_inner_variable,
spv::DecorationPatch);
} else {
state.builder.AddAnnotation(
spv::OpDecorate, variable, spv::DecorationLocation,
kind == Attribute::PatchOutput ? (tess.hs_stride + 15u) / 16u : 0u);
}
if (kind == Attribute::Factor || kind == Attribute::PatchOutput) {
state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationPatch);
}
}
}
} // namespace
void DefineModule(EmitterState& state) {
@@ -543,6 +615,7 @@ void DefineModule(EmitterState& state) {
state.program.info.outputs.size());
DefineInputs(state);
DefineOutputs(state);
DefineTessellationInterfaces(state);
DefineDescriptors(state);
if (state.requirements.function_lds) {
state.lds_variable = state.builder.AllocateId();
@@ -564,6 +637,21 @@ void DefineModule(EmitterState& state) {
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeOutputPrimitivesEXT,
state.input_info.vertex->mesh.max_primitives);
}
if (state.program.stage == ShaderType::TessellationControl ||
state.program.stage == ShaderType::TessellationEvaluation) {
const auto& tess = state.input_info.vertex->tess;
EXIT_NOT_IMPLEMENTED(tess.domain != 1u || tess.partitioning != 2u ||
tess.output_topology != 2u);
state.builder.RequireCapability(spv::CapabilityTessellation);
if (state.program.stage == ShaderType::TessellationControl) {
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeOutputVertices,
tess.output_control_points);
} else {
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeTriangles);
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSpacingFractionalOdd);
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeVertexOrderCw);
}
}
state.entry_label = state.builder.AllocateId();
state.builder.RequireCapability(spv::CapabilityShader);
@@ -898,11 +898,15 @@ void ValidateTranslateOptions(const TranslateOptions& options) {
if (options.wave_size != 32u && options.wave_size != 64u) {
EXIT("shader translation requires wave32 or wave64, got %u", options.wave_size);
}
if (options.embedded_fetch != nullptr && options.stage != ShaderType::Vertex) {
EXIT("embedded fetch requires the vertex shader stage");
if (options.embedded_fetch != nullptr && options.stage != ShaderType::Vertex &&
options.stage != ShaderType::Local) {
EXIT("embedded fetch requires a vertex or local shader");
}
switch (options.stage) {
case ShaderType::Vertex:
case ShaderType::Local:
case ShaderType::TessellationControl:
case ShaderType::TessellationEvaluation:
case ShaderType::Mesh:
if (options.input_info.vertex == nullptr) {
EXIT("vertex shader translation has no vertex input metadata");
@@ -941,6 +945,9 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
result.user_data_count = options.user_data_count;
switch (options.stage) {
case ShaderType::Vertex:
case ShaderType::Local:
case ShaderType::TessellationControl:
case ShaderType::TessellationEvaluation:
result.scratch_dwords = options.input_info.vertex->scratch_size_dwords;
break;
case ShaderType::Mesh:
@@ -1164,6 +1171,42 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
entry_ir.SetVectorReg(
static_cast<IR::VectorReg>(8),
entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
} else if (options.stage == ShaderType::Local) {
entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3), IR::U32(IR::Value(64u)));
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(2),
builtin(IR::StageInputKind::VertexIndex));
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(3), IR::U32(IR::Value(0u)));
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
builtin(IR::StageInputKind::InstanceIndex));
} else if (options.stage == ShaderType::TessellationControl) {
const auto& tess = options.input_info.vertex->tess;
entry_ir.SetScalarReg(
static_cast<IR::ScalarReg>(2),
IR::U32(entry_ir.Emit(IR::ValueOpcode::TessellationBase, {IR::Value(0u)})));
entry_ir.SetScalarReg(
static_cast<IR::ScalarReg>(4),
IR::U32(entry_ir.Emit(IR::ValueOpcode::TessellationBase, {IR::Value(1u)})));
entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3),
IR::U32(IR::Value(0x81010000u | tess.input_control_points |
(tess.output_control_points << 8u))));
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(0),
builtin(IR::StageInputKind::PrimitiveId));
entry_ir.SetVectorReg(
static_cast<IR::VectorReg>(1),
entry_ir.ShiftLeftLogical(builtin(IR::StageInputKind::InvocationId),
IR::U32(IR::Value(8u))));
} else if (options.stage == ShaderType::TessellationEvaluation) {
entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3), IR::U32(IR::Value(64u)));
entry_ir.SetScalarReg(
static_cast<IR::ScalarReg>(4),
IR::U32(entry_ir.Emit(IR::ValueOpcode::TessellationBase, {IR::Value(0u)})));
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
builtin(IR::StageInputKind::TessCoord, 0));
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(6),
builtin(IR::StageInputKind::TessCoord, 1));
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(7), IR::U32(IR::Value(0u)));
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(8),
builtin(IR::StageInputKind::PrimitiveId));
} else if (options.stage == ShaderType::Pixel) {
const auto* ps = options.input_info.pixel;
if (ps->ps_perspective_center_vgpr != UINT32_MAX) {
+17 -1
View File
@@ -150,8 +150,20 @@ struct SampledResourcePair {
bool operator==(const SampledResourcePair& other) const = default;
};
enum class TessellationAttribute {
LocalOutput,
ControlInput,
ControlOutput,
EvaluationInput,
PatchOutput,
Factor
};
enum class StageInputKind {
VertexIndex,
InvocationId,
PrimitiveId,
TessCoord,
InstanceIndex,
FragCoord,
FrontFacing,
@@ -287,8 +299,12 @@ static_assert(sizeof(PushData) == 128);
constexpr uint32_t NativePushConstantSize = sizeof(PushData);
[[nodiscard]] constexpr uint32_t NativeBinding(ShaderType stage, DescriptorBindingKind kind) {
const uint32_t group = stage == ShaderType::Pixel ? 1u
: stage == ShaderType::TessellationControl ? 2u
: stage == ShaderType::TessellationEvaluation ? 3u
: 0u;
return static_cast<uint32_t>(kind) +
(stage == ShaderType::Pixel ? static_cast<uint32_t>(DescriptorBindingKind::Count) : 0u);
group * static_cast<uint32_t>(DescriptorBindingKind::Count);
}
[[nodiscard]] constexpr ImageResourceClass ImageBindingResourceClass(DescriptorBindingKind kind) {
@@ -90,6 +90,7 @@ bool HasSideEffects(ValueOpcode opcode) {
case ValueOpcode::Reference:
case ValueOpcode::ReferenceU32:
case ValueOpcode::SetAttribute:
case ValueOpcode::SetTessellationAttribute:
case ValueOpcode::MeshAllocate:
case ValueOpcode::Barrier: return true;
default: return false;
@@ -10,6 +10,9 @@ VALUE_OPCODE(GetUserData, U32, ScalarReg)
VALUE_OPCODE(GetShaderBase, U64)
VALUE_OPCODE(MeshDrawParameter, U32, U32)
VALUE_OPCODE(MeshAllocate, Void, U32)
VALUE_OPCODE(TessellationBase, U32, U32)
VALUE_OPCODE(GetTessellationAttribute, U32, U32, U32, U1)
VALUE_OPCODE(SetTessellationAttribute, Void, U32, U32, U32, U1)
VALUE_OPCODE(GetBuiltin, U32, U32, U32)
VALUE_OPCODE(GetThreadBitScalarRegister, U1, ScalarReg)
VALUE_OPCODE(SetThreadBitScalarRegister, Void, ScalarReg, U1)
@@ -228,6 +228,15 @@ void FoldInstruction(Block& block, Block::iterator instruction,
const auto offset = Arg(inst, 1);
const auto count = Arg(inst, 2);
auto* source = value.TryInstruction();
if (source != nullptr && source->GetOpcode() == ValueOpcode::ShiftLeftLogical32 &&
IsImmediate(offset, Type::U32) && IsImmediate(count, Type::U32)) {
const auto shift = Arg(*source, 1);
if (IsImmediate(shift, Type::U32) && shift.U32() < 32u &&
offset.U32() <= shift.U32() && count.U32() <= shift.U32() - offset.U32()) {
Replace(inst, Value(0u));
return;
}
}
if (source != nullptr && source->GetOpcode() == ValueOpcode::GetBuiltin &&
source->Arg(0) == Value(static_cast<uint32_t>(StageInputKind::PackedAncillary)) &&
IsImmediate(offset, Type::U32) && IsImmediate(count, Type::U32) && count.U32() != 0u) {
@@ -43,6 +43,9 @@ void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t
void ValidateOptions(const Program& program, ShaderStageInputInfo input_info) {
switch (program.stage) {
case ShaderType::Vertex:
case ShaderType::Local:
case ShaderType::TessellationControl:
case ShaderType::TessellationEvaluation:
case ShaderType::Mesh:
if (input_info.vertex == nullptr) {
return Fail("vertex shader has no input metadata");
@@ -81,7 +84,8 @@ void ValidateValueReferences(const Program& program, ShaderStageInputInfo input_
!inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) {
return Fail("typed attribute reference is not constant");
}
if (program.stage == ShaderType::Vertex &&
if ((program.stage == ShaderType::Vertex ||
program.stage == ShaderType::Local) &&
(inst.Arg(1).U32() >= 4u ||
inst.Arg(0).U32() >=
static_cast<uint32_t>(input_info.vertex->resources_num))) {
@@ -124,6 +128,8 @@ void ValidateValueReferences(const Program& program, ShaderStageInputInfo input_
}
break;
case StageInputKind::VertexIndex:
case StageInputKind::InvocationId:
case StageInputKind::PrimitiveId:
case StageInputKind::InstanceIndex:
case StageInputKind::FrontFacing:
case StageInputKind::LocalInvocationIndex:
@@ -142,6 +148,7 @@ void ValidateValueReferences(const Program& program, ShaderStageInputInfo input_
return Fail("typed barycentric component is out of range");
}
break;
case StageInputKind::TessCoord:
case StageInputKind::WorkgroupId:
case StageInputKind::LocalInvocationId:
case StageInputKind::GlobalInvocationId:
@@ -161,8 +168,10 @@ void ValidateValueReferences(const Program& program, ShaderStageInputInfo input_
}
const auto& exp = program.export_info[index];
if (exp.kind == ExportTargetKind::Position && exp.index != 0 && exp.en != 0 &&
program.stage != ShaderType::Vertex && program.stage != ShaderType::Mesh) {
return Fail("auxiliary position export requires the vertex or mesh stage");
program.stage != ShaderType::Vertex && program.stage != ShaderType::Mesh &&
program.stage != ShaderType::TessellationEvaluation) {
return Fail("auxiliary position export requires a vertex, mesh, or "
"tessellation evaluation shader");
}
break;
}
@@ -249,6 +258,10 @@ void CollectComputeInputs(const ShaderComputeInputInfo* compute, ShaderInfo& inf
void CollectBuiltinInputs(const Program& program, ShaderInfo& info) {
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
if (program.stage == ShaderType::TessellationControl &&
inst.GetOpcode() == ValueOpcode::LaneId) {
AddInput(info, StageInputKind::InvocationId, 0, 1, "gl_InvocationID");
}
if (inst.GetOpcode() != ValueOpcode::GetBuiltin) {
continue;
}
@@ -260,6 +273,13 @@ void CollectBuiltinInputs(const Program& program, ShaderInfo& info) {
case StageInputKind::InstanceIndex:
AddInput(info, kind, 0, 1, "gl_InstanceIndex");
break;
case StageInputKind::InvocationId:
AddInput(info, kind, 0, 1, "gl_InvocationID");
break;
case StageInputKind::PrimitiveId:
AddInput(info, kind, 0, 1, "gl_PrimitiveID");
break;
case StageInputKind::TessCoord: AddInput(info, kind, 0, 3, "gl_TessCoord"); break;
case StageInputKind::FragCoord: AddInput(info, kind, 0, 4, "gl_FragCoord"); break;
case StageInputKind::FrontFacing:
AddInput(info, kind, 0, 1, "gl_FrontFacing");
@@ -380,7 +400,10 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) {
});
});
switch (program.stage) {
case ShaderType::Vertex: CollectVertexInputs(program, input_info.vertex, next); break;
case ShaderType::Vertex:
case ShaderType::Local: CollectVertexInputs(program, input_info.vertex, next); break;
case ShaderType::TessellationControl:
case ShaderType::TessellationEvaluation:
case ShaderType::Mesh: break;
case ShaderType::Pixel: CollectPixelInputs(program, input_info.pixel, next); break;
case ShaderType::Compute: CollectComputeInputs(input_info.compute, next); break;