raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-21 11:26:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

shader: run merged geometry stages as mesh shaders

Browse Source 
Co-authored-by: Brandon Strong <brandostrong@gmail.com>
Co-authored-by: fxpw <fxpw@outlook.com>
Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com>
This commit is contained in:
nmzik
2026-09-05 08:32:23 +04:00
parent 246212789f
commit 3213934d42
39 changed files with 1403 additions and 858 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp
+13
View File
@@ -2,6 +2,7 @@
#include "common/debug.h"
#include <algorithm>
#include <cstring>
namespace Libs::Graphics::ShaderRecompiler::Spirv {
@@ -190,6 +191,18 @@ void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, cons
std::vector<uint32_t> operands = {execution_model, entry_point};
AppendString(operands, name);
operands.insert(operands.end(), interfaces.begin(), interfaces.end());
if (m_version >= 0x00010400u) {
for (size_t offset = 0; offset < m_declarations.size();) {
const auto count = m_declarations[offset] >> 16u;
if ((m_declarations[offset] & 0xffffu) == 59u) {
const auto id = m_declarations[offset + 2u];
if (std::find(interfaces.begin(), interfaces.end(), id) == interfaces.end()) {
operands.push_back(id);
}
}
offset += count;
}
}
AppendInstruction(m_entry_points, 15u, operands);
}
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
+10 -3
View File
@@ -236,7 +236,8 @@ void AnalyzeProgramRequirements(IR::Program& program) {
if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
Fail(program, "shared operation has invalid resource kind");
}
if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) {
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh &&
kind == IR::ResourceKind::Lds) {
requirements.function_lds = true;
}
if (shared_access == IR::SharedAccess::Append ||
@@ -259,7 +260,6 @@ void AnalyzeProgramRequirements(IR::Program& program) {
break;
}
case IR::ValueOpcode::DppUpdateU32:
case IR::ValueOpcode::WqmMask:
case IR::ValueOpcode::WriteLane: {
MarkBallot();
requirements.subgroup_local_invocation_id = true;
@@ -308,7 +308,7 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
using namespace Emitter;
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
program.stage != ShaderType::Pixel) {
program.stage != ShaderType::Pixel && program.stage != ShaderType::Mesh) {
Fail(program, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
}
if (!program.srt_plan_complete || !program.resource_tracking_complete ||
@@ -321,6 +321,11 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
EmitterState state(program, input_info);
state.stage = program.stage;
state.wave_size = program.wave_size;
const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
state.lane_count =
workgroup != nullptr && program.wave_size == 64u && workgroup->host_subgroup_size == 32u
? 2u
: 1u;
state.inputs.reserve(program.info.inputs.size());
state.outputs.reserve(program.info.outputs.size());
state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());
@@ -329,6 +334,8 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
AllocateOutputVariables(state);
DefineModule(state);
EmitProgram(state, program);
state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
state.interface_variables);
auto binary = state.builder.Build();
if (binary.empty()) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp
+1
View File
@@ -278,6 +278,7 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
uint32_t ExecutionModelForStage(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return ExecutionModelVertex;
case ShaderType::Mesh: return 5365u; // MeshEXT
case ShaderType::Pixel: return ExecutionModelFragment;
default: return ExecutionModelGLCompute;
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
+65 -95
View File
@@ -16,26 +16,6 @@ bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t&
return true;
}
uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) {
const auto shifted_one = state.builder.AllocateId();
const auto merged_one = state.builder.AllocateId();
const auto shifted_two = state.builder.AllocateId();
const auto merged_two = state.builder.AllocateId();
const auto quad_bits = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), shifted_one, value, ConstantU32(state, 1)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one});
state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), shifted_two, merged_one, ConstantU32(state, 2)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two});
state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), quad_bits, merged_two, ConstantU32(state, 0x11111111u)});
state.builder.AddFunction(
{OpIMul, TypeU32(state), result, quad_bits, ConstantU32(state, 0x0fu)});
return result;
}
uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
const auto shifted_one = state.builder.AllocateId();
const auto merged_one = state.builder.AllocateId();
@@ -61,6 +41,24 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
if (kind == IR::StageInputKind::LocalInvocationIndex) {
return EmitLocalInvocationIndex(state);
}
if (state.lane_count == 2 && (kind == IR::StageInputKind::LocalInvocationId ||
kind == IR::StageInputKind::GlobalInvocationId)) {
const auto* cs = ShaderWorkgroupInput(state.stage, state.input_info);
uint32_t divisor = 1;
for (uint32_t axis = 0; axis < component; axis++) {
divisor *= std::max(cs->threads_num[axis], 1u);
}
const auto size = std::max(cs->threads_num[component], 1u);
const auto divided = EmitBinaryU32(state, OpUDiv, EmitLocalInvocationIndex(state),
ConstantU32(state, divisor));
const auto local = EmitBinaryU32(state, OpUMod, divided, ConstantU32(state, size));
if (kind == IR::StageInputKind::LocalInvocationId) {
return local;
}
const auto group = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, component);
return EmitAddU32(state, local,
EmitBinaryU32(state, OpIMul, group, ConstantU32(state, size)));
}
const auto variable = InputVariableForKind(state, kind);
if (variable == 0) {
return ConstantU32(state, 0);
@@ -106,40 +104,6 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
return EmitInputComponentU32(state, kind, component);
}
uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) {
auto& state = ctx.state;
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
ConstantU32(state, ScopeSubgroup), active});
const auto low = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
const auto wqm_low = EmitWqmWordU32(state, low);
const auto lane = EmitSubgroupLocalInvocationId(state);
uint32_t mask = wqm_low;
uint32_t bit_lane = lane;
if (state.wave_size == 64u) {
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
const auto wqm_high = EmitWqmWordU32(state, high);
const auto upper = state.builder.AllocateId();
mask = state.builder.AllocateId();
bit_lane = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 32)});
state.builder.AddFunction({OpSelect, TypeU32(state), mask, upper, wqm_high, wqm_low});
state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), bit_lane, lane, ConstantU32(state, 31)});
}
const auto bit = state.builder.AllocateId();
const auto hit = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), bit_lane});
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
state.builder.AddFunction({OpINotEqual, TypeBool(state), result, hit, ConstantU32(state, 0)});
return result;
}
uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags,
uint32_t exec) {
auto& state = ctx.state;
@@ -378,7 +342,10 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
const auto layer = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), layer, raw,
ConstantU32(state, 0x7ffu)});
state.builder.AddFunction({OpStore, state.layer_variable, layer});
const auto pointer = state.stage == ShaderType::Mesh
? MeshOutputPointer(state, IR::StageOutputKind::Layer)
: state.layer_variable;
state.builder.AddFunction({OpStore, pointer, layer});
continue;
}
if (!output.point_size && output.clip_distance == UINT32_MAX &&
@@ -454,12 +421,18 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
{OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
}
if (exp.kind == IR::ExportTargetKind::Null || exp.kind == IR::ExportTargetKind::Primitive ||
exp.en == 0u) {
if (exp.kind == IR::ExportTargetKind::Null || exp.en == 0u) {
return;
}
EmitIfCondition(state, exec, [&]() {
const auto data = ctx.Arg(inst, 0);
if (exp.kind == IR::ExportTargetKind::Primitive) {
if (state.stage == ShaderType::Mesh) {
state.builder.AddFunction(
{OpStore, MeshPrimitivePointer(state), ExportRawComponent(ctx, data, 0)});
}
return;
}
if (exp.kind == IR::ExportTargetKind::Position && exp.index != 0) {
EmitAuxPositionExport(ctx, data, exp);
return;
@@ -483,8 +456,9 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
}
return;
}
const auto variable = OutputVariableForExport(state, exp);
if (variable == 0) {
const auto variable =
state.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp);
if (state.stage != ShaderType::Mesh && variable == 0) {
return;
}
const bool uint_output = MrtOutputMode(state, exp) == 7u;
@@ -501,10 +475,16 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
value = mapped;
}
}
if (exp.kind == IR::ExportTargetKind::Position) {
if (state.stage == ShaderType::Vertex && state.input_info.vertex->clip_space.enabled) {
value = ConvertPositionToClipSpace(state, value);
}
if (exp.kind == IR::ExportTargetKind::Position &&
state.input_info.vertex->clip_space.enabled) {
value = ConvertPositionToClipSpace(state, value);
}
if (state.stage == ShaderType::Mesh) {
const auto kind = exp.kind == IR::ExportTargetKind::Position
? IR::StageOutputKind::Position
: IR::StageOutputKind::Parameter;
state.builder.AddFunction({OpStore, MeshOutputPointer(state, kind, exp.index), value});
} else if (exp.kind == IR::ExportTargetKind::Position) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)),
@@ -537,6 +517,19 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
ConstantU32(state, semantics)});
return true;
}
case IR::ValueOpcode::MeshAllocate: EmitMeshAllocate(ctx, inst); return true;
case IR::ValueOpcode::MeshDrawParameter: {
const auto index = inst.Arg(0).U32();
if (state.stage != ShaderType::Mesh || index >= IR::PushData::MeshDrawDwordCount) {
ctx.Fail(inst, "invalid mesh draw parameter");
}
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state),
pointer, state.push_constant_variable, ConstantU32(state, 0),
ConstantU32(state, index)});
state.builder.AddFunction({OpLoad, TypeU32(state), ctx.Result(inst), pointer});
return true;
}
case IR::ValueOpcode::GetUserData: {
const auto reg = inst.Arg(0).ScalarRegister();
uint32_t dword = 0;
@@ -561,17 +554,12 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
case IR::ValueOpcode::DppMoveU32: {
const auto flags = inst.Flags<IR::DppMoveFlags>();
const auto target = EmitDppTargetLane(state, flags.control);
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
target.lane});
const auto shuffled = ctx.Shuffle(inst, 0, target.lane);
if (flags.fetch_inactive) {
ctx.Define(inst, shuffled);
return true;
}
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
const auto ballot = ctx.Ballot(inst.Arg(1));
const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
const auto can_fetch = state.builder.AllocateId();
state.builder.AddFunction(
@@ -588,30 +576,18 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
case IR::ValueOpcode::WqmU64:
ctx.Define(inst, EmitWqmU64(ctx.state, ctx.Arg(inst, 0)));
return true;
case IR::ValueOpcode::WqmMask:
ctx.Define(inst, EmitWqm(ctx, ctx.Arg(inst, 0)));
return true;
case IR::ValueOpcode::LaneId:
ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
return true;
case IR::ValueOpcode::Ballot:
ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
return true;
case IR::ValueOpcode::Ballot: ctx.Define(inst, ctx.Ballot(inst.Arg(0))); return true;
case IR::ValueOpcode::ReadFirstLane: {
const auto ballot = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), lane,
ConstantU32(state, ScopeSubgroup), ballot});
ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), lane});
const auto ballot = ctx.Ballot(inst.Arg(1));
const auto lane = ctx.FirstLane(ballot);
ctx.Define(inst, ctx.Shuffle(inst, 0, lane));
return true;
}
case IR::ValueOpcode::ReadLane:
ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
ctx.Define(inst, ctx.Shuffle(inst, 0, ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::WriteLane: {
const auto hit = state.builder.AllocateId();
@@ -656,16 +632,10 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row_value, index});
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
target});
const auto shuffled = ctx.Shuffle(inst, 0, target);
uint32_t result = shuffled;
if (!flags.fetch_inactive) {
const auto source_exec = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeBool(state), source_exec,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 3),
target});
const auto source_exec = ctx.Shuffle(inst, 3, target);
result = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, TypeU32(state), result, source_exec, shuffled,
ConstantU32(state, 0)});
src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp
+6 -1
View File
@@ -126,7 +126,7 @@ uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable});
return value;
return state.lane_half == 0 ? value : EmitAddU32(state, value, ConstantU32(state, 32));
}
uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
@@ -167,6 +167,11 @@ uint32_t EmitLocalInvocationIndex(EmitterState& state) {
}
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeU32(state), value, variable});
if (state.lane_count == 2) {
const auto wave_base = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~31u));
return EmitAddU32(state, EmitAddU32(state, value, wave_base),
ConstantU32(state, state.lane_half * 32));
}
return value;
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+23 -3
View File
@@ -198,6 +198,7 @@ enum : uint32_t {
OpIMul = 132,
OpFMul = 133,
OpUDiv = 134,
OpUMod = 137,
OpFDiv = 136,
OpIAddCarry = 149,
OpUMulExtended = 151,
@@ -318,6 +319,7 @@ struct OutputBinding {
uint32_t location = 0;
uint32_t variable_id = 0;
std::string debug_name;
uint32_t mesh_data_variable = 0;
};
using ImageDimension = Decoder::ImageDimension;
@@ -345,8 +347,8 @@ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
struct EmitterState {
EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_)
: program(program_), input_info(input_info_),
requirements(*program_.spirv_requirements) {}
: builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u),
program(program_), input_info(input_info_), requirements(*program_.spirv_requirements) {}
Builder builder;
const IR::Program& program;
@@ -354,6 +356,8 @@ struct EmitterState {
const IR::SpirvRequirements& requirements;
ShaderType stage = ShaderType::Unknown;
uint32_t wave_size = 64;
uint32_t lane_count = 1;
uint32_t lane_half = 0;
uint32_t storage_buffer_variable = 0;
uint32_t storage_buffer_u64_variable = 0;
std::array<uint32_t, IR::ShaderInfo::MaxBuffers> memory_byte_offsets {};
@@ -366,10 +370,15 @@ struct EmitterState {
uint32_t shader_data_storage_variable = 0;
uint32_t flattened_srt_variable = 0;
uint32_t lds_variable = 0;
uint32_t scratch_variable = 0;
std::array<uint32_t, 2> scratch_variable {};
std::array<uint32_t, IR::ImageBindingCount> image_variables {};
uint32_t sampler_variable = 0;
uint32_t main_func = 0;
uint32_t mesh_guest_func = 0;
uint32_t mesh_allocation = 0;
uint32_t mesh_primitive_data = 0;
uint32_t mesh_primitives = 0;
uint32_t mesh_cull = 0;
uint32_t entry_label = 0;
uint32_t current_label = 0;
uint32_t pixel_valid_mask_variable = 0;
@@ -427,6 +436,10 @@ struct ValueEmitContext {
uint32_t Def(IR::Value value);
uint32_t Arg(const IR::Inst& inst, size_t index);
uint32_t HalfArg(const IR::Inst& inst, size_t index, uint32_t half);
uint32_t Ballot(IR::Value predicate);
uint32_t FirstLane(uint32_t ballot);
uint32_t Shuffle(const IR::Inst& inst, size_t index, uint32_t lane);
uint32_t Result(const IR::Inst& inst);
uint32_t TypeId(IR::Type type) const;
uint32_t Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
@@ -448,6 +461,8 @@ struct ValueEmitContext {
std::unordered_map<const IR::Inst*, std::pair<uint32_t, uint32_t>> dispatcher_block_loads;
const IR::Block* current_block = nullptr;
uint32_t scratch_u32_variable = 0;
ValueEmitContext* other_half = nullptr;
uint32_t half = 0;
};
enum class VertexInputScalarKind { Float, Sint, Uint };
@@ -575,6 +590,11 @@ void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name
void AddDescriptorAnnotationsAndNames(EmitterState& state);
void DefineModule(EmitterState& state);
void DefineMeshOutputs(EmitterState& state);
void EmitMeshEntryPoint(EmitterState& state);
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index = 0);
uint32_t MeshPrimitivePointer(EmitterState& state);
uint32_t EmitTrueBool(EmitterState& state);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+19 -9
View File
@@ -32,6 +32,9 @@ uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) {
uint32_t EmitDsMaskedLaneRead(EmitterState& state, uint32_t source, uint32_t target,
uint32_t exec) {
if (state.lane_count == 2) {
target = Binary(state, OpBitwiseAnd, TypeU32(state), target, ConstantU32(state, 31));
}
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
ConstantU32(state, ScopeSubgroup), source, target});
@@ -731,6 +734,9 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR
uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) {
auto& state = ctx.state;
if (ctx.half == 1) {
return ctx.other_half->Def(IR::Value(const_cast<IR::Inst*>(&inst)));
}
const auto m0 = ctx.Arg(inst, 0);
const auto base =
Binary(state, OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
@@ -743,9 +749,7 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto index = EmitMemoryElementIndex(state, access, raw_index);
const auto exec = ctx.Arg(inst, 1);
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
ConstantU32(state, ScopeSubgroup), exec});
const auto ballot = ctx.Ballot(inst.Arg(1));
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
@@ -753,11 +757,12 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
const auto count =
Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), low),
Unary(state, OpBitCount, TypeU32(state), high));
const auto first = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), first,
ConstantU32(state, ScopeSubgroup), ballot});
const auto first = ctx.FirstLane(ballot);
const auto source_lane = state.lane_count == 2 ? Binary(state, OpBitwiseAnd, TypeU32(state),
first, ConstantU32(state, 31))
: first;
const auto is_first =
Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), first);
Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), source_lane);
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
const auto m0_bounds =
mem.kind == IR::ResourceKind::Gds
@@ -765,7 +770,12 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
: Binary(state, OpULessThan, TypeBool(state),
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
const auto condition = AndCondition(
state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds)));
state, is_first,
AndCondition(state,
state.lane_count == 2
? Binary(state, OpINotEqual, TypeBool(state), count, ConstantU32(state, 0))
: exec,
AndCondition(state, storage_bounds, m0_bounds)));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
@@ -777,7 +787,7 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
});
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), result,
ConstantU32(state, ScopeSubgroup), atomic, first});
ConstantU32(state, ScopeSubgroup), atomic, source_lane});
return result;
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+8 -6
View File
@@ -65,14 +65,15 @@ void EmitMemoryOffsets(EmitterState& state) {
}
uint32_t LdsDwordCount(const EmitterState& state) {
return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u;
const auto* workgroup = ShaderWorkgroupInput(state.stage, state.input_info);
return workgroup != nullptr ? workgroup->lds_size_dwords : 8192u;
}
static void EnsureLdsStorage(EmitterState& state) {
if (state.lds_variable != 0) {
return;
}
if (state.stage != ShaderType::Compute) {
if (ShaderWorkgroupInput(state.stage, state.input_info) == nullptr) {
EXIT("function LDS was not prepared before SPIR-V function emission\n");
}
state.lds_variable = state.builder.DefineGlobalVariable(
@@ -122,10 +123,10 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
access.length = state.gds_length;
return access;
case IR::ResourceKind::Scratch:
if (state.scratch_variable == 0) {
if (state.scratch_variable[state.lane_half] == 0) {
EXIT("scratch storage was not prepared before SPIR-V function emission\n");
}
access.object_pointer = state.scratch_variable;
access.object_pointer = state.scratch_variable[state.lane_half];
access.length = ConstantU32(state, state.program.scratch_dwords);
return access;
case IR::ResourceKind::ScalarAddress:
@@ -167,8 +168,9 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
const auto pointer = state.builder.AllocateId();
const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction
: state.stage == ShaderType::Compute ? StorageClassWorkgroup
: StorageClassFunction;
: ShaderWorkgroupInput(state.stage, state.input_info) != nullptr
? StorageClassWorkgroup
: StorageClassFunction;
state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class),
pointer, access.object_pointer, index});
return pointer;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp
+186
View File
@@ -0,0 +1,186 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
uint32_t MeshArray(EmitterState& state, uint32_t storage, uint32_t type, uint32_t count) {
const auto array = state.builder.Type(OpTypeArray, {type, ConstantU32(state, count)});
return state.builder.DefineGlobalVariable(TypePointer(state, storage, array), storage);
}
uint32_t MeshElement(EmitterState& state, uint32_t variable, uint32_t storage, uint32_t type,
uint32_t index) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, TypePointer(state, storage, type), pointer, variable, index});
return pointer;
}
uint32_t MeshLoad(EmitterState& state, uint32_t variable, uint32_t type, uint32_t index) {
const auto pointer = MeshElement(state, variable, StorageClassWorkgroup, type, index);
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, type, value, pointer});
return value;
}
uint32_t MeshOutputType(EmitterState& state, IR::StageOutputKind kind) {
return kind == IR::StageOutputKind::Layer ? TypeU32(state) : TypeF32Vector(state, 4);
}
} // namespace
void DefineMeshOutputs(EmitterState& state) {
const auto& mesh = state.input_info.vertex->mesh;
for (auto& output: state.outputs) {
if (output.kind != IR::StageOutputKind::Position &&
output.kind != IR::StageOutputKind::Parameter &&
output.kind != IR::StageOutputKind::Layer) {
EXIT("unsupported mesh output kind=%u\n", static_cast<uint32_t>(output.kind));
}
const auto type = MeshOutputType(state, output.kind);
output.variable_id = MeshArray(
state, StorageClassOutput, type,
output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives : mesh.max_vertices);
output.mesh_data_variable =
MeshArray(state, StorageClassWorkgroup, type, mesh.max_vertices);
state.interface_variables.push_back(output.variable_id);
state.builder.AddName(output.variable_id, output.debug_name.c_str());
if (output.kind == IR::StageOutputKind::Parameter) {
state.builder.AddAnnotation(
{OpDecorate, output.variable_id, DecorationLocation, output.location});
} else {
state.builder.AddAnnotation(
{OpDecorate, output.variable_id, DecorationBuiltIn,
output.kind == IR::StageOutputKind::Layer ? BuiltInLayer : BuiltInPosition});
}
if (output.kind == IR::StageOutputKind::Layer) {
state.builder.AddAnnotation({OpDecorate, output.variable_id, 5271u}); // PerPrimitiveEXT
}
}
state.mesh_allocation = MeshArray(state, StorageClassWorkgroup, TypeU32(state), 2);
state.mesh_primitive_data =
MeshArray(state, StorageClassWorkgroup, TypeU32(state), mesh.max_primitives);
state.mesh_primitives =
MeshArray(state, StorageClassOutput, TypeU32Vector(state, 3), mesh.max_primitives);
state.mesh_cull = MeshArray(state, StorageClassOutput, TypeBool(state), mesh.max_primitives);
state.interface_variables.push_back(state.mesh_primitives);
state.interface_variables.push_back(state.mesh_cull);
state.builder.AddAnnotation({OpDecorate, state.mesh_primitives, DecorationBuiltIn,
5296u}); // PrimitiveTriangleIndicesEXT
state.builder.AddAnnotation(
{OpDecorate, state.mesh_cull, DecorationBuiltIn, 5299u}); // CullPrimitiveEXT
state.builder.AddAnnotation({OpDecorate, state.mesh_cull, 5271u});
}
uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index) {
const auto output = std::ranges::find_if(state.outputs, [=](const OutputBinding& binding) {
return binding.kind == kind && binding.index == index;
});
if (output == state.outputs.end()) {
EXIT("mesh export has no output binding: kind=%u index=%u\n", static_cast<uint32_t>(kind),
index);
}
return MeshElement(state, output->mesh_data_variable, StorageClassWorkgroup,
MeshOutputType(state, kind), EmitLocalInvocationIndex(state));
}
uint32_t MeshPrimitivePointer(EmitterState& state) {
return MeshElement(state, state.mesh_primitive_data, StorageClassWorkgroup, TypeU32(state),
EmitLocalInvocationIndex(state));
}
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto first = state.builder.AllocateId();
state.builder.AddFunction(
{OpIEqual, TypeBool(state), first, EmitLocalInvocationIndex(state), ConstantU32(state, 0)});
EmitIfCondition(state, first, [&] {
const auto allocation = ctx.Arg(inst, 0);
for (uint32_t field = 0; field < 2; field++) {
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), value, allocation,
ConstantU32(state, field * 12u),
ConstantU32(state, field == 0 ? 10u : 11u)});
const auto pointer = MeshElement(state, state.mesh_allocation, StorageClassWorkgroup,
TypeU32(state), ConstantU32(state, field));
state.builder.AddFunction({OpStore, pointer, value});
}
});
}
void EmitMeshEntryPoint(EmitterState& state) {
state.builder.AddFunction(
{OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)});
EmitLabel(state, state.builder.AllocateId());
state.builder.AddFunction(
{OpFunctionCall, TypeVoid(state), state.builder.AllocateId(), state.mesh_guest_func});
// All guest waves finish before the uniform Vulkan allocation and output stores.
state.builder.AddFunction(
{OpControlBarrier, ConstantU32(state, ScopeWorkgroup), ConstantU32(state, ScopeWorkgroup),
ConstantU32(state, MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory)});
const auto vertices =
MeshLoad(state, state.mesh_allocation, TypeU32(state), ConstantU32(state, 0));
const auto primitives =
MeshLoad(state, state.mesh_allocation, TypeU32(state), ConstantU32(state, 1));
state.builder.AddFunction({5295u, vertices, primitives}); // OpSetMeshOutputsEXT
for (uint32_t half = 0; half < state.lane_count; half++) {
state.lane_half = half;
const auto index = EmitLocalInvocationIndex(state);
const auto is_vertex = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, TypeBool(state), is_vertex, index, vertices});
EmitIfCondition(state, is_vertex, [&] {
for (const auto& output: state.outputs) {
if (output.kind == IR::StageOutputKind::Layer) {
continue;
}
const auto type = MeshOutputType(state, output.kind);
const auto value = MeshLoad(state, output.mesh_data_variable, type, index);
const auto pointer =
MeshElement(state, output.variable_id, StorageClassOutput, type, index);
state.builder.AddFunction({OpStore, pointer, value});
}
});
const auto is_primitive = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, TypeBool(state), is_primitive, index, primitives});
EmitIfCondition(state, is_primitive, [&] {
const auto packed = MeshLoad(state, state.mesh_primitive_data, TypeU32(state), index);
uint32_t vertex[3] {};
for (uint32_t component = 0; component < 3; component++) {
vertex[component] = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), vertex[component],
packed, ConstantU32(state, component * 10u),
ConstantU32(state, 10)});
}
const auto triangle = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 3), triangle,
vertex[0], vertex[1], vertex[2]});
const auto triangle_pointer = MeshElement(
state, state.mesh_primitives, StorageClassOutput, TypeU32Vector(state, 3), index);
state.builder.AddFunction({OpStore, triangle_pointer, triangle});
const auto null_bit =
EmitBinaryU32(state, OpBitwiseAnd, packed, ConstantU32(state, 0x80000000u));
const auto culled = state.builder.AllocateId();
state.builder.AddFunction(
{OpINotEqual, TypeBool(state), culled, null_bit, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpStore,
MeshElement(state, state.mesh_cull, StorageClassOutput, TypeBool(state), index),
culled});
for (const auto& output: state.outputs) {
if (output.kind != IR::StageOutputKind::Layer) {
continue;
}
const auto layer = MeshLoad(state, output.mesh_data_variable, TypeU32(state),
vertex[state.input_info.vertex->mesh.provoking_vertex]);
const auto pointer = MeshElement(state, output.variable_id, StorageClassOutput,
TypeU32(state), index);
state.builder.AddFunction({OpStore, pointer, layer});
}
});
}
state.lane_half = 0;
state.builder.AddFunction({OpReturn});
state.builder.AddFunction({OpFunctionEnd});
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp
+55 -10
View File
@@ -202,7 +202,7 @@ void DefineDescriptorVariables(EmitterState& state) {
state.fault_buffer_variable = state.builder.DefineGlobalVariable(
TypeStorageBufferPointer(state), StorageClassStorageBuffer);
}
if (state.program.bindings.UsesPushData()) {
if (state.program.bindings.UsesPushData() || state.stage == ShaderType::Mesh) {
const auto pointer_type =
TypePointer(state, StorageClassPushConstant, PushConstantBlockType(state));
state.push_constant_variable =
@@ -430,6 +430,22 @@ static bool MrtUsesUintOutput(const EmitterState& state, uint32_t index) {
}
void AllocateInputVariables(EmitterState& state) {
if (state.lane_count == 2) {
const auto add_builtin = [&](IR::StageInputKind kind, uint32_t components,
const char* name) {
if (std::ranges::none_of(state.inputs, [kind](const InputBinding& input) {
return input.kind == kind;
})) {
state.inputs.push_back({kind, 0, components, 0, name});
}
};
add_builtin(IR::StageInputKind::LocalInvocationIndex, 1, "gl_LocalInvocationIndex");
if (std::ranges::any_of(state.inputs, [](const InputBinding& input) {
return input.kind == IR::StageInputKind::GlobalInvocationId;
})) {
add_builtin(IR::StageInputKind::WorkgroupId, 3, "gl_WorkGroupID");
}
}
for (auto& binding: state.inputs) {
binding.variable_id = state.builder.AllocateId();
state.interface_variables.push_back(binding.variable_id);
@@ -454,6 +470,10 @@ static uint32_t AllocateSharedOutputVariable(EmitterState& state, uint32_t& vari
}
void AllocateOutputVariables(EmitterState& state) {
if (state.stage == ShaderType::Mesh) {
DefineMeshOutputs(state);
return;
}
for (auto& binding: state.outputs) {
switch (binding.kind) {
case IR::StageOutputKind::Position:
@@ -548,6 +568,9 @@ void AddInputAnnotationsAndNames(EmitterState& state) {
}
void AddOutputAnnotationsAndNames(EmitterState& state) {
if (state.stage == ShaderType::Mesh) {
return;
}
if (state.per_vertex_variable != 0) {
state.builder.AddName(PerVertexType(state), "gl_PerVertex");
state.builder.AddName(state.per_vertex_variable, "outPerVertex");
@@ -653,9 +676,21 @@ void DefineModule(EmitterState& state) {
state.lds_variable = state.builder.AllocateId();
}
if (state.requirements.function_scratch) {
state.scratch_variable = state.builder.AllocateId();
for (uint32_t half = 0; half < state.lane_count; half++) {
state.scratch_variable[half] = state.builder.AllocateId();
}
}
state.main_func = state.builder.AllocateId();
if (state.stage == ShaderType::Mesh) {
state.mesh_guest_func = state.builder.AllocateId();
state.builder.RequireCapability(5283u); // MeshShadingEXT
state.builder.RequireExtension("SPV_EXT_mesh_shader");
state.builder.AddExecutionMode({state.main_func, 5298u}); // OutputTrianglesEXT
state.builder.AddExecutionMode(
{state.main_func, 26u, state.input_info.vertex->mesh.max_vertices});
state.builder.AddExecutionMode(
{state.main_func, 5270u, state.input_info.vertex->mesh.max_primitives});
}
state.entry_label = state.builder.AllocateId();
state.builder.RequireCapability(CapabilityShader);
@@ -682,11 +717,11 @@ void DefineModule(EmitterState& state) {
if (state.requirements.image_gather_extended) {
state.builder.RequireCapability(CapabilityImageGatherExtended);
}
if (state.requirements.subgroup_ballot || state.requirements.subgroup_shuffle ||
state.requirements.subgroup_local_invocation_id) {
if (state.lane_count == 2 || state.requirements.subgroup_ballot ||
state.requirements.subgroup_shuffle || state.requirements.subgroup_local_invocation_id) {
state.builder.RequireCapability(CapabilityGroupNonUniform);
}
if (state.requirements.subgroup_ballot) {
if (state.lane_count == 2 || state.requirements.subgroup_ballot) {
state.builder.RequireCapability(CapabilityGroupNonUniformBallot);
}
if (state.requirements.subgroup_shuffle) {
@@ -711,19 +746,24 @@ void DefineModule(EmitterState& state) {
{state.program.info.uses_dma ? AddressingModelPhysicalStorageBuffer64
: AddressingModelLogical,
MemoryModelGLSL450});
state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
state.interface_variables);
// GCN/RDNA arithmetic preserves 32-bit signed zero, infinity, and NaN. Declaring that
// contract prevents host compilers from treating synthesized IEEE values as finite.
state.builder.AddExecutionMode({state.main_func, ExecutionModeSignedZeroInfNanPreserve, 32u});
if (state.stage == ShaderType::Compute) {
if (const auto* cs = ShaderWorkgroupInput(state.stage, state.input_info)) {
uint32_t local_x = state.requirements.compute_derivatives ? 2u : 1u;
uint32_t local_y = state.requirements.compute_derivatives ? 2u : 1u;
uint32_t local_z = 1u;
const auto* cs = state.input_info.compute;
local_x = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
local_y = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
local_z = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z;
if (state.lane_count == 2) {
local_x = ((local_x * local_y * local_z + 63u) / 64u) * 32u;
local_y = local_z = 1u;
if (state.requirements.compute_derivatives) {
local_y = local_x / 2u;
local_x = 2u;
}
}
state.builder.AddExecutionMode(
{state.main_func, ExecutionModeLocalSize, local_x, local_y, local_z});
}
@@ -746,7 +786,9 @@ void DefineModule(EmitterState& state) {
state.builder.AddName(state.lds_variable, "lds_dwords");
}
if (state.requirements.function_scratch) {
state.builder.AddName(state.scratch_variable, "scratch_dwords");
for (uint32_t half = 0; half < state.lane_count; half++) {
state.builder.AddName(state.scratch_variable[half], "scratch_dwords");
}
}
AddInputAnnotationsAndNames(state);
AddOutputAnnotationsAndNames(state);
@@ -797,6 +839,9 @@ void DefineModule(EmitterState& state) {
}
state.builder.DefineGlobalVariable(input.variable_id, ptr_type, StorageClassInput);
}
if (state.stage == ShaderType::Mesh) {
return;
}
if (state.per_vertex_variable != 0) {
state.builder.DefineGlobalVariable(
state.per_vertex_variable, TypePointer(state, StorageClassOutput, PerVertexType(state)),
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
+185 -40
View File
@@ -45,6 +45,7 @@ uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
struct DeferredPhiPatch {
DeferredPhi phi;
const IR::Inst* instruction = nullptr;
uint32_t half = 0;
};
struct StructuredFunctionState {
@@ -53,7 +54,7 @@ struct StructuredFunctionState {
};
struct DispatcherFunctionState {
std::unordered_map<const IR::Inst*, uint32_t> spills;
std::array<std::unordered_map<const IR::Inst*, uint32_t>, 2> spills;
uint32_t header_label = 0;
uint32_t select_label = 0;
uint32_t after_switch_label = 0;
@@ -73,7 +74,7 @@ void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState
for (size_t index = 0; index < phi.NumArgs(); index++) {
if (phi.PhiBlock(index) == from) {
ctx.state.builder.AddFunction(
{OpStore, dispatcher.spills.at(&phi), ctx.Def(phi.Arg(index))});
{OpStore, dispatcher.spills[ctx.half].at(&phi), ctx.Def(phi.Arg(index))});
break;
}
}
@@ -94,6 +95,27 @@ void EmitReturn(ValueEmitContext& ctx) {
ctx.state.builder.AddFunction({OpReturn});
}
uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
if (ctx.other_half == nullptr ||
info.terminator.condition == CFG::BranchCondition::GotoVariable) {
return ctx.Def(info.condition);
}
const auto ballot = ctx.Ballot(info.condition);
const auto low = ctx.state.builder.AllocateId();
const auto high = ctx.state.builder.AllocateId();
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0});
ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1});
const auto kind = info.terminator.condition;
const bool zero = kind == CFG::BranchCondition::ExecZero ||
kind == CFG::BranchCondition::VccZero ||
kind == CFG::BranchCondition::SccZero;
const auto combined = EmitBinaryU32(ctx.state, zero ? OpBitwiseAnd : OpBitwiseOr, low, high);
ctx.state.builder.AddFunction({zero ? OpIEqual : OpINotEqual, TypeBool(ctx.state), result,
combined, ConstantU32(ctx.state, zero ? ~0u : 0u)});
return result;
}
void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
const IR::BlockInfo& info) {
const auto& term = info.terminator;
@@ -132,7 +154,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
EmitReturn(ctx);
return;
}
const auto condition = ctx.Def(info.condition);
const auto condition = BranchCondition(ctx, info);
emit_merge();
ctx.state.builder.AddFunction(
{OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)});
@@ -147,6 +169,9 @@ void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState&
const auto* block = TargetBlock(ctx.program, target);
if (block != nullptr) {
StoreDispatcherPhiEdge(ctx, dispatcher, from, block);
if (ctx.other_half != nullptr) {
StoreDispatcherPhiEdge(*ctx.other_half, dispatcher, from, block);
}
}
}
@@ -162,7 +187,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
const auto selected = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), selected,
ctx.Def(info.condition),
BranchCondition(ctx, info),
ConstantU32(ctx.state, term.true_block),
ConstantU32(ctx.state, term.false_block)});
return selected;
@@ -219,7 +244,8 @@ void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& s
}
}
structured.deferred_phis.push_back(
{ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst});
{ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst,
ctx.half});
return;
}
EmitDirectInstruction(ctx, inst);
@@ -233,11 +259,12 @@ void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionSt
ctx.Fail(inst, "cannot be loaded by the dispatcher");
}
ctx.state.builder.AddFunction(
{OpLoad, type, ctx.Result(inst), dispatcher.spills.at(&inst)});
{OpLoad, type, ctx.Result(inst), dispatcher.spills[ctx.half].at(&inst)});
return;
}
EmitDirectInstruction(ctx, inst);
if (const auto found = dispatcher.spills.find(&inst); found != dispatcher.spills.end()) {
if (const auto found = dispatcher.spills[ctx.half].find(&inst);
found != dispatcher.spills[ctx.half].end()) {
ctx.state.builder.AddFunction(
{OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
}
@@ -246,6 +273,9 @@ void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionSt
template <typename EmitInstruction>
void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& emit_instruction) {
ctx.current_block = block;
if (ctx.other_half != nullptr) {
ctx.other_half->current_block = block;
}
EmitLabel(ctx.state, ctx.Label(block));
bool emitted_non_phi = false;
for (const auto& inst: *block) {
@@ -256,12 +286,22 @@ void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&&
} else {
emitted_non_phi = true;
}
emit_instruction(inst);
for (uint32_t half = 0; half < ctx.state.lane_count; half++) {
auto& lane = half == 0 ? ctx : *ctx.other_half;
lane.current_block = block;
ctx.state.lane_half = half;
if (half == 0 || (inst.GetOpcode() != IR::ValueOpcode::Barrier &&
inst.GetOpcode() != IR::ValueOpcode::MeshAllocate)) {
emit_instruction(lane, inst);
}
}
ctx.state.lane_half = 0;
}
}
void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structured) {
for (const auto& deferred: structured.deferred_phis) {
auto& lane = deferred.half == 0 ? ctx : *ctx.other_half;
for (size_t index = 0; index < deferred.instruction->NumArgs(); index++) {
const auto* predecessor = deferred.instruction->PhiBlock(index);
const auto found = structured.block_exit_labels.find(predecessor);
@@ -269,7 +309,7 @@ void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structu
ctx.Fail(*deferred.instruction, "has a predecessor that was not emitted");
}
ctx.state.builder.PatchDeferredPhi(
deferred.phi, index, ctx.Def(deferred.instruction->Arg(index)), found->second);
deferred.phi, index, lane.Def(deferred.instruction->Arg(index)), found->second);
}
}
}
@@ -279,8 +319,8 @@ void EmitStructuredFunction(ValueEmitContext& ctx) {
ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
for (size_t index = 0; index < ctx.program.blocks.size(); index++) {
const auto* block = ctx.program.blocks[index];
EmitBlock(ctx, block, [&](const IR::Inst& inst) {
EmitStructuredInstruction(ctx, structured, inst);
EmitBlock(ctx, block, [&](ValueEmitContext& lane, const IR::Inst& inst) {
EmitStructuredInstruction(lane, structured, inst);
});
structured.block_exit_labels.emplace(block, ctx.state.current_label);
EmitStructuredTerminator(ctx, block, ctx.program.block_info[index]);
@@ -292,8 +332,8 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
auto& state = ctx.state;
const auto* entry = ctx.program.blocks.front();
state.builder.AddFunction({OpBranch, ctx.Label(entry)});
EmitBlock(ctx, entry, [&](const IR::Inst& inst) {
EmitDispatcherInstruction(ctx, dispatcher, inst);
EmitBlock(ctx, entry, [&](ValueEmitContext& lane, const IR::Inst& inst) {
EmitDispatcherInstruction(lane, dispatcher, inst);
});
const auto initial_pc =
EmitDispatcherNextPc(ctx, dispatcher, entry, ctx.program.block_info.front());
@@ -326,9 +366,10 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
ConstantU32(state, UINT32_MAX), dispatcher.select_label};
for (size_t index = 1; index < ctx.program.blocks.size(); index++) {
EmitBlock(ctx, ctx.program.blocks[index], [&](const IR::Inst& inst) {
EmitDispatcherInstruction(ctx, dispatcher, inst);
});
EmitBlock(ctx, ctx.program.blocks[index],
[&](ValueEmitContext& lane, const IR::Inst& inst) {
EmitDispatcherInstruction(lane, dispatcher, inst);
});
const auto selected = EmitDispatcherNextPc(ctx, dispatcher, ctx.program.blocks[index],
ctx.program.block_info[index]);
next_pc_words.push_back(selected);
@@ -408,6 +449,83 @@ uint32_t ValueEmitContext::Arg(const IR::Inst& inst, size_t index) {
return Def(inst.Arg(index));
}
uint32_t ValueEmitContext::HalfArg(const IR::Inst& inst, size_t index, uint32_t lane_half) {
return lane_half == half ? Arg(inst, index) : other_half->Arg(inst, index);
}
uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
const auto ballot_type = TypeU32Vector(state, 4);
const auto scope = ConstantU32(state, ScopeSubgroup);
const auto low = state.builder.AllocateId();
state.builder.AddFunction(
{OpGroupNonUniformBallot, ballot_type, low, scope,
other_half == nullptr || half == 0 ? Def(predicate) : other_half->Def(predicate)});
if (other_half == nullptr) {
return low;
}
const auto high = state.builder.AllocateId();
const auto low_word = state.builder.AllocateId();
const auto high_word = state.builder.AllocateId();
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, ballot_type, high, scope,
half == 1 ? Def(predicate) : other_half->Def(predicate)});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low_word, low, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high_word, high, 0});
state.builder.AddFunction({OpCompositeConstruct, ballot_type, ballot, low_word, high_word,
ConstantU32(state, 0), ConstantU32(state, 0)});
return ballot;
}
uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
if (other_half == nullptr) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), result,
ConstantU32(state, ScopeSubgroup), ballot});
return result;
}
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
const auto low_first = state.builder.AllocateId();
const auto high_first = state.builder.AllocateId();
const auto low_active = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
state.builder.AddFunction(
{OpExtInst, TypeU32(state), low_first, GlslStd450(state), GlslFindILsb, low});
state.builder.AddFunction(
{OpExtInst, TypeU32(state), high_first, GlslStd450(state), GlslFindILsb, high});
state.builder.AddFunction(
{OpINotEqual, TypeBool(state), low_active, low, ConstantU32(state, 0)});
state.builder.AddFunction({OpSelect, TypeU32(state), result, low_active, low_first,
EmitAddU32(state, high_first, ConstantU32(state, 32))});
return result;
}
uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) {
const auto type = TypeId(inst.Arg(index).GetType());
const auto scope = ConstantU32(state, ScopeSubgroup);
const auto low = state.builder.AllocateId();
if (other_half == nullptr) {
state.builder.AddFunction(
{OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), lane});
return low;
}
const auto physical_lane = EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 31));
const auto high = state.builder.AllocateId();
const auto in_high = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0), physical_lane});
state.builder.AddFunction(
{OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1), physical_lane});
state.builder.AddFunction({OpINotEqual, TypeBool(state), in_high,
EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 32)),
ConstantU32(state, 0)});
state.builder.AddFunction({OpSelect, type, value, in_high, high, low});
return value;
}
uint32_t ValueEmitContext::Result(const IR::Inst& inst) {
if (const auto found = definitions.find(&inst); found != definitions.end()) {
return found->second;
@@ -478,14 +596,22 @@ uint32_t ValueEmitContext::Label(const IR::Block* block) const {
}
void EmitProgram(EmitterState& state, const IR::Program& program) {
ValueEmitContext ctx(state, program);
ValueEmitContext ctx(state, program);
ValueEmitContext high(state, program);
if (state.lane_count == 2) {
ctx.other_half = &high;
high.other_half = &ctx;
high.half = 1;
}
std::optional<DispatcherFunctionState> dispatcher;
if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
state.pixel_valid_mask_variable = state.builder.AllocateId();
state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
}
for (const auto* block: program.blocks) {
ctx.labels.emplace(block, state.builder.AllocateId());
const auto label = state.builder.AllocateId();
ctx.labels.emplace(block, label);
high.labels.emplace(block, label);
}
if (state.program.dispatcher_fallback) {
auto& dispatch = dispatcher.emplace();
@@ -498,7 +624,7 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
ctx.Fail(inst, "cannot be stored by the dispatcher");
break;
}
dispatch.spills.emplace(&inst, state.builder.AllocateId());
dispatch.spills[0].emplace(&inst, state.builder.AllocateId());
}
}
const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) {
@@ -512,8 +638,8 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
ctx.Fail(*definition, "cannot be stored by the dispatcher");
return;
}
if (!dispatch.spills.contains(definition)) {
dispatch.spills.emplace(definition, state.builder.AllocateId());
if (!dispatch.spills[0].contains(definition)) {
dispatch.spills[0].emplace(definition, state.builder.AllocateId());
}
};
for (const auto* block: program.blocks) {
@@ -534,7 +660,13 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
dispatch.after_switch_label = state.builder.AllocateId();
dispatch.continue_label = state.builder.AllocateId();
dispatch.merge_label = state.builder.AllocateId();
ctx.dispatcher_spills = &dispatch.spills;
ctx.dispatcher_spills = &dispatch.spills[0];
if (state.lane_count == 2) {
for (const auto& [inst, id]: dispatch.spills[0]) {
dispatch.spills[1].emplace(inst, state.builder.AllocateId());
}
high.dispatcher_spills = &dispatch.spills[1];
}
}
DefineGetBdaPointer(state);
for (const auto* block: program.blocks) {
@@ -544,11 +676,15 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
})) {
ctx.scratch_u32_variable = state.builder.AllocateId();
if (state.lane_count == 2) {
high.scratch_u32_variable = state.builder.AllocateId();
}
break;
}
}
state.builder.AddFunction(
{OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)});
state.builder.AddFunction({OpFunction, TypeVoid(state),
state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func,
FunctionControlNone, TypeFunction(state)});
EmitLabel(state, state.entry_label);
if (state.requirements.function_lds) {
state.builder.AddFunction(
@@ -556,31 +692,37 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
state.lds_variable, StorageClassFunction});
}
if (state.requirements.function_scratch) {
state.builder.AddFunction(
{OpVariable,
TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords),
state.scratch_variable, StorageClassFunction});
for (uint32_t half = 0; half < state.lane_count; half++) {
state.builder.AddFunction(
{OpVariable,
TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords),
state.scratch_variable[half], StorageClassFunction});
}
}
if (state.pixel_valid_mask_variable != 0) {
state.builder.AddFunction({OpVariable,
TypePointer(state, StorageClassFunction, TypeU32(state)),
state.pixel_valid_mask_variable, StorageClassFunction});
}
if (state.program.dispatcher_fallback) {
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
if (const auto found = dispatcher->spills.find(&inst);
found != dispatcher->spills.end()) {
state.builder.AddFunction({OpVariable, SpillPointerType(ctx, inst.GetType()),
found->second, StorageClassFunction});
for (uint32_t half = 0; half < state.lane_count; half++) {
auto& lane = half == 0 ? ctx : high;
if (state.program.dispatcher_fallback) {
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
if (const auto found = dispatcher->spills[half].find(&inst);
found != dispatcher->spills[half].end()) {
state.builder.AddFunction({OpVariable,
SpillPointerType(lane, inst.GetType()),
found->second, StorageClassFunction});
}
}
}
}
}
if (ctx.scratch_u32_variable != 0) {
state.builder.AddFunction({OpVariable,
TypePointer(state, StorageClassFunction, TypeU32(state)),
ctx.scratch_u32_variable, StorageClassFunction});
if (lane.scratch_u32_variable != 0) {
state.builder.AddFunction({OpVariable,
TypePointer(state, StorageClassFunction, TypeU32(state)),
lane.scratch_u32_variable, StorageClassFunction});
}
}
if (state.gds_variable != 0) {
state.gds_length = state.builder.AllocateId();
@@ -600,6 +742,9 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
EmitStructuredFunction(ctx);
}
state.builder.AddFunction({OpFunctionEnd});
if (state.stage == ShaderType::Mesh) {
EmitMeshEntryPoint(state);
}
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter