raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-22 11:56:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

shader: run merged geometry stages as mesh shaders

Browse Source 
Co-authored-by: Brandon Strong <brandostrong@gmail.com>
Co-authored-by: fxpw <fxpw@outlook.com>
Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com>
This commit is contained in:
nmzik
2026-09-05 08:32:23 +04:00
parent 246212789f
commit 3213934d42
39 changed files with 1403 additions and 858 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/ShaderRecompiler.cpp
+50 -14
View File
@@ -14,7 +14,6 @@
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
#include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h"
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
@@ -48,6 +47,7 @@ const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Compute: return "CS";
case ShaderType::Vertex: return "VS";
case ShaderType::Mesh: return "MS";
case ShaderType::Pixel: return "PS";
default: return "unknown";
}
@@ -477,6 +477,45 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded
return data;
}
Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<const uint32_t> back,
std::vector<uint32_t>& joined_code) {
EXIT_IF(back.empty());
Decoder::Program result;
uint32_t front_words = 0;
while (front_words < front.size()) {
auto& inst = result.instructions.emplace_back();
Decoder::DecodeInstruction(front, front_words, inst);
front_words += inst.word_count;
if (inst.opcode == Decoder::Opcode::S_SETPC_B64) {
EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr ||
inst.src0.reg != 6u);
break;
}
EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_ENDPGM);
}
EXIT_IF(result.instructions.empty() ||
result.instructions.back().opcode != Decoder::Opcode::S_SETPC_B64);
joined_code.assign(front.begin(), front.begin() + front_words);
joined_code.insert(joined_code.end(), back.begin(), back.end());
// The merged-stage ABI passes the back shader in s[6:7]. Give that handoff an
// ordinary CFG edge, retaining both bodies in one register and LDS lifetime.
joined_code[front_words - 1u] = 0xbf820000u; // s_branch to the following instruction
result.instructions.back() = {};
Decoder::DecodeInstruction(joined_code, front_words - 1u, result.instructions.back());
Decoder::Program back_program;
Decoder::DecodeProgram(back, back_program);
const auto back_pc = front_words * sizeof(uint32_t);
for (auto& inst: back_program.instructions) {
// A back-stage PC-relative data reference requires its guest code address.
EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_GETPC_B64);
inst.pc += back_pc;
inst.branch_target += back_pc;
result.instructions.push_back(std::move(inst));
}
result.code = joined_code;
return result;
}
} // namespace
TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOptions& options) {
@@ -484,7 +523,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
EXIT("shader recompiler input is empty\n");
}
if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex &&
options.stage != ShaderType::Pixel) {
options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh) {
EXIT("shader recompiler received unsupported stage %u\n",
static_cast<unsigned>(options.stage));
}
@@ -501,7 +540,12 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
static_cast<uint64_t>(code.size()));
Decoder::Program decoded;
Decoder::DecodeProgram(code, decoded);
std::vector<uint32_t> joined_code;
if (options.stage == ShaderType::Mesh) {
decoded = DecodeFusedProgram(code, options.back_code, joined_code);
} else {
Decoder::DecodeProgram(code, decoded);
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " decode instructions=%" PRIu64
" elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
@@ -560,8 +604,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
const ShaderComputeInputInfo* compute = nullptr;
switch (options.stage) {
case ShaderType::Vertex:
vertex = options.input_info.vertex;
break;
case ShaderType::Mesh: vertex = options.input_info.vertex; break;
case ShaderType::Pixel:
pixel = options.input_info.pixel;
break;
@@ -617,14 +660,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
IR::RemoveIdentities(ir.blocks);
IR::EliminateDeadCode(ir.blocks);
}
if (options.stage == ShaderType::Compute) {
const auto lds_barriers =
IR::InsertSharedMemoryBarriers(ir, ir.wave_size, *compute);
if (lds_barriers.inserted_barriers != 0) {
LOGF("%s wave64 LDS synchronization: barriers=%" PRIu32 "\n", GetDumpLabel(options),
lds_barriers.inserted_barriers);
}
}
IR::BuildSrtPlan(ir);
IR::EliminateDeadCode(ir.blocks);
IR::TrackResources(ir);
@@ -654,7 +689,8 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
const ShaderPixelInputInfo* pixel = nullptr;
const ShaderComputeInputInfo* compute = nullptr;
switch (options.stage) {
case ShaderType::Vertex: vertex = options.input_info.vertex; break;
case ShaderType::Vertex:
case ShaderType::Mesh: vertex = options.input_info.vertex; break;
case ShaderType::Pixel: pixel = options.input_info.pixel; break;
case ShaderType::Compute: compute = options.input_info.compute; break;
default: EXIT("invalid shader stage\n");
src/graphics/shader/recompiler/ShaderRecompiler.h
+1
View File
@@ -21,6 +21,7 @@ struct CompileOptions {
bool early_dump = false;
const char* dump_label = nullptr;
std::span<const uint32_t> user_data;
std::span<const uint32_t> back_code;
ShaderStageInputInfo input_info;
};
src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp
+13
View File
@@ -2,6 +2,7 @@
#include "common/debug.h"
#include <algorithm>
#include <cstring>
namespace Libs::Graphics::ShaderRecompiler::Spirv {
@@ -190,6 +191,18 @@ void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, cons
std::vector<uint32_t> operands = {execution_model, entry_point};
AppendString(operands, name);
operands.insert(operands.end(), interfaces.begin(), interfaces.end());
if (m_version >= 0x00010400u) {
for (size_t offset = 0; offset < m_declarations.size();) {
const auto count = m_declarations[offset] >> 16u;
if ((m_declarations[offset] & 0xffffu) == 59u) {
const auto id = m_declarations[offset + 2u];
if (std::find(interfaces.begin(), interfaces.end(), id) == interfaces.end()) {
operands.push_back(id);
}
}
offset += count;
}
}
AppendInstruction(m_entry_points, 15u, operands);
}
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
+10 -3
View File
@@ -236,7 +236,8 @@ void AnalyzeProgramRequirements(IR::Program& program) {
if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
Fail(program, "shared operation has invalid resource kind");
}
if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) {
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh &&
kind == IR::ResourceKind::Lds) {
requirements.function_lds = true;
}
if (shared_access == IR::SharedAccess::Append ||
@@ -259,7 +260,6 @@ void AnalyzeProgramRequirements(IR::Program& program) {
break;
}
case IR::ValueOpcode::DppUpdateU32:
case IR::ValueOpcode::WqmMask:
case IR::ValueOpcode::WriteLane: {
MarkBallot();
requirements.subgroup_local_invocation_id = true;
@@ -308,7 +308,7 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
using namespace Emitter;
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
program.stage != ShaderType::Pixel) {
program.stage != ShaderType::Pixel && program.stage != ShaderType::Mesh) {
Fail(program, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
}
if (!program.srt_plan_complete || !program.resource_tracking_complete ||
@@ -321,6 +321,11 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
EmitterState state(program, input_info);
state.stage = program.stage;
state.wave_size = program.wave_size;
const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
state.lane_count =
workgroup != nullptr && program.wave_size == 64u && workgroup->host_subgroup_size == 32u
? 2u
: 1u;
state.inputs.reserve(program.info.inputs.size());
state.outputs.reserve(program.info.outputs.size());
state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());
@@ -329,6 +334,8 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
AllocateOutputVariables(state);
DefineModule(state);
EmitProgram(state, program);
state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
state.interface_variables);
auto binary = state.builder.Build();
if (binary.empty()) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp
+1
View File
@@ -278,6 +278,7 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
uint32_t ExecutionModelForStage(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return ExecutionModelVertex;
case ShaderType::Mesh: return 5365u; // MeshEXT
case ShaderType::Pixel: return ExecutionModelFragment;
default: return ExecutionModelGLCompute;
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
+65 -95
View File
@@ -16,26 +16,6 @@ bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t&
return true;
}
uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) {
const auto shifted_one = state.builder.AllocateId();
const auto merged_one = state.builder.AllocateId();
const auto shifted_two = state.builder.AllocateId();
const auto merged_two = state.builder.AllocateId();
const auto quad_bits = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), shifted_one, value, ConstantU32(state, 1)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one});
state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), shifted_two, merged_one, ConstantU32(state, 2)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two});
state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), quad_bits, merged_two, ConstantU32(state, 0x11111111u)});
state.builder.AddFunction(
{OpIMul, TypeU32(state), result, quad_bits, ConstantU32(state, 0x0fu)});
return result;
}
uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
const auto shifted_one = state.builder.AllocateId();
const auto merged_one = state.builder.AllocateId();
@@ -61,6 +41,24 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
if (kind == IR::StageInputKind::LocalInvocationIndex) {
return EmitLocalInvocationIndex(state);
}
if (state.lane_count == 2 && (kind == IR::StageInputKind::LocalInvocationId ||
kind == IR::StageInputKind::GlobalInvocationId)) {
const auto* cs = ShaderWorkgroupInput(state.stage, state.input_info);
uint32_t divisor = 1;
for (uint32_t axis = 0; axis < component; axis++) {
divisor *= std::max(cs->threads_num[axis], 1u);
}
const auto size = std::max(cs->threads_num[component], 1u);
const auto divided = EmitBinaryU32(state, OpUDiv, EmitLocalInvocationIndex(state),
ConstantU32(state, divisor));
const auto local = EmitBinaryU32(state, OpUMod, divided, ConstantU32(state, size));
if (kind == IR::StageInputKind::LocalInvocationId) {
return local;
}
const auto group = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, component);
return EmitAddU32(state, local,
EmitBinaryU32(state, OpIMul, group, ConstantU32(state, size)));
}
const auto variable = InputVariableForKind(state, kind);
if (variable == 0) {
return ConstantU32(state, 0);
@@ -106,40 +104,6 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
return EmitInputComponentU32(state, kind, component);
}
uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) {
auto& state = ctx.state;
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
ConstantU32(state, ScopeSubgroup), active});
const auto low = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
const auto wqm_low = EmitWqmWordU32(state, low);
const auto lane = EmitSubgroupLocalInvocationId(state);
uint32_t mask = wqm_low;
uint32_t bit_lane = lane;
if (state.wave_size == 64u) {
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
const auto wqm_high = EmitWqmWordU32(state, high);
const auto upper = state.builder.AllocateId();
mask = state.builder.AllocateId();
bit_lane = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 32)});
state.builder.AddFunction({OpSelect, TypeU32(state), mask, upper, wqm_high, wqm_low});
state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), bit_lane, lane, ConstantU32(state, 31)});
}
const auto bit = state.builder.AllocateId();
const auto hit = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), bit_lane});
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
state.builder.AddFunction({OpINotEqual, TypeBool(state), result, hit, ConstantU32(state, 0)});
return result;
}
uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags,
uint32_t exec) {
auto& state = ctx.state;
@@ -378,7 +342,10 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
const auto layer = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), layer, raw,
ConstantU32(state, 0x7ffu)});
state.builder.AddFunction({OpStore, state.layer_variable, layer});
const auto pointer = state.stage == ShaderType::Mesh
? MeshOutputPointer(state, IR::StageOutputKind::Layer)
: state.layer_variable;
state.builder.AddFunction({OpStore, pointer, layer});
continue;
}
if (!output.point_size && output.clip_distance == UINT32_MAX &&
@@ -454,12 +421,18 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
{OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
}
if (exp.kind == IR::ExportTargetKind::Null || exp.kind == IR::ExportTargetKind::Primitive ||
exp.en == 0u) {
if (exp.kind == IR::ExportTargetKind::Null || exp.en == 0u) {
return;
}
EmitIfCondition(state, exec, [&]() {
const auto data = ctx.Arg(inst, 0);
if (exp.kind == IR::ExportTargetKind::Primitive) {
if (state.stage == ShaderType::Mesh) {
state.builder.AddFunction(
{OpStore, MeshPrimitivePointer(state), ExportRawComponent(ctx, data, 0)});
}
return;
}
if (exp.kind == IR::ExportTargetKind::Position && exp.index != 0) {
EmitAuxPositionExport(ctx, data, exp);
return;
@@ -483,8 +456,9 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
}
return;
}
const auto variable = OutputVariableForExport(state, exp);
if (variable == 0) {
const auto variable =
state.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp);
if (state.stage != ShaderType::Mesh && variable == 0) {
return;
}
const bool uint_output = MrtOutputMode(state, exp) == 7u;
@@ -501,10 +475,16 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
value = mapped;
}
}
if (exp.kind == IR::ExportTargetKind::Position) {
if (state.stage == ShaderType::Vertex && state.input_info.vertex->clip_space.enabled) {
value = ConvertPositionToClipSpace(state, value);
}
if (exp.kind == IR::ExportTargetKind::Position &&
state.input_info.vertex->clip_space.enabled) {
value = ConvertPositionToClipSpace(state, value);
}
if (state.stage == ShaderType::Mesh) {
const auto kind = exp.kind == IR::ExportTargetKind::Position
? IR::StageOutputKind::Position
: IR::StageOutputKind::Parameter;
state.builder.AddFunction({OpStore, MeshOutputPointer(state, kind, exp.index), value});
} else if (exp.kind == IR::ExportTargetKind::Position) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)),
@@ -537,6 +517,19 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
ConstantU32(state, semantics)});
return true;
}
case IR::ValueOpcode::MeshAllocate: EmitMeshAllocate(ctx, inst); return true;
case IR::ValueOpcode::MeshDrawParameter: {
const auto index = inst.Arg(0).U32();
if (state.stage != ShaderType::Mesh || index >= IR::PushData::MeshDrawDwordCount) {
ctx.Fail(inst, "invalid mesh draw parameter");
}
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state),
pointer, state.push_constant_variable, ConstantU32(state, 0),
ConstantU32(state, index)});
state.builder.AddFunction({OpLoad, TypeU32(state), ctx.Result(inst), pointer});
return true;
}
case IR::ValueOpcode::GetUserData: {
const auto reg = inst.Arg(0).ScalarRegister();
uint32_t dword = 0;
@@ -561,17 +554,12 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
case IR::ValueOpcode::DppMoveU32: {
const auto flags = inst.Flags<IR::DppMoveFlags>();
const auto target = EmitDppTargetLane(state, flags.control);
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
target.lane});
const auto shuffled = ctx.Shuffle(inst, 0, target.lane);
if (flags.fetch_inactive) {
ctx.Define(inst, shuffled);
return true;
}
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
const auto ballot = ctx.Ballot(inst.Arg(1));
const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
const auto can_fetch = state.builder.AllocateId();
state.builder.AddFunction(
@@ -588,30 +576,18 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
case IR::ValueOpcode::WqmU64:
ctx.Define(inst, EmitWqmU64(ctx.state, ctx.Arg(inst, 0)));
return true;
case IR::ValueOpcode::WqmMask:
ctx.Define(inst, EmitWqm(ctx, ctx.Arg(inst, 0)));
return true;
case IR::ValueOpcode::LaneId:
ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
return true;
case IR::ValueOpcode::Ballot:
ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
return true;
case IR::ValueOpcode::Ballot: ctx.Define(inst, ctx.Ballot(inst.Arg(0))); return true;
case IR::ValueOpcode::ReadFirstLane: {
const auto ballot = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), lane,
ConstantU32(state, ScopeSubgroup), ballot});
ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), lane});
const auto ballot = ctx.Ballot(inst.Arg(1));
const auto lane = ctx.FirstLane(ballot);
ctx.Define(inst, ctx.Shuffle(inst, 0, lane));
return true;
}
case IR::ValueOpcode::ReadLane:
ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
ctx.Define(inst, ctx.Shuffle(inst, 0, ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::WriteLane: {
const auto hit = state.builder.AllocateId();
@@ -656,16 +632,10 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row_value, index});
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
target});
const auto shuffled = ctx.Shuffle(inst, 0, target);
uint32_t result = shuffled;
if (!flags.fetch_inactive) {
const auto source_exec = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeBool(state), source_exec,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 3),
target});
const auto source_exec = ctx.Shuffle(inst, 3, target);
result = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, TypeU32(state), result, source_exec, shuffled,
ConstantU32(state, 0)});
src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp
+6 -1
View File
@@ -126,7 +126,7 @@ uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable});
return value;
return state.lane_half == 0 ? value : EmitAddU32(state, value, ConstantU32(state, 32));
}
uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
@@ -167,6 +167,11 @@ uint32_t EmitLocalInvocationIndex(EmitterState& state) {
}
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeU32(state), value, variable});
if (state.lane_count == 2) {
const auto wave_base = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~31u));
return EmitAddU32(state, EmitAddU32(state, value, wave_base),
ConstantU32(state, state.lane_half * 32));
}
return value;
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+23 -3
View File
@@ -198,6 +198,7 @@ enum : uint32_t {
OpIMul = 132,
OpFMul = 133,
OpUDiv = 134,
OpUMod = 137,
OpFDiv = 136,
OpIAddCarry = 149,
OpUMulExtended = 151,
@@ -318,6 +319,7 @@ struct OutputBinding {
uint32_t location = 0;
uint32_t variable_id = 0;
std::string debug_name;
uint32_t mesh_data_variable = 0;
};
using ImageDimension = Decoder::ImageDimension;
@@ -345,8 +347,8 @@ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
struct EmitterState {
EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_)
: program(program_), input_info(input_info_),
requirements(*program_.spirv_requirements) {}
: builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u),
program(program_), input_info(input_info_), requirements(*program_.spirv_requirements) {}
Builder builder;
const IR::Program& program;
@@ -354,6 +356,8 @@ struct EmitterState {
const IR::SpirvRequirements& requirements;
ShaderType stage = ShaderType::Unknown;
uint32_t wave_size = 64;
uint32_t lane_count = 1;
uint32_t lane_half = 0;
uint32_t storage_buffer_variable = 0;
uint32_t storage_buffer_u64_variable = 0;
std::array<uint32_t, IR::ShaderInfo::MaxBuffers> memory_byte_offsets {};
@@ -366,10 +370,15 @@ struct EmitterState {
uint32_t shader_data_storage_variable = 0;
uint32_t flattened_srt_variable = 0;
uint32_t lds_variable = 0;
uint32_t scratch_variable = 0;
std::array<uint32_t, 2> scratch_variable {};
std::array<uint32_t, IR::ImageBindingCount> image_variables {};
uint32_t sampler_variable = 0;
uint32_t main_func = 0;
uint32_t mesh_guest_func = 0;
uint32_t mesh_allocation = 0;
uint32_t mesh_primitive_data = 0;
uint32_t mesh_primitives = 0;
uint32_t mesh_cull = 0;
uint32_t entry_label = 0;
uint32_t current_label = 0;
uint32_t pixel_valid_mask_variable = 0;
@@ -427,6 +436,10 @@ struct ValueEmitContext {
uint32_t Def(IR::Value value);
uint32_t Arg(const IR::Inst& inst, size_t index);
uint32_t HalfArg(const IR::Inst& inst, size_t index, uint32_t half);
uint32_t Ballot(IR::Value predicate);
uint32_t FirstLane(uint32_t ballot);
uint32_t Shuffle(const IR::Inst& inst, size_t index, uint32_t lane);
uint32_t Result(const IR::Inst& inst);
uint32_t TypeId(IR::Type type) const;
uint32_t Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
@@ -448,6 +461,8 @@ struct ValueEmitContext {
std::unordered_map<const IR::Inst*, std::pair<uint32_t, uint32_t>> dispatcher_block_loads;
const IR::Block* current_block = nullptr;
uint32_t scratch_u32_variable = 0;
ValueEmitContext* other_half = nullptr;
uint32_t half = 0;
};
enum class VertexInputScalarKind { Float, Sint, Uint };
@@ -575,6 +590,11 @@ void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name
void AddDescriptorAnnotationsAndNames(EmitterState& state);
void DefineModule(EmitterState& state);
void DefineMeshOutputs(EmitterState& state);
void EmitMeshEntryPoint(EmitterState& state);
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index = 0);
uint32_t MeshPrimitivePointer(EmitterState& state);
uint32_t EmitTrueBool(EmitterState& state);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+19 -9
View File
@@ -32,6 +32,9 @@ uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) {
uint32_t EmitDsMaskedLaneRead(EmitterState& state, uint32_t source, uint32_t target,
uint32_t exec) {
if (state.lane_count == 2) {
target = Binary(state, OpBitwiseAnd, TypeU32(state), target, ConstantU32(state, 31));
}
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
ConstantU32(state, ScopeSubgroup), source, target});
@@ -731,6 +734,9 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR
uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) {
auto& state = ctx.state;
if (ctx.half == 1) {
return ctx.other_half->Def(IR::Value(const_cast<IR::Inst*>(&inst)));
}
const auto m0 = ctx.Arg(inst, 0);
const auto base =
Binary(state, OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
@@ -743,9 +749,7 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto index = EmitMemoryElementIndex(state, access, raw_index);
const auto exec = ctx.Arg(inst, 1);
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
ConstantU32(state, ScopeSubgroup), exec});
const auto ballot = ctx.Ballot(inst.Arg(1));
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
@@ -753,11 +757,12 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
const auto count =
Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), low),
Unary(state, OpBitCount, TypeU32(state), high));
const auto first = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), first,
ConstantU32(state, ScopeSubgroup), ballot});
const auto first = ctx.FirstLane(ballot);
const auto source_lane = state.lane_count == 2 ? Binary(state, OpBitwiseAnd, TypeU32(state),
first, ConstantU32(state, 31))
: first;
const auto is_first =
Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), first);
Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), source_lane);
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
const auto m0_bounds =
mem.kind == IR::ResourceKind::Gds
@@ -765,7 +770,12 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
: Binary(state, OpULessThan, TypeBool(state),
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
const auto condition = AndCondition(
state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds)));
state, is_first,
AndCondition(state,
state.lane_count == 2
? Binary(state, OpINotEqual, TypeBool(state), count, ConstantU32(state, 0))
: exec,
AndCondition(state, storage_bounds, m0_bounds)));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
@@ -777,7 +787,7 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
});
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), result,
ConstantU32(state, ScopeSubgroup), atomic, first});
ConstantU32(state, ScopeSubgroup), atomic, source_lane});
return result;
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+8 -6
View File
@@ -65,14 +65,15 @@ void EmitMemoryOffsets(EmitterState& state) {
}
uint32_t LdsDwordCount(const EmitterState& state) {
return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u;
const auto* workgroup = ShaderWorkgroupInput(state.stage, state.input_info);
return workgroup != nullptr ? workgroup->lds_size_dwords : 8192u;
}
static void EnsureLdsStorage(EmitterState& state) {
if (state.lds_variable != 0) {
return;
}
if (state.stage != ShaderType::Compute) {
if (ShaderWorkgroupInput(state.stage, state.input_info) == nullptr) {
EXIT("function LDS was not prepared before SPIR-V function emission\n");
}
state.lds_variable = state.builder.DefineGlobalVariable(
@@ -122,10 +123,10 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
access.length = state.gds_length;
return access;
case IR::ResourceKind::Scratch:
if (state.scratch_variable == 0) {
if (state.scratch_variable[state.lane_half] == 0) {
EXIT("scratch storage was not prepared before SPIR-V function emission\n");
}
access.object_pointer = state.scratch_variable;
access.object_pointer = state.scratch_variable[state.lane_half];
access.length = ConstantU32(state, state.program.scratch_dwords);
return access;
case IR::ResourceKind::ScalarAddress:
@@ -167,8 +168,9 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
const auto pointer = state.builder.AllocateId();
const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction
: state.stage == ShaderType::Compute ? StorageClassWorkgroup
: StorageClassFunction;
: ShaderWorkgroupInput(state.stage, state.input_info) != nullptr
? StorageClassWorkgroup
: StorageClassFunction;
state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class),
pointer, access.object_pointer, index});
return pointer;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp
+186
View File
@@ -0,0 +1,186 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
uint32_t MeshArray(EmitterState& state, uint32_t storage, uint32_t type, uint32_t count) {
const auto array = state.builder.Type(OpTypeArray, {type, ConstantU32(state, count)});
return state.builder.DefineGlobalVariable(TypePointer(state, storage, array), storage);
}
uint32_t MeshElement(EmitterState& state, uint32_t variable, uint32_t storage, uint32_t type,
uint32_t index) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, TypePointer(state, storage, type), pointer, variable, index});
return pointer;
}
uint32_t MeshLoad(EmitterState& state, uint32_t variable, uint32_t type, uint32_t index) {
const auto pointer = MeshElement(state, variable, StorageClassWorkgroup, type, index);
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, type, value, pointer});
return value;
}
uint32_t MeshOutputType(EmitterState& state, IR::StageOutputKind kind) {
return kind == IR::StageOutputKind::Layer ? TypeU32(state) : TypeF32Vector(state, 4);
}
} // namespace
void DefineMeshOutputs(EmitterState& state) {
const auto& mesh = state.input_info.vertex->mesh;
for (auto& output: state.outputs) {
if (output.kind != IR::StageOutputKind::Position &&
output.kind != IR::StageOutputKind::Parameter &&
output.kind != IR::StageOutputKind::Layer) {
EXIT("unsupported mesh output kind=%u\n", static_cast<uint32_t>(output.kind));
}
const auto type = MeshOutputType(state, output.kind);
output.variable_id = MeshArray(
state, StorageClassOutput, type,
output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives : mesh.max_vertices);
output.mesh_data_variable =
MeshArray(state, StorageClassWorkgroup, type, mesh.max_vertices);
state.interface_variables.push_back(output.variable_id);
state.builder.AddName(output.variable_id, output.debug_name.c_str());
if (output.kind == IR::StageOutputKind::Parameter) {
state.builder.AddAnnotation(
{OpDecorate, output.variable_id, DecorationLocation, output.location});
} else {
state.builder.AddAnnotation(
{OpDecorate, output.variable_id, DecorationBuiltIn,
output.kind == IR::StageOutputKind::Layer ? BuiltInLayer : BuiltInPosition});
}
if (output.kind == IR::StageOutputKind::Layer) {
state.builder.AddAnnotation({OpDecorate, output.variable_id, 5271u}); // PerPrimitiveEXT
}
}
state.mesh_allocation = MeshArray(state, StorageClassWorkgroup, TypeU32(state), 2);
state.mesh_primitive_data =
MeshArray(state, StorageClassWorkgroup, TypeU32(state), mesh.max_primitives);
state.mesh_primitives =
MeshArray(state, StorageClassOutput, TypeU32Vector(state, 3), mesh.max_primitives);
state.mesh_cull = MeshArray(state, StorageClassOutput, TypeBool(state), mesh.max_primitives);
state.interface_variables.push_back(state.mesh_primitives);
state.interface_variables.push_back(state.mesh_cull);
state.builder.AddAnnotation({OpDecorate, state.mesh_primitives, DecorationBuiltIn,
5296u}); // PrimitiveTriangleIndicesEXT
state.builder.AddAnnotation(
{OpDecorate, state.mesh_cull, DecorationBuiltIn, 5299u}); // CullPrimitiveEXT
state.builder.AddAnnotation({OpDecorate, state.mesh_cull, 5271u});
}
uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index) {
const auto output = std::ranges::find_if(state.outputs, [=](const OutputBinding& binding) {
return binding.kind == kind && binding.index == index;
});
if (output == state.outputs.end()) {
EXIT("mesh export has no output binding: kind=%u index=%u\n", static_cast<uint32_t>(kind),
index);
}
return MeshElement(state, output->mesh_data_variable, StorageClassWorkgroup,
MeshOutputType(state, kind), EmitLocalInvocationIndex(state));
}
uint32_t MeshPrimitivePointer(EmitterState& state) {
return MeshElement(state, state.mesh_primitive_data, StorageClassWorkgroup, TypeU32(state),
EmitLocalInvocationIndex(state));
}
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto first = state.builder.AllocateId();
state.builder.AddFunction(
{OpIEqual, TypeBool(state), first, EmitLocalInvocationIndex(state), ConstantU32(state, 0)});
EmitIfCondition(state, first, [&] {
const auto allocation = ctx.Arg(inst, 0);
for (uint32_t field = 0; field < 2; field++) {
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), value, allocation,
ConstantU32(state, field * 12u),
ConstantU32(state, field == 0 ? 10u : 11u)});
const auto pointer = MeshElement(state, state.mesh_allocation, StorageClassWorkgroup,
TypeU32(state), ConstantU32(state, field));
state.builder.AddFunction({OpStore, pointer, value});
}
});
}
void EmitMeshEntryPoint(EmitterState& state) {
state.builder.AddFunction(
{OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)});
EmitLabel(state, state.builder.AllocateId());
state.builder.AddFunction(
{OpFunctionCall, TypeVoid(state), state.builder.AllocateId(), state.mesh_guest_func});
// All guest waves finish before the uniform Vulkan allocation and output stores.
state.builder.AddFunction(
{OpControlBarrier, ConstantU32(state, ScopeWorkgroup), ConstantU32(state, ScopeWorkgroup),
ConstantU32(state, MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory)});
const auto vertices =
MeshLoad(state, state.mesh_allocation, TypeU32(state), ConstantU32(state, 0));
const auto primitives =
MeshLoad(state, state.mesh_allocation, TypeU32(state), ConstantU32(state, 1));
state.builder.AddFunction({5295u, vertices, primitives}); // OpSetMeshOutputsEXT
for (uint32_t half = 0; half < state.lane_count; half++) {
state.lane_half = half;
const auto index = EmitLocalInvocationIndex(state);
const auto is_vertex = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, TypeBool(state), is_vertex, index, vertices});
EmitIfCondition(state, is_vertex, [&] {
for (const auto& output: state.outputs) {
if (output.kind == IR::StageOutputKind::Layer) {
continue;
}
const auto type = MeshOutputType(state, output.kind);
const auto value = MeshLoad(state, output.mesh_data_variable, type, index);
const auto pointer =
MeshElement(state, output.variable_id, StorageClassOutput, type, index);
state.builder.AddFunction({OpStore, pointer, value});
}
});
const auto is_primitive = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, TypeBool(state), is_primitive, index, primitives});
EmitIfCondition(state, is_primitive, [&] {
const auto packed = MeshLoad(state, state.mesh_primitive_data, TypeU32(state), index);
uint32_t vertex[3] {};
for (uint32_t component = 0; component < 3; component++) {
vertex[component] = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), vertex[component],
packed, ConstantU32(state, component * 10u),
ConstantU32(state, 10)});
}
const auto triangle = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 3), triangle,
vertex[0], vertex[1], vertex[2]});
const auto triangle_pointer = MeshElement(
state, state.mesh_primitives, StorageClassOutput, TypeU32Vector(state, 3), index);
state.builder.AddFunction({OpStore, triangle_pointer, triangle});
const auto null_bit =
EmitBinaryU32(state, OpBitwiseAnd, packed, ConstantU32(state, 0x80000000u));
const auto culled = state.builder.AllocateId();
state.builder.AddFunction(
{OpINotEqual, TypeBool(state), culled, null_bit, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpStore,
MeshElement(state, state.mesh_cull, StorageClassOutput, TypeBool(state), index),
culled});
for (const auto& output: state.outputs) {
if (output.kind != IR::StageOutputKind::Layer) {
continue;
}
const auto layer = MeshLoad(state, output.mesh_data_variable, TypeU32(state),
vertex[state.input_info.vertex->mesh.provoking_vertex]);
const auto pointer = MeshElement(state, output.variable_id, StorageClassOutput,
TypeU32(state), index);
state.builder.AddFunction({OpStore, pointer, layer});
}
});
}
state.lane_half = 0;
state.builder.AddFunction({OpReturn});
state.builder.AddFunction({OpFunctionEnd});
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp
+55 -10
View File
@@ -202,7 +202,7 @@ void DefineDescriptorVariables(EmitterState& state) {
state.fault_buffer_variable = state.builder.DefineGlobalVariable(
TypeStorageBufferPointer(state), StorageClassStorageBuffer);
}
if (state.program.bindings.UsesPushData()) {
if (state.program.bindings.UsesPushData() || state.stage == ShaderType::Mesh) {
const auto pointer_type =
TypePointer(state, StorageClassPushConstant, PushConstantBlockType(state));
state.push_constant_variable =
@@ -430,6 +430,22 @@ static bool MrtUsesUintOutput(const EmitterState& state, uint32_t index) {
}
void AllocateInputVariables(EmitterState& state) {
if (state.lane_count == 2) {
const auto add_builtin = [&](IR::StageInputKind kind, uint32_t components,
const char* name) {
if (std::ranges::none_of(state.inputs, [kind](const InputBinding& input) {
return input.kind == kind;
})) {
state.inputs.push_back({kind, 0, components, 0, name});
}
};
add_builtin(IR::StageInputKind::LocalInvocationIndex, 1, "gl_LocalInvocationIndex");
if (std::ranges::any_of(state.inputs, [](const InputBinding& input) {
return input.kind == IR::StageInputKind::GlobalInvocationId;
})) {
add_builtin(IR::StageInputKind::WorkgroupId, 3, "gl_WorkGroupID");
}
}
for (auto& binding: state.inputs) {
binding.variable_id = state.builder.AllocateId();
state.interface_variables.push_back(binding.variable_id);
@@ -454,6 +470,10 @@ static uint32_t AllocateSharedOutputVariable(EmitterState& state, uint32_t& vari
}
void AllocateOutputVariables(EmitterState& state) {
if (state.stage == ShaderType::Mesh) {
DefineMeshOutputs(state);
return;
}
for (auto& binding: state.outputs) {
switch (binding.kind) {
case IR::StageOutputKind::Position:
@@ -548,6 +568,9 @@ void AddInputAnnotationsAndNames(EmitterState& state) {
}
void AddOutputAnnotationsAndNames(EmitterState& state) {
if (state.stage == ShaderType::Mesh) {
return;
}
if (state.per_vertex_variable != 0) {
state.builder.AddName(PerVertexType(state), "gl_PerVertex");
state.builder.AddName(state.per_vertex_variable, "outPerVertex");
@@ -653,9 +676,21 @@ void DefineModule(EmitterState& state) {
state.lds_variable = state.builder.AllocateId();
}
if (state.requirements.function_scratch) {
state.scratch_variable = state.builder.AllocateId();
for (uint32_t half = 0; half < state.lane_count; half++) {
state.scratch_variable[half] = state.builder.AllocateId();
}
}
state.main_func = state.builder.AllocateId();
if (state.stage == ShaderType::Mesh) {
state.mesh_guest_func = state.builder.AllocateId();
state.builder.RequireCapability(5283u); // MeshShadingEXT
state.builder.RequireExtension("SPV_EXT_mesh_shader");
state.builder.AddExecutionMode({state.main_func, 5298u}); // OutputTrianglesEXT
state.builder.AddExecutionMode(
{state.main_func, 26u, state.input_info.vertex->mesh.max_vertices});
state.builder.AddExecutionMode(
{state.main_func, 5270u, state.input_info.vertex->mesh.max_primitives});
}
state.entry_label = state.builder.AllocateId();
state.builder.RequireCapability(CapabilityShader);
@@ -682,11 +717,11 @@ void DefineModule(EmitterState& state) {
if (state.requirements.image_gather_extended) {
state.builder.RequireCapability(CapabilityImageGatherExtended);
}
if (state.requirements.subgroup_ballot || state.requirements.subgroup_shuffle ||
state.requirements.subgroup_local_invocation_id) {
if (state.lane_count == 2 || state.requirements.subgroup_ballot ||
state.requirements.subgroup_shuffle || state.requirements.subgroup_local_invocation_id) {
state.builder.RequireCapability(CapabilityGroupNonUniform);
}
if (state.requirements.subgroup_ballot) {
if (state.lane_count == 2 || state.requirements.subgroup_ballot) {
state.builder.RequireCapability(CapabilityGroupNonUniformBallot);
}
if (state.requirements.subgroup_shuffle) {
@@ -711,19 +746,24 @@ void DefineModule(EmitterState& state) {
{state.program.info.uses_dma ? AddressingModelPhysicalStorageBuffer64
: AddressingModelLogical,
MemoryModelGLSL450});
state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
state.interface_variables);
// GCN/RDNA arithmetic preserves 32-bit signed zero, infinity, and NaN. Declaring that
// contract prevents host compilers from treating synthesized IEEE values as finite.
state.builder.AddExecutionMode({state.main_func, ExecutionModeSignedZeroInfNanPreserve, 32u});
if (state.stage == ShaderType::Compute) {
if (const auto* cs = ShaderWorkgroupInput(state.stage, state.input_info)) {
uint32_t local_x = state.requirements.compute_derivatives ? 2u : 1u;
uint32_t local_y = state.requirements.compute_derivatives ? 2u : 1u;
uint32_t local_z = 1u;
const auto* cs = state.input_info.compute;
local_x = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
local_y = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
local_z = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z;
if (state.lane_count == 2) {
local_x = ((local_x * local_y * local_z + 63u) / 64u) * 32u;
local_y = local_z = 1u;
if (state.requirements.compute_derivatives) {
local_y = local_x / 2u;
local_x = 2u;
}
}
state.builder.AddExecutionMode(
{state.main_func, ExecutionModeLocalSize, local_x, local_y, local_z});
}
@@ -746,7 +786,9 @@ void DefineModule(EmitterState& state) {
state.builder.AddName(state.lds_variable, "lds_dwords");
}
if (state.requirements.function_scratch) {
state.builder.AddName(state.scratch_variable, "scratch_dwords");
for (uint32_t half = 0; half < state.lane_count; half++) {
state.builder.AddName(state.scratch_variable[half], "scratch_dwords");
}
}
AddInputAnnotationsAndNames(state);
AddOutputAnnotationsAndNames(state);
@@ -797,6 +839,9 @@ void DefineModule(EmitterState& state) {
}
state.builder.DefineGlobalVariable(input.variable_id, ptr_type, StorageClassInput);
}
if (state.stage == ShaderType::Mesh) {
return;
}
if (state.per_vertex_variable != 0) {
state.builder.DefineGlobalVariable(
state.per_vertex_variable, TypePointer(state, StorageClassOutput, PerVertexType(state)),
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
+185 -40
View File
@@ -45,6 +45,7 @@ uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
struct DeferredPhiPatch {
DeferredPhi phi;
const IR::Inst* instruction = nullptr;
uint32_t half = 0;
};
struct StructuredFunctionState {
@@ -53,7 +54,7 @@ struct StructuredFunctionState {
};
struct DispatcherFunctionState {
std::unordered_map<const IR::Inst*, uint32_t> spills;
std::array<std::unordered_map<const IR::Inst*, uint32_t>, 2> spills;
uint32_t header_label = 0;
uint32_t select_label = 0;
uint32_t after_switch_label = 0;
@@ -73,7 +74,7 @@ void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState
for (size_t index = 0; index < phi.NumArgs(); index++) {
if (phi.PhiBlock(index) == from) {
ctx.state.builder.AddFunction(
{OpStore, dispatcher.spills.at(&phi), ctx.Def(phi.Arg(index))});
{OpStore, dispatcher.spills[ctx.half].at(&phi), ctx.Def(phi.Arg(index))});
break;
}
}
@@ -94,6 +95,27 @@ void EmitReturn(ValueEmitContext& ctx) {
ctx.state.builder.AddFunction({OpReturn});
}
uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
if (ctx.other_half == nullptr ||
info.terminator.condition == CFG::BranchCondition::GotoVariable) {
return ctx.Def(info.condition);
}
const auto ballot = ctx.Ballot(info.condition);
const auto low = ctx.state.builder.AllocateId();
const auto high = ctx.state.builder.AllocateId();
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0});
ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1});
const auto kind = info.terminator.condition;
const bool zero = kind == CFG::BranchCondition::ExecZero ||
kind == CFG::BranchCondition::VccZero ||
kind == CFG::BranchCondition::SccZero;
const auto combined = EmitBinaryU32(ctx.state, zero ? OpBitwiseAnd : OpBitwiseOr, low, high);
ctx.state.builder.AddFunction({zero ? OpIEqual : OpINotEqual, TypeBool(ctx.state), result,
combined, ConstantU32(ctx.state, zero ? ~0u : 0u)});
return result;
}
void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
const IR::BlockInfo& info) {
const auto& term = info.terminator;
@@ -132,7 +154,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
EmitReturn(ctx);
return;
}
const auto condition = ctx.Def(info.condition);
const auto condition = BranchCondition(ctx, info);
emit_merge();
ctx.state.builder.AddFunction(
{OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)});
@@ -147,6 +169,9 @@ void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState&
const auto* block = TargetBlock(ctx.program, target);
if (block != nullptr) {
StoreDispatcherPhiEdge(ctx, dispatcher, from, block);
if (ctx.other_half != nullptr) {
StoreDispatcherPhiEdge(*ctx.other_half, dispatcher, from, block);
}
}
}
@@ -162,7 +187,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
const auto selected = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), selected,
ctx.Def(info.condition),
BranchCondition(ctx, info),
ConstantU32(ctx.state, term.true_block),
ConstantU32(ctx.state, term.false_block)});
return selected;
@@ -219,7 +244,8 @@ void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& s
}
}
structured.deferred_phis.push_back(
{ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst});
{ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst,
ctx.half});
return;
}
EmitDirectInstruction(ctx, inst);
@@ -233,11 +259,12 @@ void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionSt
ctx.Fail(inst, "cannot be loaded by the dispatcher");
}
ctx.state.builder.AddFunction(
{OpLoad, type, ctx.Result(inst), dispatcher.spills.at(&inst)});
{OpLoad, type, ctx.Result(inst), dispatcher.spills[ctx.half].at(&inst)});
return;
}
EmitDirectInstruction(ctx, inst);
if (const auto found = dispatcher.spills.find(&inst); found != dispatcher.spills.end()) {
if (const auto found = dispatcher.spills[ctx.half].find(&inst);
found != dispatcher.spills[ctx.half].end()) {
ctx.state.builder.AddFunction(
{OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
}
@@ -246,6 +273,9 @@ void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionSt
template <typename EmitInstruction>
void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& emit_instruction) {
ctx.current_block = block;
if (ctx.other_half != nullptr) {
ctx.other_half->current_block = block;
}
EmitLabel(ctx.state, ctx.Label(block));
bool emitted_non_phi = false;
for (const auto& inst: *block) {
@@ -256,12 +286,22 @@ void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&&
} else {
emitted_non_phi = true;
}
emit_instruction(inst);
for (uint32_t half = 0; half < ctx.state.lane_count; half++) {
auto& lane = half == 0 ? ctx : *ctx.other_half;
lane.current_block = block;
ctx.state.lane_half = half;
if (half == 0 || (inst.GetOpcode() != IR::ValueOpcode::Barrier &&
inst.GetOpcode() != IR::ValueOpcode::MeshAllocate)) {
emit_instruction(lane, inst);
}
}
ctx.state.lane_half = 0;
}
}
void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structured) {
for (const auto& deferred: structured.deferred_phis) {
auto& lane = deferred.half == 0 ? ctx : *ctx.other_half;
for (size_t index = 0; index < deferred.instruction->NumArgs(); index++) {
const auto* predecessor = deferred.instruction->PhiBlock(index);
const auto found = structured.block_exit_labels.find(predecessor);
@@ -269,7 +309,7 @@ void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structu
ctx.Fail(*deferred.instruction, "has a predecessor that was not emitted");
}
ctx.state.builder.PatchDeferredPhi(
deferred.phi, index, ctx.Def(deferred.instruction->Arg(index)), found->second);
deferred.phi, index, lane.Def(deferred.instruction->Arg(index)), found->second);
}
}
}
@@ -279,8 +319,8 @@ void EmitStructuredFunction(ValueEmitContext& ctx) {
ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
for (size_t index = 0; index < ctx.program.blocks.size(); index++) {
const auto* block = ctx.program.blocks[index];
EmitBlock(ctx, block, [&](const IR::Inst& inst) {
EmitStructuredInstruction(ctx, structured, inst);
EmitBlock(ctx, block, [&](ValueEmitContext& lane, const IR::Inst& inst) {
EmitStructuredInstruction(lane, structured, inst);
});
structured.block_exit_labels.emplace(block, ctx.state.current_label);
EmitStructuredTerminator(ctx, block, ctx.program.block_info[index]);
@@ -292,8 +332,8 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
auto& state = ctx.state;
const auto* entry = ctx.program.blocks.front();
state.builder.AddFunction({OpBranch, ctx.Label(entry)});
EmitBlock(ctx, entry, [&](const IR::Inst& inst) {
EmitDispatcherInstruction(ctx, dispatcher, inst);
EmitBlock(ctx, entry, [&](ValueEmitContext& lane, const IR::Inst& inst) {
EmitDispatcherInstruction(lane, dispatcher, inst);
});
const auto initial_pc =
EmitDispatcherNextPc(ctx, dispatcher, entry, ctx.program.block_info.front());
@@ -326,9 +366,10 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
ConstantU32(state, UINT32_MAX), dispatcher.select_label};
for (size_t index = 1; index < ctx.program.blocks.size(); index++) {
EmitBlock(ctx, ctx.program.blocks[index], [&](const IR::Inst& inst) {
EmitDispatcherInstruction(ctx, dispatcher, inst);
});
EmitBlock(ctx, ctx.program.blocks[index],
[&](ValueEmitContext& lane, const IR::Inst& inst) {
EmitDispatcherInstruction(lane, dispatcher, inst);
});
const auto selected = EmitDispatcherNextPc(ctx, dispatcher, ctx.program.blocks[index],
ctx.program.block_info[index]);
next_pc_words.push_back(selected);
@@ -408,6 +449,83 @@ uint32_t ValueEmitContext::Arg(const IR::Inst& inst, size_t index) {
return Def(inst.Arg(index));
}
uint32_t ValueEmitContext::HalfArg(const IR::Inst& inst, size_t index, uint32_t lane_half) {
return lane_half == half ? Arg(inst, index) : other_half->Arg(inst, index);
}
uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
const auto ballot_type = TypeU32Vector(state, 4);
const auto scope = ConstantU32(state, ScopeSubgroup);
const auto low = state.builder.AllocateId();
state.builder.AddFunction(
{OpGroupNonUniformBallot, ballot_type, low, scope,
other_half == nullptr || half == 0 ? Def(predicate) : other_half->Def(predicate)});
if (other_half == nullptr) {
return low;
}
const auto high = state.builder.AllocateId();
const auto low_word = state.builder.AllocateId();
const auto high_word = state.builder.AllocateId();
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, ballot_type, high, scope,
half == 1 ? Def(predicate) : other_half->Def(predicate)});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low_word, low, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high_word, high, 0});
state.builder.AddFunction({OpCompositeConstruct, ballot_type, ballot, low_word, high_word,
ConstantU32(state, 0), ConstantU32(state, 0)});
return ballot;
}
uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
if (other_half == nullptr) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), result,
ConstantU32(state, ScopeSubgroup), ballot});
return result;
}
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
const auto low_first = state.builder.AllocateId();
const auto high_first = state.builder.AllocateId();
const auto low_active = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
state.builder.AddFunction(
{OpExtInst, TypeU32(state), low_first, GlslStd450(state), GlslFindILsb, low});
state.builder.AddFunction(
{OpExtInst, TypeU32(state), high_first, GlslStd450(state), GlslFindILsb, high});
state.builder.AddFunction(
{OpINotEqual, TypeBool(state), low_active, low, ConstantU32(state, 0)});
state.builder.AddFunction({OpSelect, TypeU32(state), result, low_active, low_first,
EmitAddU32(state, high_first, ConstantU32(state, 32))});
return result;
}
uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) {
const auto type = TypeId(inst.Arg(index).GetType());
const auto scope = ConstantU32(state, ScopeSubgroup);
const auto low = state.builder.AllocateId();
if (other_half == nullptr) {
state.builder.AddFunction(
{OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), lane});
return low;
}
const auto physical_lane = EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 31));
const auto high = state.builder.AllocateId();
const auto in_high = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0), physical_lane});
state.builder.AddFunction(
{OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1), physical_lane});
state.builder.AddFunction({OpINotEqual, TypeBool(state), in_high,
EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 32)),
ConstantU32(state, 0)});
state.builder.AddFunction({OpSelect, type, value, in_high, high, low});
return value;
}
uint32_t ValueEmitContext::Result(const IR::Inst& inst) {
if (const auto found = definitions.find(&inst); found != definitions.end()) {
return found->second;
@@ -478,14 +596,22 @@ uint32_t ValueEmitContext::Label(const IR::Block* block) const {
}
void EmitProgram(EmitterState& state, const IR::Program& program) {
ValueEmitContext ctx(state, program);
ValueEmitContext ctx(state, program);
ValueEmitContext high(state, program);
if (state.lane_count == 2) {
ctx.other_half = &high;
high.other_half = &ctx;
high.half = 1;
}
std::optional<DispatcherFunctionState> dispatcher;
if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
state.pixel_valid_mask_variable = state.builder.AllocateId();
state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
}
for (const auto* block: program.blocks) {
ctx.labels.emplace(block, state.builder.AllocateId());
const auto label = state.builder.AllocateId();
ctx.labels.emplace(block, label);
high.labels.emplace(block, label);
}
if (state.program.dispatcher_fallback) {
auto& dispatch = dispatcher.emplace();
@@ -498,7 +624,7 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
ctx.Fail(inst, "cannot be stored by the dispatcher");
break;
}
dispatch.spills.emplace(&inst, state.builder.AllocateId());
dispatch.spills[0].emplace(&inst, state.builder.AllocateId());
}
}
const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) {
@@ -512,8 +638,8 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
ctx.Fail(*definition, "cannot be stored by the dispatcher");
return;
}
if (!dispatch.spills.contains(definition)) {
dispatch.spills.emplace(definition, state.builder.AllocateId());
if (!dispatch.spills[0].contains(definition)) {
dispatch.spills[0].emplace(definition, state.builder.AllocateId());
}
};
for (const auto* block: program.blocks) {
@@ -534,7 +660,13 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
dispatch.after_switch_label = state.builder.AllocateId();
dispatch.continue_label = state.builder.AllocateId();
dispatch.merge_label = state.builder.AllocateId();
ctx.dispatcher_spills = &dispatch.spills;
ctx.dispatcher_spills = &dispatch.spills[0];
if (state.lane_count == 2) {
for (const auto& [inst, id]: dispatch.spills[0]) {
dispatch.spills[1].emplace(inst, state.builder.AllocateId());
}
high.dispatcher_spills = &dispatch.spills[1];
}
}
DefineGetBdaPointer(state);
for (const auto* block: program.blocks) {
@@ -544,11 +676,15 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
})) {
ctx.scratch_u32_variable = state.builder.AllocateId();
if (state.lane_count == 2) {
high.scratch_u32_variable = state.builder.AllocateId();
}
break;
}
}
state.builder.AddFunction(
{OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)});
state.builder.AddFunction({OpFunction, TypeVoid(state),
state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func,
FunctionControlNone, TypeFunction(state)});
EmitLabel(state, state.entry_label);
if (state.requirements.function_lds) {
state.builder.AddFunction(
@@ -556,31 +692,37 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
state.lds_variable, StorageClassFunction});
}
if (state.requirements.function_scratch) {
state.builder.AddFunction(
{OpVariable,
TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords),
state.scratch_variable, StorageClassFunction});
for (uint32_t half = 0; half < state.lane_count; half++) {
state.builder.AddFunction(
{OpVariable,
TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords),
state.scratch_variable[half], StorageClassFunction});
}
}
if (state.pixel_valid_mask_variable != 0) {
state.builder.AddFunction({OpVariable,
TypePointer(state, StorageClassFunction, TypeU32(state)),
state.pixel_valid_mask_variable, StorageClassFunction});
}
if (state.program.dispatcher_fallback) {
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
if (const auto found = dispatcher->spills.find(&inst);
found != dispatcher->spills.end()) {
state.builder.AddFunction({OpVariable, SpillPointerType(ctx, inst.GetType()),
found->second, StorageClassFunction});
for (uint32_t half = 0; half < state.lane_count; half++) {
auto& lane = half == 0 ? ctx : high;
if (state.program.dispatcher_fallback) {
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
if (const auto found = dispatcher->spills[half].find(&inst);
found != dispatcher->spills[half].end()) {
state.builder.AddFunction({OpVariable,
SpillPointerType(lane, inst.GetType()),
found->second, StorageClassFunction});
}
}
}
}
}
if (ctx.scratch_u32_variable != 0) {
state.builder.AddFunction({OpVariable,
TypePointer(state, StorageClassFunction, TypeU32(state)),
ctx.scratch_u32_variable, StorageClassFunction});
if (lane.scratch_u32_variable != 0) {
state.builder.AddFunction({OpVariable,
TypePointer(state, StorageClassFunction, TypeU32(state)),
lane.scratch_u32_variable, StorageClassFunction});
}
}
if (state.gds_variable != 0) {
state.gds_length = state.builder.AllocateId();
@@ -600,6 +742,9 @@ void EmitProgram(EmitterState& state, const IR::Program& program) {
EmitStructuredFunction(ctx);
}
state.builder.AddFunction({OpFunctionEnd});
if (state.stage == ShaderType::Mesh) {
EmitMeshEntryPoint(state);
}
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/frontend/translate/Control.cpp
+29 -34
View File
@@ -28,17 +28,23 @@ void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode op
const auto src = ReadMask(inst.src0);
const auto lhs = negate_exec ? ir.LogicalNot(old) : old;
const auto rhs = negate_source ? ir.LogicalNot(src) : src;
const auto result = IR::U1(ir.Emit(operation, {lhs, rhs}));
auto result = IR::U1(ir.Emit(operation, {lhs, rhs}));
if (write_64) {
WriteMask64(inst.dst, old);
WriteMask(inst.dst, old, true);
} else {
WriteMask(inst.dst, old);
WriteRawU32(inst.dst, ir.GetExecLo());
if (current_wave_size == 64u) {
const auto low_half =
ir.ULessThan(IR::U32(ir.Emit(IR::ValueOpcode::LaneId)), IR::U32(IR::Value(32u)));
result = IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {low_half, result, old}));
}
}
const auto mask = BallotMask(result);
ir.SetExec(result);
ir.SetExecLo(mask[0]);
ir.SetExecHi(mask[1]);
ir.SetScc(result);
ir.SetScc(
ir.INotEqual(write_64 ? ir.BitwiseOr(mask[0], mask[1]) : mask[0], IR::U32(IR::Value(0u))));
}
void Translator::ADD_U32(const Decoder::Instruction& inst, bool vector, bool use_carry_in) {
@@ -149,8 +155,13 @@ void Translator::S_BARRIER() {
ir.Emit(IR::ValueOpcode::Barrier);
}
void Translator::S_SENDMSG() {
ir.Emit(IR::ValueOpcode::Sendmsg);
void Translator::S_SENDMSG(const Decoder::Instruction& inst) {
if (program.stage == ShaderType::Mesh) {
EXIT_NOT_IMPLEMENTED(inst.src0.value != 9u); // MSG_GS_ALLOC_REQ
ir.Emit(IR::ValueOpcode::MeshAllocate, {ir.GetM0()});
} else {
ir.Emit(IR::ValueOpcode::Sendmsg);
}
}
void Translator::S_TTRACEDATA() {
@@ -207,7 +218,7 @@ void Translator::S_CSELECT_B64(const Decoder::Instruction& inst) {
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
{condition, ReadMaskValid(inst.src0), ReadMaskValid(inst.src1)}));
if (IsExecOrVcc(inst.dst)) {
WriteMask64(inst.dst, selected_mask);
WriteMask(inst.dst, selected_mask, true);
return;
}
WriteU32Pair(inst.dst,
@@ -220,9 +231,7 @@ void Translator::S_CSELECT_B64(const Decoder::Instruction& inst) {
}
void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers) {
if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) {
WriteMask(inst.dst, ReadMask(inst.src0));
} else if (apply_float_modifiers && (inst.src0.negate || inst.src0.absolute)) {
if (apply_float_modifiers && (inst.src0.negate || inst.src0.absolute)) {
WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::F32));
} else {
WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::U32));
@@ -231,7 +240,7 @@ void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modi
void Translator::S_MOV_B64(const Decoder::Instruction& inst) {
if (IsExecOrVcc(inst.dst) || IsExecOrVcc(inst.src0)) {
WriteMask64(inst.dst, ReadMask(inst.src0));
WriteMask(inst.dst, ReadMask(inst.src0), true);
return;
}
const bool scalar_copy =
@@ -250,30 +259,16 @@ void Translator::S_MOV_B64(const Decoder::Instruction& inst) {
}
void Translator::S_WQM_B64(const Decoder::Instruction& inst) {
if (!IsExecOrVcc(inst.dst) && !IsExecOrVcc(inst.src0)) {
const auto mask_valid = ReadMaskValid(inst.src0);
const auto invocation_result =
IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)}));
const auto result =
IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)}));
WriteOperand(DestinationOperand(inst), result);
if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
ir.SetThreadBitScalarReg(dst, invocation_result);
ir.SetScalarMaskTag(dst, mask_valid);
const auto raw_nonzero =
IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}));
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
} else {
ir.SetScc(
IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
}
return;
const auto mask_valid = ReadMaskValid(inst.src0);
const auto result =
IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)}));
WriteOperand(DestinationOperand(inst), result);
if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
ir.SetThreadBitScalarReg(dst, ThreadBit(ExtractU64(result)));
ir.SetScalarMaskTag(dst, mask_valid);
}
const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)}));
WriteMask64(inst.dst, result);
ir.SetScc(result);
ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
}
void Translator::V_MOVRELS_B32(const Decoder::Instruction& inst) {
src/graphics/shader/recompiler/frontend/translate/Integer.cpp
+3 -8
View File
@@ -105,8 +105,8 @@ bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
};
if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) ||
(inst.src_count > 1u && is_exec_or_vcc(inst.src1))) {
WriteMask64(inst.dst, invocation_result);
ir.SetScc(invocation_result);
const auto mask = WriteMask(inst.dst, invocation_result, true);
ir.SetScc(ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u))));
return true;
}
@@ -132,13 +132,8 @@ bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
ir.SetThreadBitScalarReg(dst, invocation_result);
ir.SetScalarMaskTag(dst, mask_valid);
const auto raw_nonzero =
ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)));
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
} else {
ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
}
ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
return true;
}
src/graphics/shader/recompiler/frontend/translate/Scalar.cpp
+1 -1
View File
@@ -208,7 +208,7 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) {
case O::S_TRAP: EmitControlNop(); return true;
case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return true;
case O::S_BARRIER: S_BARRIER(); return true;
case O::S_SENDMSG: S_SENDMSG(); return true;
case O::S_SENDMSG: S_SENDMSG(inst); return true;
case O::S_TTRACEDATA: S_TTRACEDATA(); return true;
case O::S_INST_PREFETCH: S_INST_PREFETCH(); return true;
case O::S_BRANCH:
src/graphics/shader/recompiler/frontend/translate/Translate.cpp
+108 -60
View File
@@ -110,8 +110,9 @@ Decoder::Operand Translator::PlainOperand(const Decoder::Operand& operand) {
}
std::array<IR::U32, 2> Translator::BallotMask(IR::U1 value) {
return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))),
IR::U32(IR::Value(0u))};
const auto mask = ir.Emit(IR::ValueOpcode::Ballot, {value});
return {ir.CompositeExtract(mask, 0),
current_wave_size == 64u ? ir.CompositeExtract(mask, 1) : IR::U32(IR::Value(0u))};
}
IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) {
@@ -285,7 +286,6 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
case Decoder::OperandKind::Sgpr: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg);
ir.SetScalarReg(reg, value);
ir.SetThreadBitScalarReg(reg, ThreadBit(value));
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
@@ -314,20 +314,20 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
}
case Decoder::OperandKind::VccLo:
ir.SetVccLo(IR::U32(value));
ir.SetVcc(ThreadBit(IR::U32(value)));
ir.SetVcc(ThreadBit({value, ir.GetVccHi()}));
break;
case Decoder::OperandKind::VccHi:
ir.SetVccHi(IR::U32(value));
ir.SetVcc(ThreadBit(ir.GetVccLo()));
ir.SetVcc(ThreadBit({ir.GetVccLo(), value}));
break;
case Decoder::OperandKind::M0: ir.SetM0(IR::U32(value)); break;
case Decoder::OperandKind::ExecLo:
ir.SetExecLo(IR::U32(value));
ir.SetExec(ThreadBit(IR::U32(value)));
ir.SetExec(ThreadBit({value, ir.GetExecHi()}));
break;
case Decoder::OperandKind::ExecHi:
ir.SetExecHi(IR::U32(value));
ir.SetExec(ThreadBit(ir.GetExecLo()));
ir.SetExec(ThreadBit({ir.GetExecLo(), value}));
break;
case Decoder::OperandKind::Scc:
ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u))));
@@ -576,15 +576,14 @@ void Translator::WriteU32Pair(const Decoder::Operand& operand,
if (operand.kind == Decoder::OperandKind::Null) {
return;
}
const auto thread_bit = ThreadBit(value[0]);
switch (operand.kind) {
case Decoder::OperandKind::ExecLo:
ir.SetExec(thread_bit);
ir.SetExec(ThreadBit(value));
ir.SetExecLo(value[0]);
ir.SetExecHi(value[1]);
return;
case Decoder::OperandKind::VccLo:
ir.SetVcc(thread_bit);
ir.SetVcc(ThreadBit(value));
ir.SetVccLo(value[0]);
ir.SetVccHi(value[1]);
return;
@@ -595,8 +594,14 @@ void Translator::WriteU32Pair(const Decoder::Operand& operand,
WriteRawU32(OffsetOperand(operand, 1), value[1]);
}
IR::U1 Translator::ThreadBit(IR::U32 low) {
return ir.INotEqual(low, IR::U32(IR::Value(0u)));
IR::U1 Translator::ThreadBit(const std::array<IR::U32, 2>& mask) {
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
const auto word = current_wave_size == 64u
? ir.Select(ir.ULessThan(lane, IR::U32(IR::Value(32u))), mask[0], mask[1])
: mask[0];
const auto bit = ir.BitwiseAnd(lane, IR::U32(IR::Value(31u)));
return ir.INotEqual(ir.BitwiseAnd(ir.ShiftRightLogical(word, bit), IR::U32(IR::Value(1u))),
IR::U32(IR::Value(0u)));
}
IR::U1 Translator::ReadCondition(const Decoder::Operand& operand) {
@@ -604,11 +609,6 @@ IR::U1 Translator::ReadCondition(const Decoder::Operand& operand) {
}
IR::U32 Translator::ConditionBit(const Decoder::Operand& operand) {
if (operand.kind == Decoder::OperandKind::LiteralConstant ||
operand.kind == Decoder::OperandKind::IntegerInlineConstant ||
operand.kind == Decoder::OperandKind::FloatInlineConstant) {
return IR::U32(IR::Value(operand.value & 1u));
}
return ir.Select(ReadMask(operand), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)));
}
@@ -616,12 +616,17 @@ IR::U1 Translator::ReadMask(const Decoder::Operand& operand) {
if (operand.kind == Decoder::OperandKind::LiteralConstant ||
operand.kind == Decoder::OperandKind::IntegerInlineConstant ||
operand.kind == Decoder::OperandKind::FloatInlineConstant) {
return IR::U1(IR::Value(operand.value != 0u));
return ThreadBit(ReadU32Pair(operand));
}
switch (operand.kind) {
case Decoder::OperandKind::Sgpr: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg);
return ir.GetThreadBitScalarReg(reg);
const auto mask = current_wave_size == 64u
? ReadU32Pair(operand)
: std::array {ReadRawU32(operand), IR::U32(IR::Value(0u))};
return IR::U1(ir.Emit(
IR::ValueOpcode::SelectU1,
{ir.GetScalarMaskTag(reg), ir.GetThreadBitScalarReg(reg), ThreadBit(mask)}));
}
case Decoder::OperandKind::ExecLo:
case Decoder::OperandKind::ExecHi: return ir.GetExec();
@@ -660,11 +665,12 @@ IR::U1 Translator::ReadMaskValid(const Decoder::Operand& operand) {
}
}
void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) {
std::array<IR::U32, 2> Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value,
bool write_64) {
const auto mask = BallotMask(value);
switch (operand.kind) {
case Decoder::OperandKind::Sgpr: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg);
const auto mask = BallotMask(value);
ir.SetThreadBitScalarReg(reg, value);
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
if (IR::RegIndex(reg) > 0u) {
@@ -673,57 +679,34 @@ void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value) {
}
ir.SetScalarReg(reg, mask[0]);
// A wave32 VALU mask destination must not overwrite the neighboring SGPR.
if (current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
if ((write_64 || current_wave_size == 64u) &&
IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
ir.SetScalarReg(high, mask[1]);
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
}
return;
return mask;
}
case Decoder::OperandKind::ExecLo:
case Decoder::OperandKind::ExecHi: {
const auto mask = BallotMask(value);
ir.SetExec(value);
ir.SetExecLo(mask[0]);
ir.SetExecHi(mask[1]);
return;
return mask;
}
case Decoder::OperandKind::VccLo:
case Decoder::OperandKind::VccHi: {
const auto mask = BallotMask(value);
ir.SetVcc(value);
ir.SetVccLo(mask[0]);
ir.SetVccHi(mask[1]);
return;
return mask;
}
case Decoder::OperandKind::Scc: ir.SetScc(value); return;
case Decoder::OperandKind::Scc: ir.SetScc(value); return mask;
default:
WriteRawU32(operand, ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))));
WriteRawU32(OffsetOperand(operand, 1), IR::U32(IR::Value(0u)));
return;
}
}
void Translator::WriteMask64(const Decoder::Operand& operand, IR::U1 value) {
if (operand.kind != Decoder::OperandKind::Sgpr) {
WriteMask(operand, value);
return;
}
const auto reg = static_cast<IR::ScalarReg>(operand.reg);
const auto mask = BallotMask(value);
ir.SetThreadBitScalarReg(reg, value);
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
IR::U1(IR::Value(false)));
}
ir.SetScalarReg(reg, mask[0]);
if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
ir.SetScalarReg(high, mask[1]);
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
WriteRawU32(operand, mask[0]);
WriteRawU32(OffsetOperand(operand, 1), mask[1]);
return mask;
}
}
@@ -910,6 +893,7 @@ void ValidateTranslateOptions(const TranslateOptions& options) {
}
switch (options.stage) {
case ShaderType::Vertex:
case ShaderType::Mesh:
if (options.vertex == nullptr) {
EXIT("vertex shader translation has no vertex input metadata");
}
@@ -1022,12 +1006,32 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
}
const auto value = entry_ir.GetUserData(reg);
entry_ir.SetScalarReg(reg, value);
entry_ir.SetThreadBitScalarReg(reg, entry_ir.INotEqual(value, IR::U32(IR::Value(0u))));
entry_ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
}
entry_ir.SetExec(IR::U1(IR::Value(true)));
entry_ir.SetExecLo(IR::U32(IR::Value(1u)));
entry_ir.SetExecHi(IR::U32(IR::Value(0u)));
auto initial_exec = IR::U1(IR::Value(true));
uint32_t total_threads = 0;
ShaderStageInputInfo stage_input {};
if (options.stage == ShaderType::Compute) {
stage_input.compute = options.compute;
} else {
stage_input.vertex = options.vertex;
}
const auto* workgroup = ShaderWorkgroupInput(options.stage, stage_input);
if (workgroup != nullptr) {
total_threads = std::max(workgroup->threads_num[0], 1u) *
std::max(workgroup->threads_num[1], 1u) *
std::max(workgroup->threads_num[2], 1u);
if (options.wave_size == 64u && workgroup->host_subgroup_size == 32u &&
total_threads % 64u != 0u) {
initial_exec = entry_ir.ULessThan(builtin(IR::StageInputKind::LocalInvocationIndex),
IR::U32(IR::Value(total_threads)));
}
}
entry_ir.SetExec(initial_exec);
const auto initial_mask = entry_ir.Emit(IR::ValueOpcode::Ballot, {initial_exec});
entry_ir.SetExecLo(entry_ir.CompositeExtract(initial_mask, 0));
entry_ir.SetExecHi(options.wave_size == 64u ? entry_ir.CompositeExtract(initial_mask, 1)
: IR::U32(IR::Value(0u)));
if (options.stage == ShaderType::Compute) {
const auto* cs = options.compute;
const auto thread_ids =
@@ -1046,9 +1050,6 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
}
if (cs->tg_size_en) {
const auto wave_size = cs->wave_size != 0u ? cs->wave_size : 64u;
const auto total_threads = std::max(cs->threads_num[0], 1u) *
std::max(cs->threads_num[1], 1u) *
std::max(cs->threads_num[2], 1u);
const auto waves = std::min((total_threads + wave_size - 1u) / wave_size, 0x3fu);
const auto local_index = builtin(IR::StageInputKind::LocalInvocationIndex);
const auto wave_id = IR::U32(
@@ -1062,6 +1063,53 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
entry_ir.BitwiseOr(entry_ir.BitwiseOr(wave_bits, IR::U32(IR::Value(waves))),
first_bit));
}
} else if (options.stage == ShaderType::Mesh) {
const auto& mesh = options.vertex->mesh;
EXIT_NOT_IMPLEMENTED(options.wave_size != 64u || mesh.primitives_per_group == 0u ||
mesh.vertices_per_group > 64u || total_threads > 15u * 64u);
const auto u32 = [](uint32_t value) { return IR::U32(IR::Value(value)); };
const auto draw = [&](uint32_t index) {
return IR::U32(
entry_ir.Emit(IR::ValueOpcode::MeshDrawParameter, {IR::Value(index)}));
};
const auto minimum = [&](IR::U32 lhs, IR::U32 rhs) {
return IR::U32(entry_ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs}));
};
const auto subtract_saturate = [&](IR::U32 lhs, IR::U32 rhs) {
return entry_ir.ISub(lhs, minimum(lhs, rhs));
};
const auto local = builtin(IR::StageInputKind::LocalInvocationIndex);
const auto chunk = entry_ir.IMul(builtin(IR::StageInputKind::WorkgroupId, 0),
u32(mesh.primitives_per_group));
const auto vertices =
minimum(subtract_saturate(draw(0), chunk), u32(mesh.vertices_per_group));
const auto primitives = subtract_saturate(vertices, u32(2));
const auto wave = entry_ir.ShiftRightLogical(local, u32(6));
const auto wave_base = entry_ir.BitwiseAnd(local, u32(~63u));
const auto vertex_count = minimum(subtract_saturate(vertices, wave_base), u32(64));
const auto primitive_count = minimum(subtract_saturate(primitives, wave_base), u32(64));
const auto wave_info = entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(wave, u32(24)),
u32(((total_threads + 63u) / 64u) << 28u));
entry_ir.SetScalarReg(
static_cast<IR::ScalarReg>(3),
entry_ir.BitwiseOr(wave_info, entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(
primitive_count, u32(8)),
vertex_count)));
// Triangle-strip adjacency addresses the ES records in LDS. Alternating the
// first two indices preserves winding across subgroup boundaries.
const auto parity = entry_ir.BitwiseAnd(entry_ir.IAdd(chunk, local), u32(1));
const auto first = entry_ir.IAdd(local, parity);
const auto second = entry_ir.ISub(entry_ir.IAdd(local, u32(1)), parity);
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(0),
entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(first, u32(2)),
entry_ir.ShiftLeftLogical(second, u32(18))));
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(1),
entry_ir.ShiftLeftLogical(entry_ir.IAdd(local, u32(2)), u32(2)));
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
entry_ir.IAdd(draw(1), entry_ir.IAdd(chunk, local)));
entry_ir.SetVectorReg(
static_cast<IR::VectorReg>(8),
entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
} else if (options.stage == ShaderType::Pixel) {
const auto* ps = options.pixel;
if (ps->ps_perspective_center_vgpr != UINT32_MAX) {
src/graphics/shader/recompiler/frontend/translate/Translator.h
+4 -4
View File
@@ -29,7 +29,7 @@ private:
IR::U32 ReadScalarCode(uint32_t code);
IR::U32 ApplyBitSourceModifiers(const Decoder::Operand& operand, IR::U32 value);
IR::Value ReadOperand(const Decoder::Operand& operand, IR::Type type);
IR::U1 ThreadBit(IR::U32 low);
IR::U1 ThreadBit(const std::array<IR::U32, 2>& mask);
void WriteRawU32(const Decoder::Operand& operand, IR::U32 value);
IR::F32 ApplyF32ResultModifiers(const Decoder::Operand& operand, IR::F32 value);
void WriteOperand(const Decoder::Operand& operand, IR::Value value);
@@ -53,8 +53,8 @@ private:
IR::U32 ConditionBit(const Decoder::Operand& operand);
IR::U1 ReadMask(const Decoder::Operand& operand);
IR::U1 ReadMaskValid(const Decoder::Operand& operand);
void WriteMask(const Decoder::Operand& operand, IR::U1 value);
void WriteMask64(const Decoder::Operand& operand, IR::U1 value);
std::array<IR::U32, 2> WriteMask(const Decoder::Operand& operand, IR::U1 value,
bool write_64 = false);
void WriteCompareResult(const Decoder::Operand& operand, IR::U1 value);
IR::MemoryFlags AddMemoryInfo(const IR::MemoryInfo& memory, uint32_t pc);
@@ -228,7 +228,7 @@ private:
void EmitControlNop();
void EmitWaitcnt();
void S_BARRIER();
void S_SENDMSG();
void S_SENDMSG(const Decoder::Instruction& inst);
void S_TTRACEDATA();
void S_INST_PREFETCH();
void S_GETPC_B64(const Decoder::Instruction& inst);
src/graphics/shader/recompiler/ir/ShaderIR.h
+1
View File
@@ -271,6 +271,7 @@ static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 43u);
struct PushData {
static constexpr uint32_t DwordCount = 32;
static constexpr uint32_t MeshDrawDwordCount = 3;
static constexpr uint32_t NoStart = UINT32_MAX;
std::array<uint32_t, DwordCount> dwords {};
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp
+1
View File
@@ -90,6 +90,7 @@ bool HasSideEffects(ValueOpcode opcode) {
case ValueOpcode::Reference:
case ValueOpcode::ReferenceU32:
case ValueOpcode::SetAttribute:
case ValueOpcode::MeshAllocate:
case ValueOpcode::Barrier: return true;
default: return false;
}
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc
+2 -1
View File
@@ -8,6 +8,8 @@ VALUE_OPCODE(ReferenceU32, Void, U32)
// Translation and register-state pseudo operations.
VALUE_OPCODE(GetUserData, U32, ScalarReg)
VALUE_OPCODE(GetShaderBase, U64)
VALUE_OPCODE(MeshDrawParameter, U32, U32)
VALUE_OPCODE(MeshAllocate, Void, U32)
VALUE_OPCODE(GetBuiltin, U32, U32, U32)
VALUE_OPCODE(GetThreadBitScalarRegister, U1, ScalarReg)
VALUE_OPCODE(SetThreadBitScalarRegister, Void, ScalarReg, U1)
@@ -81,7 +83,6 @@ VALUE_OPCODE(BitFieldSExtract, U32, U32, U32, U32)
VALUE_OPCODE(DppMoveU32, U32, U32, U1)
VALUE_OPCODE(DppUpdateU32, U32, U32, U32, U1)
VALUE_OPCODE(WqmU64, U64, U64)
VALUE_OPCODE(WqmMask, U1, U1)
// Core value operations used to expand architectural side effects.
VALUE_OPCODE(SelectU1, U1, U1, U1, U1)
src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp
+2
View File
@@ -43,6 +43,7 @@ void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t
void ValidateOptions(const Program& program, const ShaderInfoOptions& options) {
switch (program.stage) {
case ShaderType::Vertex:
case ShaderType::Mesh:
if (options.vertex == nullptr) {
return Fail("vertex shader has no input metadata");
}
@@ -365,6 +366,7 @@ void CollectShaderInfo(Program& program, const ShaderInfoOptions& options) {
});
switch (program.stage) {
case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break;
case ShaderType::Mesh: break;
case ShaderType::Pixel: CollectPixelInputs(program, options.pixel, next); break;
case ShaderType::Compute: CollectComputeInputs(options.compute, next); break;
default: return Fail("unsupported shader stage for info collection");
src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp
-153
View File
@@ -1,153 +0,0 @@
#include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include <algorithm>
#include <queue>
#include <unordered_map>
#include <unordered_set>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
enum class LdsPhase { None, Read, Write };
LdsPhase GetLdsPhase(const Program& program, const Inst& inst) {
const auto access = SharedAccessOf(inst.GetOpcode());
if (access != SharedAccess::Read && access != SharedAccess::Write) {
return LdsPhase::None;
}
const auto index = inst.Flags<MemoryFlags>().index;
if (index >= program.memory_info.size() ||
program.memory_info[index].kind != ResourceKind::Lds) {
return LdsPhase::None;
}
return access == SharedAccess::Read ? LdsPhase::Read : LdsPhase::Write;
}
bool DependsOnInvocation(Value value) {
std::queue<Value> queue;
std::unordered_set<Inst*> visited;
queue.push(value);
while (!queue.empty()) {
value = queue.front().Resolve();
queue.pop();
auto* inst = value.TryInstruction();
if (inst == nullptr || !visited.insert(inst).second) {
continue;
}
if (inst->GetOpcode() == ValueOpcode::LaneId) {
return true;
}
if (inst->GetOpcode() == ValueOpcode::GetBuiltin) {
const auto kind = static_cast<StageInputKind>(inst->Arg(0).U32());
if (kind == StageInputKind::LocalInvocationId ||
kind == StageInputKind::LocalInvocationIndex ||
kind == StageInputKind::GlobalInvocationId) {
return true;
}
}
for (size_t index = 0; index < inst->NumArgs(); index++) {
queue.push(inst->Arg(index));
}
}
return false;
}
uint32_t InsertBarrier(Block& block, Block::iterator position) {
block.PrependNewInst(position, ValueOpcode::Barrier);
return 1;
}
uint32_t InsertBlockBarriers(Program& program, Block& block) {
uint32_t inserted = 0;
LdsPhase phase = LdsPhase::None;
for (auto inst = block.begin(); inst != block.end(); ++inst) {
if (inst->GetOpcode() == ValueOpcode::Barrier) {
phase = LdsPhase::None;
continue;
}
const auto next = GetLdsPhase(program, *inst);
if (next == LdsPhase::None) {
continue;
}
if (phase != LdsPhase::None && phase != next) {
inserted += InsertBarrier(block, inst);
}
phase = next;
}
if (phase != LdsPhase::None) {
block.AppendNewInst(ValueOpcode::Barrier);
inserted++;
}
return inserted;
}
uint32_t InsertMergeBarrier(Block& block) {
auto position = std::ranges::find_if(
block, [](const Inst& inst) { return inst.GetOpcode() != ValueOpcode::Phi; });
if (position != block.end() && position->GetOpcode() == ValueOpcode::Barrier) {
return 0;
}
return InsertBarrier(block, position);
}
} // namespace
SharedMemoryBarrierStats InsertSharedMemoryBarriers(Program& program, uint32_t wave_size,
const ShaderComputeInputInfo& compute_info) {
SharedMemoryBarrierStats stats;
const auto threadgroup_size = compute_info.threads_num[0] * compute_info.threads_num[1] *
compute_info.threads_num[2];
if (wave_size != 64u || !compute_info.needs_lds_barriers ||
compute_info.lds_size_dwords == 0u || threadgroup_size != 64u) {
return stats;
}
if (program.blocks.size() != program.block_info.size()) {
return stats;
}
for (const auto& info: program.block_info) {
if (info.terminator.kind == CFG::TerminatorKind::ConditionalBranch &&
info.terminator.merge_block == UINT32_MAX && DependsOnInvocation(info.condition)) {
return stats;
}
}
std::unordered_map<uint32_t, Block*> blocks;
for (size_t index = 0; index < program.blocks.size(); index++) {
blocks.emplace(program.block_info[index].id, program.blocks[index]);
}
uint32_t divergence_depth = 0;
std::unordered_map<uint32_t, uint32_t> divergence_ends;
std::unordered_set<uint32_t> synchronized_merges;
for (size_t index = 0; index < program.blocks.size(); index++) {
const auto id = program.block_info[index].id;
if (const auto end = divergence_ends.find(id); end != divergence_ends.end()) {
EXIT_IF(end->second > divergence_depth);
divergence_depth -= end->second;
}
if (divergence_depth == 0) {
stats.inserted_barriers += InsertBlockBarriers(program, *program.blocks[index]);
}
const auto& info = program.block_info[index];
const auto& term = info.terminator;
if (term.kind != CFG::TerminatorKind::ConditionalBranch || term.merge_block == UINT32_MAX ||
!DependsOnInvocation(info.condition)) {
continue;
}
if (divergence_depth == 0 && synchronized_merges.insert(term.merge_block).second) {
const auto merge = blocks.find(term.merge_block);
if (merge != blocks.end()) {
stats.inserted_barriers += InsertMergeBarrier(*merge->second);
}
}
divergence_depth++;
divergence_ends[term.merge_block]++;
}
return stats;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h
-15
View File
@@ -1,15 +0,0 @@
#pragma once
#include "graphics/shader/recompiler/ir/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
struct SharedMemoryBarrierStats {
uint32_t inserted_barriers = 0;
};
[[nodiscard]] SharedMemoryBarrierStats
InsertSharedMemoryBarriers(Program& program, uint32_t wave_size,
const ShaderComputeInputInfo& compute_info);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/shader.cpp
+74 -26
View File
@@ -723,23 +723,22 @@ static void ShaderGetStaticInputInfoPS(
}
static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs,
const HW::ShaderRegisters& /*sh*/,
const ShaderMappedData& data,
ShaderComputeInputInfo& info) {
const HW::ShaderRegisters& /*sh*/,
const ShaderMappedData& data, ShaderComputeInputInfo& info) {
const bool dispatch_thread_dimensions = info.dispatch_thread_dimensions;
const bool needs_lds_barriers = info.needs_lds_barriers;
info = {};
info.dispatch_thread_dimensions = dispatch_thread_dimensions;
info.needs_lds_barriers = needs_lds_barriers;
info.threads_num[0] = regs.cs_regs.num_thread_x;
info.threads_num[1] = regs.cs_regs.num_thread_y;
info.threads_num[2] = regs.cs_regs.num_thread_z;
info.lds_size_dwords = static_cast<uint32_t>(regs.cs_regs.lds_size) * 128u;
info.scratch_size_dwords = data.scratch_size_dwords;
info.group_id[0] = regs.cs_regs.tgid_x_en != 0;
info.group_id[1] = regs.cs_regs.tgid_y_en != 0;
info.group_id[2] = regs.cs_regs.tgid_z_en != 0;
info.wave_size = regs.cs_regs.wave_size;
const auto host_subgroup_size = info.host_subgroup_size;
info = {};
info.dispatch_thread_dimensions = dispatch_thread_dimensions;
info.host_subgroup_size = host_subgroup_size;
info.threads_num[0] = regs.cs_regs.num_thread_x;
info.threads_num[1] = regs.cs_regs.num_thread_y;
info.threads_num[2] = regs.cs_regs.num_thread_z;
info.lds_size_dwords = static_cast<uint32_t>(regs.cs_regs.lds_size) * 128u;
info.scratch_size_dwords = data.scratch_size_dwords;
info.group_id[0] = regs.cs_regs.tgid_x_en != 0;
info.group_id[1] = regs.cs_regs.tgid_y_en != 0;
info.group_id[2] = regs.cs_regs.tgid_z_en != 0;
info.wave_size = regs.cs_regs.wave_size;
info.thread_ids_num = regs.cs_regs.tidig_comp_cnt + 1;
info.tg_size_en = regs.cs_regs.tg_size_en != 0;
@@ -768,6 +767,15 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t
}
}
key.push_back(info.mesh.threads_num[0]);
if (info.mesh.threads_num[0] != 0) {
const auto& mesh = info.mesh;
key.insert(key.end(), {mesh.wave_size, mesh.host_subgroup_size, mesh.lds_size_dwords,
mesh.scratch_size_dwords, mesh.input_primitive,
mesh.primitives_per_group, mesh.vertices_per_group,
mesh.max_vertices, mesh.max_primitives, mesh.provoking_vertex});
}
for (int i = 0; i < info.resources_num; i++) {
const auto& resource = info.resources[i];
const auto& destination = info.resources_dst[i];
@@ -822,10 +830,10 @@ void BuildStageStaticKey(const ShaderComputeInputInfo& info, std::vector<uint32_
key.clear();
key.push_back(info.workgroup_register);
key.push_back(info.wave_size);
key.push_back(info.host_subgroup_size);
key.push_back(info.thread_ids_num);
key.push_back(info.lds_size_dwords);
key.push_back(info.scratch_size_dwords);
key.push_back(static_cast<uint32_t>(info.needs_lds_barriers));
key.push_back(static_cast<uint32_t>(info.dispatch_thread_dimensions));
for (int i = 0; i < 3; i++) {
key.push_back(info.threads_num[i]);
@@ -834,15 +842,57 @@ void BuildStageStaticKey(const ShaderComputeInputInfo& info, std::vector<uint32_
key.push_back(static_cast<uint32_t>(info.tg_size_en));
}
ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderVertexInputInfo& info) {
ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& context,
const HW::UserConfig& user_config, ShaderVertexInputInfo& info) {
const auto& sh = context.GetShaderRegisters();
const auto data = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():");
if (!ShaderGetStaticInputInfoVS(regs, sh, data, info)) {
EXIT("failed to prepare vertex shader program\n");
}
return GetShaderParams(
regs.es_regs.data_addr, "ShaderRecompiler VS", GetDeclaredShaderHash(regs.es_regs.data_addr),
auto params = GetShaderParams(
regs.es_regs.data_addr, "ShaderRecompiler VS",
GetDeclaredShaderHash(regs.es_regs.data_addr),
std::span<const uint32_t>(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data);
if ((context.GetShaderStages() & 0x20u) == 0) {
if (!ShaderGetStaticInputInfoVS(regs, sh, data, info)) {
EXIT("failed to prepare vertex shader program\n");
}
return params;
}
EXIT_IF(regs.gs_regs.data_addr == 0);
const auto back = ShaderGetMappedData(regs.gs_regs.data_addr, "ShaderGetInputInfoGS():");
const auto back_params =
GetShaderParams(regs.gs_regs.data_addr, "ShaderRecompiler GS",
GetDeclaredShaderHash(regs.gs_regs.data_addr), {}, back);
params.back_code = back_params.code;
const uint64_t hashes[] = {params.hash, back_params.hash};
params.hash = XXH3_64bits(hashes, sizeof(hashes));
info = {};
info.pa_cl_vs_out_cntl = sh.m_paClVsOutCntl;
auto& mesh = info.mesh;
mesh.input_primitive = static_cast<uint32_t>(user_config.GetPrimType());
mesh.wave_size = (context.GetShaderStages() & 0x00400000u) != 0 ? 32u : 64u;
mesh.max_vertices = sh.m_geMaxOutputPerSubgroup;
mesh.provoking_vertex = context.GetModeControl().provoking_vtx_last ? 2u : 0u;
mesh.lds_size_dwords = static_cast<uint32_t>(regs.gs_regs.rsrc2.lds_size) * 128u;
mesh.scratch_size_dwords = std::max(data.scratch_size_dwords, back.scratch_size_dwords);
EXIT_NOT_IMPLEMENTED(regs.gs_regs.rsrc1.gs_vgpr_component_count != 3u ||
regs.gs_regs.rsrc2.es_vgpr_component_count != 3u);
const auto& group = user_config.GetGeControl();
if (user_config.GetPrimType() != Prospero::PrimitiveType::kTriStrip ||
sh.m_vgtGsOutPrimType != 2u || sh.m_vgtGsMaxVertOut < 3u || group.vertex_group_size < 3u ||
mesh.max_vertices == 0u) {
EXIT("unsupported GS assembly: input=%u output=%u vertices=%u GE=%u/%u max_output=%u\n",
mesh.input_primitive, sh.m_vgtGsOutPrimType, sh.m_vgtGsMaxVertOut,
group.primitive_group_size, group.vertex_group_size, mesh.max_vertices);
}
mesh.max_primitives = group.primitive_group_size * (sh.m_vgtGsMaxVertOut - 2u);
mesh.primitives_per_group = std::min({static_cast<uint32_t>(group.primitive_group_size),
static_cast<uint32_t>(group.vertex_group_size) - 2u,
mesh.max_vertices / sh.m_vgtGsMaxVertOut});
EXIT_IF(mesh.primitives_per_group == 0u);
mesh.vertices_per_group = mesh.primitives_per_group + 2u;
mesh.threads_num[0] =
((mesh.max_vertices + mesh.wave_size - 1u) / mesh.wave_size) * mesh.wave_size;
mesh.threads_num[1] = mesh.threads_num[2] = 1u;
return params;
}
ShaderParams PrepareProgram(
@@ -960,11 +1010,9 @@ void ShaderDbgDumpInputInfo(const ShaderComputeInputInfo& info) {
"\t thread_ids_num = %d\n"
"\t wave_size = %u\n"
"\t lds_size_dwords = %u\n"
"\t needs_lds_barriers = %s\n"
"\t threads_num = {%u, %u, %u}\n"
"\t tg_size_en = %s\n",
info.workgroup_register, info.thread_ids_num, info.wave_size, info.lds_size_dwords,
info.needs_lds_barriers ? "true" : "false",
info.threads_num[0], info.threads_num[1], info.threads_num[2],
info.tg_size_en ? "true" : "false");
LOGF("\t threadgroup_id = {%s, %s, %s}\n", info.group_id[0] ? "true" : "false",
src/graphics/shader/shader.h
+29 -7
View File
@@ -33,7 +33,7 @@ struct ComputeShaderInfo;
struct ShaderRegisters;
} // namespace HW
enum class ShaderType { Unknown, Vertex, Pixel, Fetch, Compute };
enum class ShaderType { Unknown, Vertex, Pixel, Fetch, Compute, Mesh };
namespace ShaderRecompiler::IR {
struct CompiledShaderInfo;
@@ -63,6 +63,23 @@ struct ShaderClipSpaceTransform {
bool enabled = false;
};
struct ShaderWorkgroupInputInfo {
uint32_t threads_num[3] = {0, 0, 0};
uint32_t lds_size_dwords = 0;
uint32_t scratch_size_dwords = 0;
uint32_t host_subgroup_size = 64;
uint32_t wave_size = 64;
};
struct ShaderMeshInputInfo: ShaderWorkgroupInputInfo {
uint32_t input_primitive = 0;
uint32_t primitives_per_group = 0;
uint32_t vertices_per_group = 0;
uint32_t max_vertices = 0;
uint32_t max_primitives = 0;
uint32_t provoking_vertex = 0;
};
struct ShaderVertexInputInfo {
static constexpr int RES_MAX = 32;
@@ -78,19 +95,15 @@ struct ShaderVertexInputInfo {
uint32_t scratch_size_dwords = 0;
uint32_t pa_cl_vs_out_cntl = 0;
ShaderClipSpaceTransform clip_space;
ShaderMeshInputInfo mesh;
bool fetch_external = false;
bool fetch_embedded = false;
};
struct ShaderComputeInputInfo {
uint32_t threads_num[3] = {0, 0, 0};
struct ShaderComputeInputInfo: ShaderWorkgroupInputInfo {
uint32_t dispatch_threads_num[3] = {0, 0, 0};
uint32_t lds_size_dwords = 0;
uint32_t scratch_size_dwords = 0;
bool group_id[3] = {false, false, false};
bool dispatch_thread_dimensions = false;
bool needs_lds_barriers = false;
uint32_t wave_size = 64;
int thread_ids_num = 0;
int workgroup_register = 0;
bool tg_size_en = false;
@@ -130,6 +143,15 @@ union ShaderStageInputInfo {
const ShaderComputeInputInfo* compute = nullptr;
};
inline const ShaderWorkgroupInputInfo* ShaderWorkgroupInput(ShaderType stage,
ShaderStageInputInfo input) {
switch (stage) {
case ShaderType::Compute: return input.compute;
case ShaderType::Mesh: return &input.vertex->mesh;
default: return nullptr;
}
}
uint32_t ShaderPixelParameterMappedLocation(const ShaderPixelInputInfo& info, uint32_t input);
uint32_t ShaderPixelParameterLocation(const ShaderPixelInputInfo& info,
std::span<const uint32_t> active_inputs, uint32_t input);
src/graphics/shader/shaderCompiler.h
+8 -2
View File
@@ -8,10 +8,16 @@
namespace Libs::Graphics {
namespace HW {
class Context;
class UserConfig;
} // namespace HW
struct ShaderParams {
std::span<const uint32_t> code;
std::span<const uint32_t> user_data;
uint64_t hash = 0;
std::span<const uint32_t> back_code;
[[nodiscard]] uint64_t Base() const {
return reinterpret_cast<uint64_t>(code.data());
@@ -22,8 +28,8 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& input_info, std::vector<ui
void BuildStageStaticKey(const ShaderPixelInputInfo& input_info, std::vector<uint32_t>& key);
void BuildStageStaticKey(const ShaderComputeInputInfo& input_info, std::vector<uint32_t>& key);
ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderVertexInputInfo& input_info);
ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& context,
const HW::UserConfig& user_config, ShaderVertexInputInfo& input_info);
ShaderParams PrepareProgram(
const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,