raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-26 04:46:53 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor(shader): preserve native wide shared operations

Browse Source
This commit is contained in:
nmzik
2026-08-19 09:11:25 +02:00
parent c9e942a972
commit c236ba713d
25 changed files with 1486 additions and 986 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
+40 -44
View File
@@ -95,12 +95,25 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
if (!program.info.samplers.empty()) {
Expect(Kind::Samplers, Dense(program.info.samplers.size()));
}
const auto uses_gds = program.values != nullptr &&
std::ranges::any_of(program.values->blocks, [](const IR::Block* block) {
return std::ranges::any_of(*block, [](const IR::Inst& inst) {
return inst.GetOpcode() == IR::ValueOpcode::GetGdsResource;
});
});
bool uses_gds = false;
if (program.values != nullptr) {
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) {
continue;
}
const auto index = inst.Flags<IR::MemoryFlags>().index;
if (index >= program.values->memory_info.size()) {
return Fail(error, "shared operation has invalid memory metadata");
}
const auto kind = program.values->memory_info[index].kind;
if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
return Fail(error, "shared operation has invalid resource kind");
}
uses_gds |= kind == IR::ResourceKind::Gds;
}
}
}
if (uses_gds) {
Expect(Kind::Gds);
}
@@ -249,6 +262,26 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
}
}
}
const auto shared_access = IR::SharedAccessOf(inst.GetOpcode());
if (shared_access != IR::SharedAccess::None) {
const auto index = inst.Flags<IR::MemoryFlags>().index;
if (index >= program.values->memory_info.size()) {
return Fail(error, "shared operation has invalid memory metadata");
}
const auto kind = program.values->memory_info[index].kind;
if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
return Fail(error, "shared operation has invalid resource kind");
}
if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) {
requirements.function_lds = true;
}
if (shared_access == IR::SharedAccess::Append ||
shared_access == IR::SharedAccess::Consume) {
MarkExactSubgroup();
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
}
}
switch (inst.GetOpcode()) {
case IR::ValueOpcode::Ballot: MarkExactSubgroup(); break;
case IR::ValueOpcode::DppMoveU32:
@@ -268,9 +301,7 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
requirements.subgroup_local_invocation_id = true;
break;
}
case IR::ValueOpcode::Permlane16U32:
case IR::ValueOpcode::GdsDataAppend:
case IR::ValueOpcode::GdsDataConsume: {
case IR::ValueOpcode::Permlane16U32: {
MarkExactSubgroup();
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
@@ -282,41 +313,6 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
requirements.subgroup_local_invocation_id = true;
break;
}
case IR::ValueOpcode::DataAppend:
case IR::ValueOpcode::DataConsume: {
MarkExactSubgroup();
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
[[fallthrough]];
}
case IR::ValueOpcode::LoadSharedU8:
case IR::ValueOpcode::LoadSharedU16:
case IR::ValueOpcode::LoadSharedU32:
case IR::ValueOpcode::WriteSharedU8:
case IR::ValueOpcode::WriteSharedU16:
case IR::ValueOpcode::WriteSharedU32:
case IR::ValueOpcode::SharedAtomicFMin32:
case IR::ValueOpcode::SharedAtomicFMax32:
case IR::ValueOpcode::SharedAtomicSwap32:
case IR::ValueOpcode::SharedAtomicIAdd32:
case IR::ValueOpcode::SharedAtomicISub32:
case IR::ValueOpcode::SharedAtomicSMin32:
case IR::ValueOpcode::SharedAtomicUMin32:
case IR::ValueOpcode::SharedAtomicSMax32:
case IR::ValueOpcode::SharedAtomicUMax32:
case IR::ValueOpcode::SharedAtomicAnd32:
case IR::ValueOpcode::SharedAtomicOr32:
case IR::ValueOpcode::SharedAtomicXor32: {
const auto index = inst.Flags<IR::MemoryFlags>().index;
if (index >= program.values->memory_info.size()) {
return Fail(error, "shared operation has invalid memory metadata");
}
if (program.stage != ShaderType::Compute &&
program.values->memory_info[index].kind == IR::ResourceKind::Lds) {
requirements.function_lds = true;
}
break;
}
case IR::ValueOpcode::LaneId:
requirements.subgroup_local_invocation_id = true;
break;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
+10 -14
View File
@@ -25,16 +25,14 @@ uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) {
const auto merged_two = state.builder.AllocateId();
const auto quad_bits = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), shifted_one, value,
ConstantU32(state, 1)});
state.builder.AddFunction(
{OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one});
state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), shifted_two, merged_one,
ConstantU32(state, 2)});
{OpShiftRightLogical, TypeU32(state), shifted_one, value, ConstantU32(state, 1)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one});
state.builder.AddFunction(
{OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two});
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), quad_bits, merged_two,
ConstantU32(state, 0x11111111u)});
{OpShiftRightLogical, TypeU32(state), shifted_two, merged_one, ConstantU32(state, 2)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two});
state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), quad_bits, merged_two, ConstantU32(state, 0x11111111u)});
state.builder.AddFunction(
{OpIMul, TypeU32(state), result, quad_bits, ConstantU32(state, 0x0fu)});
return result;
@@ -103,7 +101,7 @@ uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) {
ConstantU32(state, ScopeSubgroup), active});
const auto low = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
const auto wqm_low = EmitWqmWordU32(state, low);
const auto wqm_low = EmitWqmWordU32(state, low);
const auto lane = EmitSubgroupLocalInvocationId(state);
uint32_t mask = wqm_low;
uint32_t bit_lane = lane;
@@ -120,9 +118,9 @@ uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) {
state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), bit_lane, lane, ConstantU32(state, 31)});
}
const auto bit = state.builder.AllocateId();
const auto hit = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
const auto hit = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), bit_lane});
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
@@ -531,8 +529,6 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
case IR::ValueOpcode::GetAddressResource:
case IR::ValueOpcode::GetImageResource:
case IR::ValueOpcode::GetSamplerResource:
case IR::ValueOpcode::GetLdsResource:
case IR::ValueOpcode::GetGdsResource:
case IR::ValueOpcode::MakeImageAddress: return true;
default: return false;
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+1
View File
@@ -314,6 +314,7 @@ struct EmitterState {
std::array<uint32_t, IR::ShaderInfo::MaxBuffers> storage_buffer_offsets {};
uint32_t address_memory_variable = 0;
uint32_t gds_variable = 0;
uint32_t gds_length = 0;
uint32_t push_constant_variable = 0;
uint32_t vsharp_storage_variable = 0;
uint32_t flattened_srt_variable = 0;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+108 -89
View File
@@ -141,7 +141,11 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo
ctx.Arg(inst, 3));
}
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
return ctx.Arg(inst, 1);
if (mem.offset == 0u) {
return ctx.Arg(inst, 0);
}
return Binary(ctx.state, OpIAdd, TypeU32(ctx.state), ctx.Arg(inst, 0),
ConstantU32(ctx.state, mem.offset));
}
return AddressByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2));
}
@@ -493,35 +497,25 @@ void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor
uint32_t AtomicOpcode(IR::ValueOpcode opcode) {
switch (opcode) {
case IR::ValueOpcode::BufferAtomicSwap32:
case IR::ValueOpcode::SharedAtomicSwap32:
case IR::ValueOpcode::GdsAtomicSwap32: return OpAtomicExchange;
case IR::ValueOpcode::SharedAtomicSwap32: return OpAtomicExchange;
case IR::ValueOpcode::BufferAtomicIAdd32:
case IR::ValueOpcode::SharedAtomicIAdd32:
case IR::ValueOpcode::GdsAtomicIAdd32: return OpAtomicIAdd;
case IR::ValueOpcode::SharedAtomicIAdd32: return OpAtomicIAdd;
case IR::ValueOpcode::BufferAtomicISub32:
case IR::ValueOpcode::SharedAtomicISub32:
case IR::ValueOpcode::GdsAtomicISub32: return OpAtomicISub;
case IR::ValueOpcode::SharedAtomicISub32: return OpAtomicISub;
case IR::ValueOpcode::BufferAtomicSMin32:
case IR::ValueOpcode::SharedAtomicSMin32:
case IR::ValueOpcode::GdsAtomicSMin32: return OpAtomicSMin;
case IR::ValueOpcode::SharedAtomicSMin32: return OpAtomicSMin;
case IR::ValueOpcode::BufferAtomicUMin32:
case IR::ValueOpcode::SharedAtomicUMin32:
case IR::ValueOpcode::GdsAtomicUMin32: return OpAtomicUMin;
case IR::ValueOpcode::SharedAtomicUMin32: return OpAtomicUMin;
case IR::ValueOpcode::BufferAtomicSMax32:
case IR::ValueOpcode::SharedAtomicSMax32:
case IR::ValueOpcode::GdsAtomicSMax32: return OpAtomicSMax;
case IR::ValueOpcode::SharedAtomicSMax32: return OpAtomicSMax;
case IR::ValueOpcode::BufferAtomicUMax32:
case IR::ValueOpcode::SharedAtomicUMax32:
case IR::ValueOpcode::GdsAtomicUMax32: return OpAtomicUMax;
case IR::ValueOpcode::SharedAtomicUMax32: return OpAtomicUMax;
case IR::ValueOpcode::BufferAtomicAnd32:
case IR::ValueOpcode::SharedAtomicAnd32:
case IR::ValueOpcode::GdsAtomicAnd32: return OpAtomicAnd;
case IR::ValueOpcode::SharedAtomicAnd32: return OpAtomicAnd;
case IR::ValueOpcode::BufferAtomicOr32:
case IR::ValueOpcode::SharedAtomicOr32:
case IR::ValueOpcode::GdsAtomicOr32: return OpAtomicOr;
case IR::ValueOpcode::SharedAtomicOr32: return OpAtomicOr;
case IR::ValueOpcode::BufferAtomicXor32:
case IR::ValueOpcode::SharedAtomicXor32:
case IR::ValueOpcode::GdsAtomicXor32: return OpAtomicXor;
case IR::ValueOpcode::SharedAtomicXor32: return OpAtomicXor;
default: return 0;
}
}
@@ -614,7 +608,7 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
ctx.state.builder.AddFunction(
{OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)});
{OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)});
const auto data = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable});
@@ -623,7 +617,7 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR
mem.kind, [&](uint32_t old) {
const auto old_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), old);
const auto compare_f =
Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 3));
Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2));
const auto data_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), data);
const auto compare =
Binary(ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan,
@@ -637,9 +631,9 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR
return 0;
}
uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bool append) {
uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) {
auto& state = ctx.state;
const auto m0 = ctx.Arg(inst, 1);
const auto m0 = ctx.Arg(inst, 0);
const auto base =
Binary(state, OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
const auto size = Binary(state, OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
@@ -647,11 +641,10 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo
Binary(state, OpIAdd, TypeU32(state), base, ConstantU32(state, ctx.Memory(inst).offset));
const auto raw_index =
Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
auto mem = ctx.Memory(inst);
mem.kind = gds ? IR::ResourceKind::Gds : IR::ResourceKind::Lds;
const auto mem = ctx.Memory(inst);
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto index = EmitMemoryElementIndex(state, access, raw_index);
const auto exec = ctx.Arg(inst, 2);
const auto exec = ctx.Arg(inst, 1);
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
ConstantU32(state, ScopeSubgroup), exec});
@@ -659,8 +652,8 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
const auto count_low = state.per_invocation_masks ? low : ctx.Arg(inst, 3);
const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 4);
const auto count_low = state.per_invocation_masks ? low : ctx.Arg(inst, 2);
const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 3);
const auto count =
Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), count_low),
Unary(state, OpBitCount, TypeU32(state), count_high));
@@ -671,17 +664,19 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo
Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), first);
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
const auto m0_bounds =
gds ? Binary(state, OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
mem.kind == IR::ResourceKind::Gds
? Binary(state, OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
: Binary(state, OpULessThan, TypeBool(state),
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
const auto condition = AndCondition(
state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds)));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction({append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value,
EmitMemoryElementPointer(state, access, index),
ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
ConstantU32(state, MemorySemanticsNone), count});
state.builder.AddFunction(
{append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value,
EmitMemoryElementPointer(state, access, index),
ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? ScopeDevice : ScopeWorkgroup),
ConstantU32(state, MemorySemanticsNone), count});
return value;
});
const auto result = state.builder.AllocateId();
@@ -706,9 +701,9 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::
const MemoryResourceAccess& resource,
uint32_t components, FormattedAccess access) {
PreparedFormattedMemory plan;
plan.format = BufferFormat(ctx, inst, mem);
plan.info = Format::GetFormatInfo(plan.format);
plan.resource = resource;
plan.format = BufferFormat(ctx, inst, mem);
plan.info = Format::GetFormatInfo(plan.format);
plan.resource = resource;
std::array<bool, 4> required_components {};
if (access == FormattedAccess::Load) {
for (uint32_t output = 0; output < components; output++) {
@@ -718,8 +713,8 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::
}
}
} else {
for (uint32_t component = 0;
component < std::min(components, plan.info.component_count); component++) {
for (uint32_t component = 0; component < std::min(components, plan.info.component_count);
component++) {
required_components[component] = true;
}
}
@@ -812,7 +807,7 @@ uint32_t LoadWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t co
mem.formatted ? BufferFormat(ctx, inst, mem) : Prospero::BufferFormat::kInvalid;
if (Format::IsKnownFormat(format)) {
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, components,
FormattedAccess::Load);
FormattedAccess::Load);
return EmitValueOrDefaultIfCondition(
state, plan.in_bounds, TypeU32Composite(state, components),
FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() {
@@ -842,7 +837,7 @@ void StoreWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
mem.formatted ? BufferFormat(ctx, inst, mem) : Prospero::BufferFormat::kInvalid;
if (Format::IsKnownFormat(format)) {
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, components,
FormattedAccess::Store);
FormattedAccess::Store);
EmitIfCondition(state, plan.in_bounds, [&]() {
for (uint32_t component = 0; component < components; component++) {
const auto data = state.builder.AllocateId();
@@ -862,6 +857,54 @@ void StoreWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
});
}
uint32_t LoadWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
auto& state = ctx.state;
return EmitValueOrDefaultIfCondition(
state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU32Composite(state, components),
ConstantU32CompositeZero(state, components), [&]() {
const auto mem = ctx.Memory(inst);
const auto resource = PrepareMemoryResourceAccess(state, mem);
const auto base = ByteAddress(ctx, inst, mem);
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; component++) {
const auto address = component == 0u
? base
: Binary(state, OpIAdd, TypeU32(state), base,
ConstantU32(state, component * 4u));
const auto raw_index = Binary(state, OpShiftRightLogical, TypeU32(state), address,
ConstantU32(state, 2));
const auto access = PrepareMemoryElement(ctx, mem, resource, raw_index);
values[component] = EmitValueOrZeroIfCondition(
state, EmitMemoryElementInBounds(state, access.resource, access.index),
[&]() { return LoadWordInBounds(ctx, access.resource, access.index); });
}
return ConstructU32Composite(state, components, values);
});
}
void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
auto& state = ctx.state;
EmitIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto mem = ctx.Memory(inst);
const auto resource = PrepareMemoryResourceAccess(state, mem);
const auto base = ByteAddress(ctx, inst, mem);
for (uint32_t component = 0; component < components; component++) {
const auto address = component == 0u
? base
: Binary(state, OpIAdd, TypeU32(state), base,
ConstantU32(state, component * 4u));
const auto raw_index =
Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto access = PrepareMemoryElement(ctx, mem, resource, raw_index);
EmitIfCondition(state, EmitMemoryElementInBounds(state, access.resource, access.index),
[&]() {
StoreWordInBounds(ctx, access.resource, access.index,
ctx.Arg(inst, component + 1u));
});
}
});
}
} // namespace
bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -876,6 +919,15 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
}
return true;
}
const auto shared_components = IR::SharedComponentCount(op);
if (shared_components > 1u) {
if (IR::SharedAccessOf(op) == IR::SharedAccess::Read) {
ctx.Define(inst, LoadWideShared(ctx, inst, shared_components));
} else {
StoreWideShared(ctx, inst, shared_components);
}
return true;
}
if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) &&
ctx.Memory(inst).planning_only) {
return true;
@@ -917,25 +969,17 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 ||
op == IR::ValueOpcode::LoadBufferU16 ||
op == IR::ValueOpcode::LoadBufferU32;
const bool load_shared =
op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadSharedU16 ||
op == IR::ValueOpcode::LoadSharedU32 || op == IR::ValueOpcode::LoadGdsU8 ||
op == IR::ValueOpcode::LoadGdsU16 || op == IR::ValueOpcode::LoadGdsU32;
const bool load_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Read;
if (load_address || load_buffer || load_shared) {
auto mem = ctx.Memory(inst);
if (load_shared)
mem.kind = op == IR::ValueOpcode::LoadGdsU8 || op == IR::ValueOpcode::LoadGdsU16 ||
op == IR::ValueOpcode::LoadGdsU32
? IR::ResourceKind::Gds
: IR::ResourceKind::Lds;
uint32_t value = 0;
const auto mem = ctx.Memory(inst);
uint32_t value = 0;
if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
value = FormattedLoad(ctx, inst, mem);
else if (op == IR::ValueOpcode::LoadAddressU8 || op == IR::ValueOpcode::LoadBufferU8 ||
op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadGdsU8)
op == IR::ValueOpcode::LoadSharedU8)
value = LoadSubword(ctx, inst, mem, 8, false);
else if (op == IR::ValueOpcode::LoadAddressU16 || op == IR::ValueOpcode::LoadBufferU16 ||
op == IR::ValueOpcode::LoadSharedU16 || op == IR::ValueOpcode::LoadGdsU16)
op == IR::ValueOpcode::LoadSharedU16)
value = LoadSubword(ctx, inst, mem, 16, false);
else
value = LoadWord(ctx, inst, mem);
@@ -948,24 +992,16 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
const bool store_buffer = op == IR::ValueOpcode::StoreBufferU8 ||
op == IR::ValueOpcode::StoreBufferU16 ||
op == IR::ValueOpcode::StoreBufferU32;
const bool store_shared =
op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteSharedU16 ||
op == IR::ValueOpcode::WriteSharedU32 || op == IR::ValueOpcode::WriteGdsU8 ||
op == IR::ValueOpcode::WriteGdsU16 || op == IR::ValueOpcode::WriteGdsU32;
const bool store_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Write;
if (store_address || store_buffer || store_shared) {
auto mem = ctx.Memory(inst);
if (store_shared)
mem.kind = op == IR::ValueOpcode::WriteGdsU8 || op == IR::ValueOpcode::WriteGdsU16 ||
op == IR::ValueOpcode::WriteGdsU32
? IR::ResourceKind::Gds
: IR::ResourceKind::Lds;
const auto mem = ctx.Memory(inst);
if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted)
FormattedStore(ctx, inst, mem);
else if (op == IR::ValueOpcode::StoreAddressU8 || op == IR::ValueOpcode::StoreBufferU8 ||
op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteGdsU8)
op == IR::ValueOpcode::WriteSharedU8)
StoreSubword(ctx, inst, mem, 8);
else if (op == IR::ValueOpcode::StoreAddressU16 || op == IR::ValueOpcode::StoreBufferU16 ||
op == IR::ValueOpcode::WriteSharedU16 || op == IR::ValueOpcode::WriteGdsU16)
op == IR::ValueOpcode::WriteSharedU16)
StoreSubword(ctx, inst, mem, 16);
else
StoreWord(ctx, inst, mem);
@@ -973,12 +1009,7 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
}
const auto atomic_opcode = AtomicOpcode(op);
if (atomic_opcode != 0) {
auto mem = ctx.Memory(inst);
if (op >= IR::ValueOpcode::SharedAtomicSwap32 && op <= IR::ValueOpcode::SharedAtomicXor32)
mem.kind = IR::ResourceKind::Lds;
else if (op >= IR::ValueOpcode::GdsAtomicSwap32 && op <= IR::ValueOpcode::GdsAtomicXor32)
mem.kind = IR::ResourceKind::Gds;
ctx.Define(inst, EmitAtomic(ctx, inst, mem, true));
ctx.Define(inst, EmitAtomic(ctx, inst, ctx.Memory(inst), true));
return true;
}
if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) {
@@ -986,24 +1017,12 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
op == IR::ValueOpcode::BufferAtomicFMax32));
return true;
}
if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32 ||
op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32) {
auto mem = ctx.Memory(inst);
mem.kind = op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32
? IR::ResourceKind::Gds
: IR::ResourceKind::Lds;
SharedFloatAtomic(ctx, inst, mem,
op == IR::ValueOpcode::SharedAtomicFMax32 ||
op == IR::ValueOpcode::GdsAtomicFMax32);
if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32) {
SharedFloatAtomic(ctx, inst, ctx.Memory(inst), op == IR::ValueOpcode::SharedAtomicFMax32);
return true;
}
if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume ||
op == IR::ValueOpcode::GdsDataAppend || op == IR::ValueOpcode::GdsDataConsume) {
ctx.Define(inst, AppendConsume(ctx, inst,
op == IR::ValueOpcode::GdsDataAppend ||
op == IR::ValueOpcode::GdsDataConsume,
op == IR::ValueOpcode::DataAppend ||
op == IR::ValueOpcode::GdsDataAppend));
if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume) {
ctx.Define(inst, AppendConsume(ctx, inst, op == IR::ValueOpcode::DataAppend));
return true;
}
if (op == IR::ValueOpcode::SwizzleU32) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+5 -1
View File
@@ -93,7 +93,11 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
"GDS binding was not emitted");
}
access.object_pointer = state.gds_variable;
break;
if (state.gds_length == 0) {
EXIT("GDS length was not prepared at function entry\n");
}
access.length = state.gds_length;
return access;
case IR::ResourceKind::ScalarAddress:
case IR::ResourceKind::Flat:
case IR::ResourceKind::Global:
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
+6 -3
View File
@@ -592,9 +592,7 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std:
if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32 ||
inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMin32 ||
inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMax32;
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
})) {
ctx.scratch_u32_variable = state.builder.AllocateId();
break;
@@ -631,6 +629,11 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std:
TypePointer(state, StorageClassFunction, TypeU32(state)),
ctx.scratch_u32_variable, StorageClassFunction});
}
if (state.gds_variable != 0) {
state.gds_length = state.builder.AllocateId();
state.builder.AddFunction(
{OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0});
}
if (state.pixel_valid_mask_variable != 0) {
state.builder.AddFunction(
{OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp
+4 -3
View File
@@ -96,7 +96,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
{0x29u, Opcode::DS_AND_RTN_B32, 1, 32}, {0x2au, Opcode::DS_OR_RTN_B32, 1, 32},
{0x2bu, Opcode::DS_XOR_RTN_B32, 1, 32}, {0x2du, Opcode::DS_WRXCHG_RTN_B32, 1, 32},
{0x35u, Opcode::DS_SWIZZLE_B32, 1, 32}, {0x36u, Opcode::DS_READ_B32, 1, 32},
{0x37u, Opcode::DS_READ2_B32, 2, 32}, {0x38u, Opcode::DS_READ2_B32, 2, 32},
{0x37u, Opcode::DS_READ2_B32, 2, 32}, {0x38u, Opcode::DS_READ2ST64_B32, 2, 32},
{0x39u, Opcode::DS_READ_I8, 1, 8, true}, {0x3au, Opcode::DS_READ_U8, 1, 8},
{0x3bu, Opcode::DS_READ_I16, 1, 16, true}, {0x3cu, Opcode::DS_READ_U16, 1, 16},
{0x3du, Opcode::DS_CONSUME, 1, 32}, {0x3eu, Opcode::DS_APPEND, 1, 32},
@@ -403,10 +403,11 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
SetUnsupported(inst, Family::DS, opcode,
"DS read addtid data operands are not implemented");
}
if (inst.opcode == Opcode::DS_WRITE2_B32 || opcode == 0x37u) {
if (inst.opcode == Opcode::DS_WRITE2_B32 || inst.opcode == Opcode::DS_READ2_B32) {
inst.offset = offset0 * 4u;
inst.secondary_offset = offset1 * 4u;
} else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 || opcode == 0x38u) {
} else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 ||
inst.opcode == Opcode::DS_READ2ST64_B32) {
inst.offset = offset0 * 256u;
inst.secondary_offset = offset1 * 256u;
} else if (inst.opcode == Opcode::DS_WRITE2_B64 || inst.opcode == Opcode::DS_READ2_B64) {
src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp
+1
View File
@@ -658,6 +658,7 @@ std::string InstructionToString(const Instruction& inst) {
case Opcode::DS_READ_I16:
case Opcode::DS_READ_U16:
case Opcode::DS_READ2_B32:
case Opcode::DS_READ2ST64_B32:
case Opcode::DS_READ_B32:
case Opcode::DS_READ_B64:
case Opcode::DS_READ2_B64:
src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h
+1
View File
@@ -490,6 +490,7 @@ enum class Opcode {
DS_READ_I16,
DS_READ_U16,
DS_READ2_B32,
DS_READ2ST64_B32,
DS_READ_B32,
DS_READ_B64,
DS_READ2_B64,
src/graphics/shader/recompiler/frontend/translate/Memory.cpp
+124 -64
View File
@@ -65,11 +65,6 @@ IR::Value Translator::GetSamplerResource(const IR::MemoryInfo& memory) {
GetResourceDword(memory.sampler, 2), GetResourceDword(memory.sampler, 3)});
}
IR::Value Translator::GetSharedResource(IR::ResourceKind kind) {
return ir.Emit(kind == IR::ResourceKind::Gds ? IR::ValueOpcode::GetGdsResource
: IR::ValueOpcode::GetLdsResource);
}
IR::Value Translator::MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base) {
std::array<IR::Value, 13> components {};
components[0] = ReadRawU32(PlainOperand(base));
@@ -170,22 +165,7 @@ IR::ValueOpcode Translator::BufferAtomicOpcode(IR::Opcode opcode) {
}
}
IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) {
if (gds) {
switch (opcode) {
case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::GdsAtomicSwap32;
case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::GdsAtomicIAdd32;
case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::GdsAtomicISub32;
case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::GdsAtomicSMin32;
case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::GdsAtomicUMin32;
case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::GdsAtomicSMax32;
case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::GdsAtomicUMax32;
case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::GdsAtomicAnd32;
case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::GdsAtomicOr32;
case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::GdsAtomicXor32;
default: EXIT("invalid GDS atomic opcode");
}
}
IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode) {
switch (opcode) {
case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::SharedAtomicSwap32;
case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::SharedAtomicIAdd32;
@@ -197,7 +177,7 @@ IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) {
case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::SharedAtomicAnd32;
case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::SharedAtomicOr32;
case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::SharedAtomicXor32;
default: EXIT("invalid LDS atomic opcode");
default: EXIT("invalid shared atomic opcode");
}
}
@@ -363,13 +343,9 @@ bool Translator::TranslateAtomicMemory(const IR::Instruction& inst) {
}
case IR::ResourceKind::Lds:
case IR::ResourceKind::Gds: {
const bool gds = inst.memory.kind == IR::ResourceKind::Gds;
const auto resource = GetSharedResource(inst.memory.kind);
const auto address =
ir.IAdd(ReadU32(inst.src[1]), IR::U32(IR::Value(inst.memory.offset)));
result = ir.Emit(SharedAtomicOpcode(inst.op, gds),
{resource, address, ReadU32(inst.src[0]), ir.GetExec()},
AddMemoryInfo(inst));
const auto address = ReadU32(inst.src[1]);
result = ir.Emit(SharedAtomicOpcode(inst.op),
{address, ReadU32(inst.src[0]), ir.GetExec()}, AddMemoryInfo(inst));
break;
}
default: return false;
@@ -518,10 +494,51 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
if (!shared && inst.op != IR::Opcode::DsSwizzleB32) {
return false;
}
const bool gds = inst.memory.kind == IR::ResourceKind::Gds;
const auto resource = shared ? GetSharedResource(inst.memory.kind) : IR::Value {};
const auto shared_address = [&](uint32_t source) {
return ir.IAdd(ReadU32(inst.src[source]), IR::U32(IR::Value(inst.memory.offset)));
const auto shared_address = [&](uint32_t source) { return ReadU32(inst.src[source]); };
const auto load_u32 = [&](uint32_t width, IR::U32 address, const IR::Instruction& source) {
IR::ValueOpcode opcode;
switch (width) {
case 1u: opcode = IR::ValueOpcode::LoadSharedU32; break;
case 2u: opcode = IR::ValueOpcode::LoadSharedU32x2; break;
case 3u: opcode = IR::ValueOpcode::LoadSharedU32x3; break;
case 4u: opcode = IR::ValueOpcode::LoadSharedU32x4; break;
default: EXIT("invalid shared load width");
}
return ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(source));
};
const auto extract_u32 = [&](IR::Value value, uint32_t width, uint32_t index) {
if (width == 1u) {
return value;
}
const auto opcode = width == 2u ? IR::ValueOpcode::CompositeExtractU32x2
: width == 3u ? IR::ValueOpcode::CompositeExtractU32x3
: IR::ValueOpcode::CompositeExtractU32x4;
return ir.Emit(opcode, {value, IR::Value(index)});
};
const auto write_u32 = [&](uint32_t width, IR::U32 address,
const std::array<IR::Value, 4>& values,
const IR::Instruction& source) {
switch (width) {
case 1u:
ir.Emit(IR::ValueOpcode::WriteSharedU32, {address, values[0], ir.GetExec()},
AddMemoryInfo(source));
break;
case 2u:
ir.Emit(IR::ValueOpcode::WriteSharedU32x2,
{address, values[0], values[1], ir.GetExec()}, AddMemoryInfo(source));
break;
case 3u:
ir.Emit(IR::ValueOpcode::WriteSharedU32x3,
{address, values[0], values[1], values[2], ir.GetExec()},
AddMemoryInfo(source));
break;
case 4u:
ir.Emit(IR::ValueOpcode::WriteSharedU32x4,
{address, values[0], values[1], values[2], values[3], ir.GetExec()},
AddMemoryInfo(source));
break;
default: EXIT("invalid shared store width");
}
};
switch (inst.op) {
case IR::Opcode::DsReadUbyte:
@@ -533,65 +550,109 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
uint32_t bits;
bool sign;
if (inst.op == IR::Opcode::DsReadUbyte || inst.op == IR::Opcode::DsReadSbyte) {
opcode = gds ? IR::ValueOpcode::LoadGdsU8 : IR::ValueOpcode::LoadSharedU8;
opcode = IR::ValueOpcode::LoadSharedU8;
bits = 8u;
sign = inst.op == IR::Opcode::DsReadSbyte;
} else if (inst.op == IR::Opcode::DsReadUshort || inst.op == IR::Opcode::DsReadSshort) {
opcode = gds ? IR::ValueOpcode::LoadGdsU16 : IR::ValueOpcode::LoadSharedU16;
opcode = IR::ValueOpcode::LoadSharedU16;
bits = 16u;
sign = inst.op == IR::Opcode::DsReadSshort;
} else {
opcode = gds ? IR::ValueOpcode::LoadGdsU32 : IR::ValueOpcode::LoadSharedU32;
bits = 32u;
sign = false;
const auto width = inst.memory.data_dwords;
const auto loaded = load_u32(width, shared_address(0), inst);
for (uint32_t index = 0; index < width; index++) {
WriteOperand(OffsetOperand(inst.dst, index), extract_u32(loaded, width, index));
}
return true;
}
const auto loaded =
ir.Emit(opcode, {resource, shared_address(0), ir.GetExec()}, AddMemoryInfo(inst));
ir.Emit(opcode, {shared_address(0), ir.GetExec()}, AddMemoryInfo(inst));
WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign));
return true;
}
case IR::Opcode::DsRead2B32: {
const auto width = inst.memory.data_dwords / 2u;
const auto address = shared_address(0);
auto first = inst;
first.memory.data_dwords = width;
first.memory.component_count = width;
const auto first_value = load_u32(width, address, first);
IR::Value second_value = first_value;
if (inst.memory.secondary_offset != inst.memory.offset) {
auto second = first;
second.memory.offset = inst.memory.secondary_offset;
second_value = load_u32(width, address, second);
}
for (uint32_t index = 0; index < width; index++) {
WriteOperand(OffsetOperand(inst.dst, index),
extract_u32(first_value, width, index));
WriteOperand(OffsetOperand(inst.dst, width + index),
extract_u32(second_value, width, index));
}
return true;
}
case IR::Opcode::DsWriteByte:
case IR::Opcode::DsWriteShort:
case IR::Opcode::DsWriteB32: {
const auto data = ReadU32(inst.src[0]);
const auto width = inst.memory.data_dwords;
std::array<IR::Value, 4> values {};
for (uint32_t index = 0; index < width; index++) {
values[index] = ReadU32(OffsetOperand(inst.src[0], index));
}
const auto data = IR::U32(values[0]);
const auto address = shared_address(1);
IR::ValueOpcode opcode;
IR::Value value;
if (inst.op == IR::Opcode::DsWriteByte) {
opcode = gds ? IR::ValueOpcode::WriteGdsU8 : IR::ValueOpcode::WriteSharedU8;
opcode = IR::ValueOpcode::WriteSharedU8;
value = NarrowSubdword(data, 8u);
} else if (inst.op == IR::Opcode::DsWriteShort) {
opcode = gds ? IR::ValueOpcode::WriteGdsU16 : IR::ValueOpcode::WriteSharedU16;
opcode = IR::ValueOpcode::WriteSharedU16;
value = NarrowSubdword(data, 16u);
} else {
opcode = gds ? IR::ValueOpcode::WriteGdsU32 : IR::ValueOpcode::WriteSharedU32;
value = data;
write_u32(width, address, values, inst);
return true;
}
ir.Emit(opcode, {address, value, ir.GetExec()}, AddMemoryInfo(inst));
return true;
}
case IR::Opcode::DsWrite2B32: {
const auto width = inst.memory.data_dwords / 2u;
const auto address = shared_address(1);
std::array<IR::Value, 4> first_values {};
std::array<IR::Value, 4> second_values {};
for (uint32_t index = 0; index < width; index++) {
first_values[index] = ReadU32(OffsetOperand(inst.src[0], index));
second_values[index] = ReadU32(OffsetOperand(inst.src[2], index));
}
auto first = inst;
first.memory.data_dwords = width;
first.memory.component_count = width;
write_u32(width, address, first_values, first);
if (inst.memory.secondary_offset != inst.memory.offset) {
auto second = first;
second.memory.offset = inst.memory.secondary_offset;
write_u32(width, address, second_values, second);
}
ir.Emit(opcode, {resource, address, value, ir.GetExec()}, AddMemoryInfo(inst));
return true;
}
case IR::Opcode::DsMinF32:
case IR::Opcode::DsMaxF32: {
const auto opcode =
gds ? (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::GdsAtomicFMin32
: IR::ValueOpcode::GdsAtomicFMax32)
: (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::SharedAtomicFMin32
: IR::ValueOpcode::SharedAtomicFMax32);
const auto opcode = inst.op == IR::Opcode::DsMinF32
? IR::ValueOpcode::SharedAtomicFMin32
: IR::ValueOpcode::SharedAtomicFMax32;
ir.Emit(opcode,
{resource, shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]),
ir.GetExec()},
{shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]), ir.GetExec()},
AddMemoryInfo(inst));
return true;
}
case IR::Opcode::DsAppend:
case IR::Opcode::DsConsume: {
const bool append = inst.op == IR::Opcode::DsAppend;
const auto opcode =
gds ? (append ? IR::ValueOpcode::GdsDataAppend : IR::ValueOpcode::GdsDataConsume)
: (append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume);
const auto opcode = append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume;
WriteOperand(inst.dst, ir.Emit(opcode,
{resource, ReadU32(inst.src[0]), ir.GetExec(),
ir.GetExecLo(), ir.GetExecHi()},
{ReadU32(inst.src[0]), ir.GetExec(), ir.GetExecLo(),
ir.GetExecHi()},
AddMemoryInfo(inst)));
return true;
}
@@ -601,16 +662,13 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
const auto base =
ir.BitwiseAnd(ReadU32(inst.src[m0_index]), IR::U32(IR::Value(0xffffu)));
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
const auto address = ir.IAdd(ir.IAdd(base, IR::U32(IR::Value(inst.memory.offset))),
ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u))));
const auto address = ir.IAdd(base, ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u))));
if (inst.op == IR::Opcode::DsWriteAddtidB32) {
ir.Emit(IR::ValueOpcode::WriteSharedU32,
{resource, address, ReadU32(inst.src[0]), ir.GetExec()},
AddMemoryInfo(inst));
{address, ReadU32(inst.src[0]), ir.GetExec()}, AddMemoryInfo(inst));
} else {
WriteOperand(inst.dst,
ir.Emit(IR::ValueOpcode::LoadSharedU32,
{resource, address, ir.GetExec()}, AddMemoryInfo(inst)));
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::LoadSharedU32,
{address, ir.GetExec()}, AddMemoryInfo(inst)));
}
return true;
}
@@ -676,9 +734,11 @@ bool Translator::TranslateMemoryOperation(const IR::Instruction& inst,
case IR::Opcode::DsReadUshort:
case IR::Opcode::DsReadSshort:
case IR::Opcode::DsReadB32:
case IR::Opcode::DsRead2B32:
case IR::Opcode::DsWriteByte:
case IR::Opcode::DsWriteShort:
case IR::Opcode::DsWriteB32:
case IR::Opcode::DsWrite2B32:
case IR::Opcode::DsMinF32:
case IR::Opcode::DsMaxF32:
case IR::Opcode::DsSwizzleB32:
src/graphics/shader/recompiler/frontend/translate/Translate.cpp
+8
View File
@@ -768,6 +768,14 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u));
} else if (inst.op == IR::Opcode::BufferStoreDword) {
include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u));
} else if (inst.op == IR::Opcode::DsReadB32 || inst.op == IR::Opcode::DsRead2B32) {
include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u));
} else if (inst.op == IR::Opcode::DsWriteB32) {
include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u));
} else if (inst.op == IR::Opcode::DsWrite2B32) {
const auto width = std::max(inst.memory.data_dwords / 2u, 1u);
include_vector(inst.src[0], width);
include_vector(inst.src[2], width);
} else if (inst.op == IR::Opcode::LoadInputF32) {
include_vector(inst.dst, std::max(inst.input_info.component_count, 1u));
}
src/graphics/shader/recompiler/frontend/translate/Translator.h
+1 -2
View File
@@ -75,7 +75,6 @@ private:
IR::Value GetScalarAddressResource(uint32_t base);
IR::Value GetImageResource(const IR::MemoryInfo& memory);
IR::Value GetSamplerResource(const IR::MemoryInfo& memory);
IR::Value GetSharedResource(IR::ResourceKind kind);
IR::Value MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base);
IR::Value ConstructU32x4(const IR::Operand& base, uint32_t count);
void WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask,
@@ -85,7 +84,7 @@ private:
IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign);
IR::Value NarrowSubdword(IR::U32 value, uint32_t bits);
IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode);
IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode, bool gds);
IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode);
bool IsScalarMemoryLoadOperation(IR::Opcode opcode);
ScalarMemorySourceValues ReadScalarMemorySource(const IR::Instruction& inst);
bool TranslateScalarMemory(const IR::Instruction& inst,
src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp
+46 -62
View File
@@ -168,42 +168,33 @@ ResourceKind DsMemoryKind(const Decoder::Instruction& decoded) {
}
bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto ir_op = DsReadOpcode(decoded);
const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
for (uint32_t i = 0; i < count; i++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.src_count = 1;
inst.memory = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block.instructions.push_back(inst);
const auto ir_op = DsReadOpcode(decoded);
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.src_count = 1;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block.instructions.push_back(inst);
return true;
}
bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock& block, uint32_t dwords_per_read,
std::string* error) {
const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset};
for (uint32_t read = 0; read < 2u; read++) {
for (uint32_t dword = 0; dword < dwords_per_read; dword++) {
const auto index = read * dwords_per_read + dword;
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::DsReadB32;
inst.src_count = 1;
inst.memory =
ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[read] + dword * 4u);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, index), inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block.instructions.push_back(inst);
}
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::DsRead2B32;
inst.src_count = 1;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
inst.memory.data_dwords = dwords_per_read * 2u;
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block.instructions.push_back(inst);
return true;
}
@@ -408,45 +399,36 @@ bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock& block, std:
}
bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto ir_op = DsWriteOpcode(decoded.data_bits);
const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
for (uint32_t i = 0; i < count; i++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.src_count = 2;
inst.memory = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.src1, i), inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error)) {
return false;
}
block.instructions.push_back(inst);
const auto ir_op = DsWriteOpcode(decoded.data_bits);
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.src_count = 2;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error)) {
return false;
}
block.instructions.push_back(inst);
return true;
}
bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock& block,
uint32_t dwords_per_write, std::string* error) {
const Decoder::Operand* data[] = {&decoded.src1, &decoded.src2};
const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset};
for (uint32_t write = 0; write < 2u; write++) {
for (uint32_t dword = 0; dword < dwords_per_write; dword++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::DsWriteB32;
inst.src_count = 2;
inst.memory =
ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[write] + dword * 4u);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(*data[write], dword), inst.src[0],
error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error)) {
return false;
}
block.instructions.push_back(inst);
}
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::DsWrite2B32;
inst.src_count = 3;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
inst.memory.data_dwords = dwords_per_write * 2u;
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error) ||
!LowerSourceOperand(decoded.src2, inst.src[2], error)) {
return false;
}
block.instructions.push_back(inst);
return true;
}
@@ -618,7 +600,8 @@ bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& blo
return LowerDsWriteAddtidB32(decoded, block, error);
case Decoder::Opcode::DS_READ_ADDTID_B32:
return LowerDsReadAddtidB32(decoded, block, error);
case Decoder::Opcode::DS_READ2_B32: return LowerDsRead2(decoded, block, 1, error);
case Decoder::Opcode::DS_READ2_B32:
case Decoder::Opcode::DS_READ2ST64_B32: return LowerDsRead2(decoded, block, 1, error);
case Decoder::Opcode::DS_READ2_B64:
case Decoder::Opcode::DS_READ2ST64_B64: return LowerDsRead2(decoded, block, 2, error);
case Decoder::Opcode::DS_READ_I8:
@@ -765,6 +748,7 @@ bool IsMemoryOpcode(Decoder::Opcode opcode) {
case Decoder::Opcode::DS_READ_I16:
case Decoder::Opcode::DS_READ_U16:
case Decoder::Opcode::DS_READ2_B32:
case Decoder::Opcode::DS_READ2ST64_B32:
case Decoder::Opcode::DS_READ_B32:
case Decoder::Opcode::DS_READ_B64:
case Decoder::Opcode::DS_READ2_B64:
src/graphics/shader/recompiler/ir/Type.cpp
-2
View File
@@ -31,8 +31,6 @@ std::string TypeName(Type type) {
"AddressResource",
"ImageResource",
"SamplerResource",
"LdsResource",
"GdsResource",
"ImageAddress",
};
const auto bits = static_cast<uint32_t>(type);
src/graphics/shader/recompiler/ir/Type.h
+1 -3
View File
@@ -28,9 +28,7 @@ enum class Type : uint32_t {
AddressResource = 1u << 18u,
ImageResource = 1u << 19u,
SamplerResource = 1u << 20u,
LdsResource = 1u << 21u,
GdsResource = 1u << 22u,
ImageAddress = 1u << 23u,
ImageAddress = 1u << 21u,
};
constexpr Type operator|(Type lhs, Type rhs) {
src/graphics/shader/recompiler/ir/ValueProgram.cpp
+33 -4
View File
@@ -338,6 +338,34 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st
ValueOpcodeName(inst.GetOpcode())));
}
}
const auto shared_components = SharedComponentCount(inst.GetOpcode());
if (shared_components != 0u) {
const auto memory_index = inst.Flags<MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) {
return Fail(error, fmt::format("{} has an invalid memory-info index",
ValueOpcodeName(inst.GetOpcode())));
}
const auto& memory = program.memory_info[memory_index];
if ((memory.kind != ResourceKind::Lds && memory.kind != ResourceKind::Gds) ||
memory.resource != 0u || memory.sampler != 0u || memory.component_count == 0u ||
memory.component_index >= memory.component_count) {
return Fail(error, fmt::format("{} has invalid shared-memory metadata",
ValueOpcodeName(inst.GetOpcode())));
}
uint32_t expected_bits = 32u;
if (inst.GetOpcode() == ValueOpcode::LoadSharedU8 ||
inst.GetOpcode() == ValueOpcode::WriteSharedU8) {
expected_bits = 8u;
} else if (inst.GetOpcode() == ValueOpcode::LoadSharedU16 ||
inst.GetOpcode() == ValueOpcode::WriteSharedU16) {
expected_bits = 16u;
}
if (memory.data_bits != expected_bits || memory.data_dwords != shared_components ||
(shared_components > 1u && memory.component_index != 0u)) {
return Fail(error, fmt::format("{} has inconsistent shared-memory width",
ValueOpcodeName(inst.GetOpcode())));
}
}
uint32_t composite_components = 0u;
switch (inst.GetOpcode()) {
case ValueOpcode::CompositeExtractU32x2: composite_components = 2u; break;
@@ -358,10 +386,11 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st
ValueOpcodeName(inst.GetOpcode())));
}
if (fixed_signature && arg.GetType() != ArgTypeOf(inst.GetOpcode(), arg_index)) {
return Fail(error, fmt::format("{} argument {} has type {}, expected {}",
ValueOpcodeName(inst.GetOpcode()), arg_index,
TypeName(arg.GetType()),
TypeName(ArgTypeOf(inst.GetOpcode(), arg_index))));
return Fail(error,
fmt::format("{} argument {} has type {}, expected {}",
ValueOpcodeName(inst.GetOpcode()), arg_index,
TypeName(arg.GetType()),
TypeName(ArgTypeOf(inst.GetOpcode(), arg_index))));
}
if (const auto* definition = arg.TryInstruction();
definition != nullptr && !instructions.contains(definition)) {
src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp
+8 -7
View File
@@ -392,19 +392,20 @@ constexpr LowerMap LOWER_OPS[] = {
{Decoder::Opcode::DS_READ_U8, Opcode::DsReadUbyte},
{Decoder::Opcode::DS_READ_I16, Opcode::DsReadSshort},
{Decoder::Opcode::DS_READ_U16, Opcode::DsReadUshort},
{Decoder::Opcode::DS_READ2_B32, Opcode::DsReadB32},
{Decoder::Opcode::DS_READ2_B32, Opcode::DsRead2B32},
{Decoder::Opcode::DS_READ2ST64_B32, Opcode::DsRead2B32},
{Decoder::Opcode::DS_READ_B32, Opcode::DsReadB32},
{Decoder::Opcode::DS_READ_B64, Opcode::DsReadB32},
{Decoder::Opcode::DS_READ2_B64, Opcode::DsReadB32},
{Decoder::Opcode::DS_READ2ST64_B64, Opcode::DsReadB32},
{Decoder::Opcode::DS_READ2_B64, Opcode::DsRead2B32},
{Decoder::Opcode::DS_READ2ST64_B64, Opcode::DsRead2B32},
{Decoder::Opcode::DS_READ_B96, Opcode::DsReadB32},
{Decoder::Opcode::DS_READ_B128, Opcode::DsReadB32},
{Decoder::Opcode::DS_WRITE_B8, Opcode::DsWriteByte},
{Decoder::Opcode::DS_WRITE_B16, Opcode::DsWriteShort},
{Decoder::Opcode::DS_WRITE2_B32, Opcode::DsWriteB32},
{Decoder::Opcode::DS_WRITE2ST64_B32, Opcode::DsWriteB32},
{Decoder::Opcode::DS_WRITE2_B64, Opcode::DsWriteB32},
{Decoder::Opcode::DS_WRITE2ST64_B64, Opcode::DsWriteB32},
{Decoder::Opcode::DS_WRITE2_B32, Opcode::DsWrite2B32},
{Decoder::Opcode::DS_WRITE2ST64_B32, Opcode::DsWrite2B32},
{Decoder::Opcode::DS_WRITE2_B64, Opcode::DsWrite2B32},
{Decoder::Opcode::DS_WRITE2ST64_B64, Opcode::DsWrite2B32},
{Decoder::Opcode::DS_WRITE_B32, Opcode::DsWriteB32},
{Decoder::Opcode::DS_WRITE_B64, Opcode::DsWriteB32},
{Decoder::Opcode::DS_WRITE_B96, Opcode::DsWriteB32},
src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc
+2
View File
@@ -331,9 +331,11 @@ IR_OPCODE(DsReadSbyte, General, Memory)
IR_OPCODE(DsReadUshort, General, Memory)
IR_OPCODE(DsReadSshort, General, Memory)
IR_OPCODE(DsReadB32, General, Memory)
IR_OPCODE(DsRead2B32, General, Memory)
IR_OPCODE(DsWriteByte, General, Memory)
IR_OPCODE(DsWriteShort, General, Memory)
IR_OPCODE(DsWriteB32, General, Memory)
IR_OPCODE(DsWrite2B32, General, Memory)
IR_OPCODE(DsMinF32, General, Memory)
IR_OPCODE(DsMaxF32, General, Memory)
IR_OPCODE(DsSwizzleB32, General, Memory)
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp
+49 -36
View File
@@ -42,8 +42,6 @@ constexpr Type BufferResource = Type::BufferResource;
constexpr Type AddressResource = Type::AddressResource;
constexpr Type ImageResource = Type::ImageResource;
constexpr Type SamplerResource = Type::SamplerResource;
constexpr Type LdsResource = Type::LdsResource;
constexpr Type GdsResource = Type::GdsResource;
constexpr Type ImageAddress = Type::ImageAddress;
constexpr std::array<OpcodeMeta, static_cast<size_t>(ValueOpcode::Count)> MetaTable = {{
@@ -76,52 +74,23 @@ bool HasSideEffects(ValueOpcode opcode) {
if (buffer_access == BufferAccess::Write || buffer_access == BufferAccess::Atomic) {
return true;
}
const auto shared_access = SharedAccessOf(opcode);
if (shared_access == SharedAccess::Write || shared_access == SharedAccess::Atomic ||
shared_access == SharedAccess::Append || shared_access == SharedAccess::Consume) {
return true;
}
switch (opcode) {
case ValueOpcode::Reference:
case ValueOpcode::ReferenceU32:
case ValueOpcode::StoreAddressU8:
case ValueOpcode::StoreAddressU16:
case ValueOpcode::StoreAddressU32:
case ValueOpcode::WriteSharedU8:
case ValueOpcode::WriteSharedU16:
case ValueOpcode::WriteSharedU32:
case ValueOpcode::WriteGdsU8:
case ValueOpcode::WriteGdsU16:
case ValueOpcode::WriteGdsU32:
case ValueOpcode::SharedAtomicFMin32:
case ValueOpcode::SharedAtomicFMax32:
case ValueOpcode::GdsAtomicFMin32:
case ValueOpcode::GdsAtomicFMax32:
case ValueOpcode::SharedAtomicSwap32:
case ValueOpcode::SharedAtomicIAdd32:
case ValueOpcode::SharedAtomicISub32:
case ValueOpcode::SharedAtomicSMin32:
case ValueOpcode::SharedAtomicUMin32:
case ValueOpcode::SharedAtomicSMax32:
case ValueOpcode::SharedAtomicUMax32:
case ValueOpcode::SharedAtomicAnd32:
case ValueOpcode::SharedAtomicOr32:
case ValueOpcode::SharedAtomicXor32:
case ValueOpcode::GdsAtomicSwap32:
case ValueOpcode::GdsAtomicIAdd32:
case ValueOpcode::GdsAtomicISub32:
case ValueOpcode::GdsAtomicSMin32:
case ValueOpcode::GdsAtomicUMin32:
case ValueOpcode::GdsAtomicSMax32:
case ValueOpcode::GdsAtomicUMax32:
case ValueOpcode::GdsAtomicAnd32:
case ValueOpcode::GdsAtomicOr32:
case ValueOpcode::GdsAtomicXor32:
case ValueOpcode::ImageAtomicIAdd32:
case ValueOpcode::ImageAtomicUMin32:
case ValueOpcode::ImageAtomicUMax32:
case ValueOpcode::ImageAtomicAnd32:
case ValueOpcode::ImageAtomicOr32:
case ValueOpcode::ImageAtomicXor32:
case ValueOpcode::DataAppend:
case ValueOpcode::DataConsume:
case ValueOpcode::GdsDataAppend:
case ValueOpcode::GdsDataConsume:
case ValueOpcode::ImageWrite:
case ValueOpcode::SetAttribute:
case ValueOpcode::Barrier: return true;
@@ -172,4 +141,48 @@ uint32_t BufferComponentCount(ValueOpcode opcode) {
}
}
SharedAccess SharedAccessOf(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::LoadSharedU8:
case ValueOpcode::LoadSharedU16:
case ValueOpcode::LoadSharedU32:
case ValueOpcode::LoadSharedU32x2:
case ValueOpcode::LoadSharedU32x3:
case ValueOpcode::LoadSharedU32x4: return SharedAccess::Read;
case ValueOpcode::WriteSharedU8:
case ValueOpcode::WriteSharedU16:
case ValueOpcode::WriteSharedU32:
case ValueOpcode::WriteSharedU32x2:
case ValueOpcode::WriteSharedU32x3:
case ValueOpcode::WriteSharedU32x4: return SharedAccess::Write;
case ValueOpcode::SharedAtomicFMin32:
case ValueOpcode::SharedAtomicFMax32:
case ValueOpcode::SharedAtomicSwap32:
case ValueOpcode::SharedAtomicIAdd32:
case ValueOpcode::SharedAtomicISub32:
case ValueOpcode::SharedAtomicSMin32:
case ValueOpcode::SharedAtomicUMin32:
case ValueOpcode::SharedAtomicSMax32:
case ValueOpcode::SharedAtomicUMax32:
case ValueOpcode::SharedAtomicAnd32:
case ValueOpcode::SharedAtomicOr32:
case ValueOpcode::SharedAtomicXor32: return SharedAccess::Atomic;
case ValueOpcode::DataAppend: return SharedAccess::Append;
case ValueOpcode::DataConsume: return SharedAccess::Consume;
default: return SharedAccess::None;
}
}
uint32_t SharedComponentCount(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::LoadSharedU32x2:
case ValueOpcode::WriteSharedU32x2: return 2u;
case ValueOpcode::LoadSharedU32x3:
case ValueOpcode::WriteSharedU32x3: return 3u;
case ValueOpcode::LoadSharedU32x4:
case ValueOpcode::WriteSharedU32x4: return 4u;
default: return SharedAccessOf(opcode) == SharedAccess::None ? 0u : 1u;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h
+3
View File
@@ -17,6 +17,7 @@ enum class ValueOpcode {
};
enum class BufferAccess { None, Read, Write, Atomic };
enum class SharedAccess { None, Read, Write, Atomic, Append, Consume };
struct DppMoveFlags {
uint16_t control = 0;
@@ -56,6 +57,8 @@ static_assert(std::is_trivially_copyable_v<ExportFlags>);
[[nodiscard]] bool HasSideEffects(ValueOpcode opcode);
[[nodiscard]] BufferAccess BufferAccessOf(ValueOpcode opcode);
[[nodiscard]] uint32_t BufferComponentCount(ValueOpcode opcode);
[[nodiscard]] SharedAccess SharedAccessOf(ValueOpcode opcode);
[[nodiscard]] uint32_t SharedComponentCount(ValueOpcode opcode);
[[nodiscard]] std::string_view ValueOpcodeName(ValueOpcode opcode);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc
+26 -42
View File
@@ -202,8 +202,6 @@ VALUE_OPCODE(GetBufferResource, BufferResource, U32, U32, U32, U32)
VALUE_OPCODE(GetAddressResource, AddressResource, U32, U32)
VALUE_OPCODE(GetImageResource, ImageResource, U32, U32, U32, U32, U32, U32, U32, U32)
VALUE_OPCODE(GetSamplerResource, SamplerResource, U32, U32, U32, U32)
VALUE_OPCODE(GetLdsResource, LdsResource)
VALUE_OPCODE(GetGdsResource, GdsResource)
VALUE_OPCODE(MakeImageAddress, ImageAddress, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32,
U32, U32)
@@ -240,46 +238,32 @@ VALUE_OPCODE(BufferAtomicXor32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicFMin32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicFMax32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(LoadSharedU8, U8, LdsResource, U32, U1)
VALUE_OPCODE(LoadSharedU16, U16, LdsResource, U32, U1)
VALUE_OPCODE(LoadSharedU32, U32, LdsResource, U32, U1)
VALUE_OPCODE(WriteSharedU8, Void, LdsResource, U32, U8, U1)
VALUE_OPCODE(WriteSharedU16, Void, LdsResource, U32, U16, U1)
VALUE_OPCODE(WriteSharedU32, Void, LdsResource, U32, U32, U1)
VALUE_OPCODE(LoadGdsU8, U8, GdsResource, U32, U1)
VALUE_OPCODE(LoadGdsU16, U16, GdsResource, U32, U1)
VALUE_OPCODE(LoadGdsU32, U32, GdsResource, U32, U1)
VALUE_OPCODE(WriteGdsU8, Void, GdsResource, U32, U8, U1)
VALUE_OPCODE(WriteGdsU16, Void, GdsResource, U32, U16, U1)
VALUE_OPCODE(WriteGdsU32, Void, GdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicFMin32, Void, LdsResource, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicFMax32, Void, LdsResource, U32, U32, U32, U1)
VALUE_OPCODE(GdsAtomicFMin32, Void, GdsResource, U32, U32, U32, U1)
VALUE_OPCODE(GdsAtomicFMax32, Void, GdsResource, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicSwap32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicIAdd32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicISub32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicSMin32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicUMin32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicSMax32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicUMax32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicAnd32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicOr32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicXor32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicSwap32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicIAdd32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicISub32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicSMin32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicUMin32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicSMax32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicUMax32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicAnd32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicOr32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicXor32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(DataAppend, U32, LdsResource, U32, U1, U32, U32)
VALUE_OPCODE(DataConsume, U32, LdsResource, U32, U1, U32, U32)
VALUE_OPCODE(GdsDataAppend, U32, GdsResource, U32, U1, U32, U32)
VALUE_OPCODE(GdsDataConsume, U32, GdsResource, U32, U1, U32, U32)
VALUE_OPCODE(LoadSharedU8, U8, U32, U1)
VALUE_OPCODE(LoadSharedU16, U16, U32, U1)
VALUE_OPCODE(LoadSharedU32, U32, U32, U1)
VALUE_OPCODE(LoadSharedU32x2, U32x2, U32, U1)
VALUE_OPCODE(LoadSharedU32x3, U32x3, U32, U1)
VALUE_OPCODE(LoadSharedU32x4, U32x4, U32, U1)
VALUE_OPCODE(WriteSharedU8, Void, U32, U8, U1)
VALUE_OPCODE(WriteSharedU16, Void, U32, U16, U1)
VALUE_OPCODE(WriteSharedU32, Void, U32, U32, U1)
VALUE_OPCODE(WriteSharedU32x2, Void, U32, U32, U32, U1)
VALUE_OPCODE(WriteSharedU32x3, Void, U32, U32, U32, U32, U1)
VALUE_OPCODE(WriteSharedU32x4, Void, U32, U32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicISub32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicSMin32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicUMin32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicSMax32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1)
VALUE_OPCODE(DataAppend, U32, U32, U1, U32, U32)
VALUE_OPCODE(DataConsume, U32, U32, U1, U32, U32)
VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp
+21 -9
View File
@@ -115,20 +115,25 @@ void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
{kind, static_cast<uint32_t>(layout.descriptors.size()), std::move(resources)});
}
bool UsesGds(const Program& program) {
bool UsesGds(const Program& program, bool& uses_gds) {
uses_gds = false;
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() == ValueOpcode::GetGdsResource && inst.HasUses()) {
return true;
if (SharedAccessOf(inst.GetOpcode()) == SharedAccess::None) {
continue;
}
for (size_t index = 0; index < inst.NumArgs(); index++) {
if (inst.Arg(index).GetType() == Type::GdsResource) {
return true;
}
const auto index = inst.Flags<MemoryFlags>().index;
if (index >= program.values->memory_info.size()) {
return false;
}
const auto kind = program.values->memory_info[index].kind;
if (kind != ResourceKind::Lds && kind != ResourceKind::Gds) {
return false;
}
uses_gds |= kind == ResourceKind::Gds;
}
}
return false;
return true;
}
} // namespace
@@ -220,7 +225,14 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std
}
AddBinding(next, DescriptorBindingKind::Samplers, std::move(resources));
}
if (UsesGds(program)) {
bool uses_gds = false;
if (!UsesGds(program, uses_gds)) {
if (error != nullptr) {
*error = "typed shader contains invalid shared-memory metadata";
}
return false;
}
if (uses_gds) {
AddBinding(next, DescriptorBindingKind::Gds);
}
if (!program.info.addresses.empty()) {
tests/ResourceTrackingTests.cpp
+40 -37
View File
@@ -256,8 +256,8 @@ void TestInvariantIndirectImageMaterialization() {
fixture->program.values->descriptor_sources[source]
.indirect_image.has_value(),
"indirect image source was not retained for runtime proof");
const auto image_handle = std::ranges::find_if(
*fixture->block, [](const Inst &inst) {
const auto image_handle =
std::ranges::find_if(*fixture->block, [](const Inst &inst) {
return inst.GetOpcode() == ValueOpcode::GetImageResource;
});
Check(image_handle != fixture->block->end() &&
@@ -326,11 +326,12 @@ void TestInvariantIndirectImageMaterialization() {
memory.words[(0x2020u - memory.base) / 4u + 3u] =
image_descriptor[3] ^ (1u << 28u);
ResourceSnapshot null_snapshot;
Check(MaterializeResources(fixture->program, runtime, null_snapshot, &error) &&
null_snapshot.indirect_images.empty() &&
std::ranges::all_of(null_snapshot.images[0].dwords,
[](uint32_t dword) { return dword == 0u; }),
"stale typed null image descriptors were not canonicalized");
Check(
MaterializeResources(fixture->program, runtime, null_snapshot, &error) &&
null_snapshot.indirect_images.empty() &&
std::ranges::all_of(null_snapshot.images[0].dwords,
[](uint32_t dword) { return dword == 0u; }),
"stale typed null image descriptors were not canonicalized");
for (uint32_t dword = 0; dword < image_descriptor.size(); dword++) {
memory.words[(0x2000u - memory.base) / 4u + dword] =
@@ -380,11 +381,12 @@ void TestInvariantIndirectImageMaterialization() {
ResourceSnapshot capacity_snapshot;
for (const uint32_t records : {1u, 3u}) {
user_data[2] = records;
Check(MaterializeResources(fixture->program, runtime, capacity_snapshot,
&error) &&
ValidateResourceSpecialization(fixture->program,
capacity_snapshot, &error),
"runtime indirect key mapping rejected a fitting material-table size");
Check(
MaterializeResources(fixture->program, runtime, capacity_snapshot,
&error) &&
ValidateResourceSpecialization(fixture->program, capacity_snapshot,
&error),
"runtime indirect key mapping rejected a fitting material-table size");
}
user_data[2] = 2u;
memory.words[(0x2020u - memory.base) / 4u] =
@@ -396,11 +398,12 @@ void TestInvariantIndirectImageMaterialization() {
image_descriptor[dword];
}
memory.words[(0x1000u - memory.base + 68u) / 4u] = 2u;
Check(!MaterializeResources(fixture->program, runtime, rebound_snapshot,
&error) &&
error.find("candidate topology") != std::string::npos &&
same_snapshot(rebound_snapshot, collapsed_snapshot),
"larger indirect candidate topology reused or mutated a cached snapshot");
Check(
!MaterializeResources(fixture->program, runtime, rebound_snapshot,
&error) &&
error.find("candidate topology") != std::string::npos &&
same_snapshot(rebound_snapshot, collapsed_snapshot),
"larger indirect candidate topology reused or mutated a cached snapshot");
auto memory_backed = MakeIndirectImageFixture(false, 0u, true);
memory_backed->PlanAndTrack();
@@ -985,14 +988,14 @@ void TestExecMaskedFlatAddressProvenance() {
Fixture fixture;
const auto low_root = fixture.UserData(0);
const auto high_root = fixture.UserData(1);
const auto active = fixture.Emit(
ValueOpcode::INotEqual32, {fixture.UserData(2), Value(0u)});
const auto active =
fixture.Emit(ValueOpcode::INotEqual32, {fixture.UserData(2), Value(0u)});
const auto inactive_low = fixture.Emit(ValueOpcode::UndefU32);
const auto inactive_high = fixture.Emit(ValueOpcode::UndefU32);
const auto low = fixture.Emit(ValueOpcode::SelectU32,
{active, low_root, inactive_low});
const auto high = fixture.Emit(ValueOpcode::SelectU32,
{active, high_root, inactive_high});
const auto low =
fixture.Emit(ValueOpcode::SelectU32, {active, low_root, inactive_low});
const auto high =
fixture.Emit(ValueOpcode::SelectU32, {active, high_root, inactive_high});
const auto address = fixture.Address(low, high, 0xa4);
MemoryInfo flat;
flat.kind = ResourceKind::Flat;
@@ -1015,19 +1018,18 @@ void TestExecMaskedFlatAddressProvenance() {
"exec-masked FLAT address materialized the wrong user-data root");
Fixture mismatch;
const auto mismatch_active = mismatch.Emit(
ValueOpcode::INotEqual32, {mismatch.UserData(2), Value(0u)});
const auto other_active = mismatch.Emit(ValueOpcode::LogicalNot,
{mismatch_active});
const auto mismatch_active = mismatch.Emit(ValueOpcode::INotEqual32,
{mismatch.UserData(2), Value(0u)});
const auto other_active =
mismatch.Emit(ValueOpcode::LogicalNot, {mismatch_active});
const auto mismatch_low = mismatch.Emit(
ValueOpcode::SelectU32,
{mismatch_active, mismatch.UserData(0),
mismatch.Emit(ValueOpcode::UndefU32)});
ValueOpcode::SelectU32, {mismatch_active, mismatch.UserData(0),
mismatch.Emit(ValueOpcode::UndefU32)});
const auto mismatch_high = mismatch.Emit(
ValueOpcode::SelectU32,
{mismatch_active, mismatch.UserData(1),
mismatch.Emit(ValueOpcode::UndefU32)});
const auto mismatch_address = mismatch.Address(mismatch_low, mismatch_high, 0xa4);
ValueOpcode::SelectU32, {mismatch_active, mismatch.UserData(1),
mismatch.Emit(ValueOpcode::UndefU32)});
const auto mismatch_address =
mismatch.Address(mismatch_low, mismatch_high, 0xa4);
mismatch.Emit(ValueOpcode::LoadAddressU8,
{mismatch_address, mismatch_low, mismatch_high, other_active},
mismatch.AddMemory(flat, 0xa4));
@@ -1086,9 +1088,10 @@ void TestShaderInfoAndBindingLayout() {
{Value(static_cast<uint32_t>(StageInputKind::GlobalInvocationId)),
Value(2u)});
fixture.Emit(ValueOpcode::BitwiseXor32, {Value(1u), Value(2u)});
const auto gds = fixture.Emit(ValueOpcode::GetGdsResource);
fixture.Emit(ValueOpcode::WriteGdsU32,
{gds, Value(0u), Value(1u), Value(true)});
MemoryInfo gds;
gds.kind = ResourceKind::Gds;
fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), Value(1u), Value(true)},
fixture.AddMemory(gds, 8));
fixture.PlanAndTrack();
ShaderComputeInputInfo compute{};
tests/ShaderRecompilerComputeTests.cpp
+694 -487
View File
File diff suppressed because it is too large Load Diff
tests/shaderCfgTests.cpp
+254 -77
View File
@@ -976,7 +976,7 @@ void TestSpirvRequirementsAnalysis() {
block->AppendNewInst(ValueOpcode::ImageGatherRaw,
{Value(0u), Value(0u), Value(0u)});
auto &shared = block->AppendNewInst(ValueOpcode::LoadSharedU32,
{Value(0u), Value(0u), Value(true)});
{Value(0u), Value(true)});
shared.SetFlags(MemoryFlags{.index = 0});
auto &export_value =
block->AppendNewInst(ValueOpcode::SetAttribute, {Value(0u), Value(true)});
@@ -1095,11 +1095,9 @@ void TestNativeSubgroupPolicy() {
} else if (opcode == ValueOpcode::Ballot) {
block->AppendNewInst(opcode, {Value(true)});
} else if (opcode == ValueOpcode::DataAppend) {
auto &resource = block->AppendNewInst(ValueOpcode::GetLdsResource);
program.values->memory_info.push_back({.kind = ResourceKind::Lds});
auto &append =
block->AppendNewInst(opcode, {Value(&resource), Value(0u),
Value(true), Value(1u), Value(0u)});
auto &append = block->AppendNewInst(
opcode, {Value(0u), Value(true), Value(1u), Value(0u)});
append.SetFlags(MemoryFlags{.index = 0});
}
std::string error;
@@ -1748,6 +1746,7 @@ void TestNewShaderRecompilerScalarVectorAlu() {
Check(SpirvContainsOpcode(result.spirv, 199),
"SPIR-V binary does not contain OpBitwiseAnd");
CheckSpirvBinaryValidates(result.spirv);
}
void TestNewShaderRecompilerVop3LaneReadDestinationEncoding() {
@@ -4102,8 +4101,6 @@ void TestNewShaderRecompilerMemoryFamilyLowering() {
"MIMG sample did not lower to IR");
Check(SpirvContainsOpcode(result.spirv, 65),
"SPIR-V binary does not contain OpAccessChain");
Check(SpirvContainsOpcode(result.spirv, 61),
"SPIR-V binary does not contain OpLoad");
Check(SpirvContainsOpcode(result.spirv, 62),
"SPIR-V binary does not contain OpStore");
Check(SpirvContainsOpcode(result.spirv, 95),
@@ -5329,8 +5326,6 @@ void TestNewShaderRecompilerVintrpLowering() {
Check(ProgramInputCount(result.program,
ShaderRecompiler::IR::StageInputKind::Parameter) == 2,
"VINTRP pixel shader did not reflect parameter inputs");
Check(SpirvContainsOpcode(result.spirv, 61),
"SPIR-V binary does not contain OpLoad");
Check(SpirvContainsOpcode(result.spirv, 62),
"SPIR-V binary does not contain OpStore");
Check(SpirvContainsOpcode(result.spirv, 81),
@@ -5608,8 +5603,6 @@ void TestNewShaderRecompilerWideMemoryLowering() {
"global segment load did not lower with global metadata");
Check(SpirvContainsOpcode(result.spirv, 65),
"SPIR-V binary does not contain OpAccessChain");
Check(SpirvContainsOpcode(result.spirv, 61),
"SPIR-V binary does not contain OpLoad");
Check(SpirvContainsOpcode(result.spirv, 62),
"SPIR-V binary does not contain OpStore");
Check(SpirvContainsOpcode(result.spirv, 194),
@@ -5807,8 +5800,6 @@ void TestNewShaderRecompilerMubufFormatLowering() {
"MUBUF formatted stores were not preserved as native-width operations");
Check(SpirvContainsOpcode(result.spirv, 65),
"SPIR-V binary does not contain OpAccessChain");
Check(SpirvContainsOpcode(result.spirv, 61),
"SPIR-V binary does not contain OpLoad");
Check(SpirvContainsOpcode(result.spirv, 62),
"SPIR-V binary does not contain OpStore");
CheckSpirvBinaryValidates(result.spirv);
@@ -6283,18 +6274,60 @@ void TestNewShaderRecompilerDsReadWrite2Lowering() {
0xbf810000u,
};
ShaderRecompiler::Decoder::Program decoded;
ShaderRecompiler::CFG::Graph graph;
ShaderRecompiler::IR::Program lowered;
std::string error;
Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded,
&error) &&
ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) &&
ShaderRecompiler::IR::LowerProgram(
decoded, graph, ShaderType::Compute, 64, lowered, &error),
error.c_str());
uint32_t pair_reads = 0;
uint32_t pair_writes = 0;
for (const auto &block : lowered.blocks) {
for (const auto &inst : block.instructions) {
pair_reads += inst.op == ShaderRecompiler::IR::Opcode::DsRead2B32;
pair_writes += inst.op == ShaderRecompiler::IR::Opcode::DsWrite2B32;
}
}
Check(pair_reads == 3u && pair_writes == 1u,
"DS read/write2 was split before typed SSA");
ShaderRecompiler::IR::ValueProgram typed;
ShaderComputeInputInfo compute{};
Check(ShaderRecompiler::Frontend::TranslateProgram(lowered, typed, nullptr,
nullptr, &compute, &error),
error.c_str());
uint32_t scalar_loads = 0;
uint32_t vector_loads = 0;
uint32_t scalar_stores = 0;
for (const auto *block : typed.blocks) {
for (const auto &inst : *block) {
scalar_loads +=
inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::LoadSharedU32;
vector_loads += inst.GetOpcode() ==
ShaderRecompiler::IR::ValueOpcode::LoadSharedU32x2;
scalar_stores +=
inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::WriteSharedU32;
}
}
Check(scalar_loads == 4u && vector_loads == 2u && scalar_stores == 2u,
"DS read/write2 did not use the native scalar/x2 transaction shape");
auto options = MakeCompileOptions(ShaderType::Compute);
options.dump_ir = true;
ShaderRecompiler::CompileResult result;
std::string error;
Check(ShaderRecompiler::TryRecompile(shader, options, result, &error),
error.c_str());
Check(Common::ContainsStr(result.decoded_dump, "ds_write2_b32"),
Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE2_B32"),
"new decoder did not decode old-backed DS write2");
Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b32"),
Check(Common::ContainsStr(result.decoded_dump, "DS_READ2_B32"),
"new decoder did not decode old-backed DS read2");
Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b64"),
Check(Common::ContainsStr(result.decoded_dump, "DS_READ2ST64_B32"),
"new decoder did not preserve the DS read2st64 opcode identity");
Check(Common::ContainsStr(result.decoded_dump, "DS_READ2_B64"),
"new decoder did not decode old-backed DS read2 b64");
Check(Common::ContainsStr(result.decoded_dump, "offset=4"),
"DS write2 decode did not scale offset0 to bytes");
@@ -6316,24 +6349,6 @@ void TestNewShaderRecompilerDsReadWrite2Lowering() {
"DS read/write2 decode did not preserve two-dword metadata");
Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"),
"DS read2 b64 decode did not preserve four-dword metadata");
Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v60"),
"DS write2 did not lower first dword through shared LDS store IR");
Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v61"),
"DS write2 did not lower second dword through shared LDS store IR");
Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v70"),
"DS read2 did not lower first dword through shared LDS load IR");
Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v71"),
"DS read2 did not lower second dword through shared LDS load IR");
Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v72"),
"DS read2 st64 did not lower first dword through shared LDS load IR");
Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v73"),
"DS read2 st64 did not lower second dword through shared LDS load IR");
Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v80"),
"DS read2 b64 did not lower first dword through shared LDS load IR");
Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v83"),
"DS read2 b64 did not lower fourth dword through shared LDS load IR");
Check(SpirvContainsOpcode(result.spirv, 61),
"SPIR-V binary does not contain OpLoad");
Check(SpirvContainsOpcode(result.spirv, 62),
"SPIR-V binary does not contain OpStore");
Check(SpirvContainsOpcode(result.spirv, 65),
@@ -6420,49 +6435,80 @@ void TestNewShaderRecompilerDsWideAndAtomicLowering() {
0xbf810000u,
};
ShaderRecompiler::Decoder::Program decoded;
ShaderRecompiler::CFG::Graph graph;
ShaderRecompiler::IR::Program lowered;
std::string error;
Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded,
&error) &&
ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) &&
ShaderRecompiler::IR::LowerProgram(
decoded, graph, ShaderType::Compute, 64, lowered, &error),
error.c_str());
uint32_t native_reads = 0;
uint32_t native_writes = 0;
for (const auto &block : lowered.blocks) {
for (const auto &inst : block.instructions) {
native_reads += inst.op == ShaderRecompiler::IR::Opcode::DsReadB32;
native_writes += inst.op == ShaderRecompiler::IR::Opcode::DsWriteB32;
}
}
Check(native_reads == 3u && native_writes == 3u,
"wide DS transfers were split before typed SSA");
ShaderRecompiler::IR::ValueProgram typed;
ShaderComputeInputInfo compute{};
Check(ShaderRecompiler::Frontend::TranslateProgram(lowered, typed, nullptr,
nullptr, &compute, &error),
error.c_str());
std::array<uint32_t, 3> load_widths{};
std::array<uint32_t, 3> store_widths{};
for (const auto *block : typed.blocks) {
for (const auto &inst : *block) {
using ShaderRecompiler::IR::ValueOpcode;
if (inst.GetOpcode() == ValueOpcode::LoadSharedU32x2)
load_widths[0]++;
if (inst.GetOpcode() == ValueOpcode::LoadSharedU32x3)
load_widths[1]++;
if (inst.GetOpcode() == ValueOpcode::LoadSharedU32x4)
load_widths[2]++;
if (inst.GetOpcode() == ValueOpcode::WriteSharedU32x2)
store_widths[0]++;
if (inst.GetOpcode() == ValueOpcode::WriteSharedU32x3)
store_widths[1]++;
if (inst.GetOpcode() == ValueOpcode::WriteSharedU32x4)
store_widths[2]++;
}
}
Check(load_widths == std::array<uint32_t, 3>{1u, 1u, 1u} &&
store_widths == std::array<uint32_t, 3>{1u, 1u, 1u},
"wide DS transfers retained scalar sibling operations");
auto options = MakeCompileOptions(ShaderType::Compute);
options.dump_ir = true;
ShaderRecompiler::CompileResult result;
std::string error;
Check(ShaderRecompiler::TryRecompile(shader, options, result, &error),
error.c_str());
Check(Common::ContainsStr(result.decoded_dump, "ds_write_b64"),
Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B64"),
"new decoder did not decode DS b64 write");
Check(Common::ContainsStr(result.decoded_dump, "ds_write_b96"),
Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B96"),
"new decoder did not decode DS b96 write");
Check(Common::ContainsStr(result.decoded_dump, "ds_write_b128"),
Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B128"),
"new decoder did not decode DS b128 write");
Check(Common::ContainsStr(result.decoded_dump, "ds_read_b64"),
Check(Common::ContainsStr(result.decoded_dump, "DS_READ_B64"),
"new decoder did not decode DS b64 read");
Check(Common::ContainsStr(result.decoded_dump, "ds_read_b96"),
Check(Common::ContainsStr(result.decoded_dump, "DS_READ_B96"),
"new decoder did not decode DS b96 read");
Check(Common::ContainsStr(result.decoded_dump, "ds_read_b128"),
Check(Common::ContainsStr(result.decoded_dump, "DS_READ_B128"),
"new decoder did not decode DS b128 read");
Check(Common::ContainsStr(result.decoded_dump, "ds_min_u32"),
Check(Common::ContainsStr(result.decoded_dump, "DS_MIN_U32"),
"new decoder did not decode DS min atomic");
Check(Common::ContainsStr(result.decoded_dump, "ds_max_u32"),
Check(Common::ContainsStr(result.decoded_dump, "DS_MAX_U32"),
"new decoder did not decode DS max atomic");
Check(Common::ContainsStr(result.decoded_dump, "ds_or_b32"),
Check(Common::ContainsStr(result.decoded_dump, "DS_OR_B32"),
"new decoder did not decode DS or atomic");
Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"),
"new decoder did not expose DS wide width metadata");
Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v10"),
"DS b64 write did not expand to first dword store");
Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v19"),
"DS b128 write did not expand to last dword store");
Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v20"),
"DS b64 read did not expand to first dword load");
Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v31"),
"DS b128 read did not expand to last dword load");
Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 null, v32"),
"DS min atomic did not lower to shared atomic IR");
Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 null, v33"),
"DS max atomic did not lower to shared atomic IR");
Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 null, v34"),
"DS or atomic did not lower to shared atomic IR");
Check(SpirvContainsOpcode(result.spirv, 61),
"SPIR-V binary does not contain OpLoad");
Check(SpirvContainsOpcode(result.spirv, 62),
"SPIR-V binary does not contain OpStore");
Check(SpirvContainsOpcode(result.spirv, 237),
@@ -6570,6 +6616,7 @@ void TestNewShaderRecompilerDsAddtidLowering() {
Check(SpirvContainsOpcode(result.spirv, 199),
"SPIR-V binary does not contain OpBitwiseAnd");
CheckSpirvBinaryValidates(result.spirv);
}
void TestNewShaderRecompilerDsFloatMinMaxLowering() {
@@ -7838,17 +7885,16 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() {
auto program = result.program;
IR::IREmitter definition(program.values->blocks[0]);
const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU32x3,
{IR::Value(1u), IR::Value(2u),
IR::Value(3u)});
const auto vector =
definition.Emit(IR::ValueOpcode::CompositeConstructU32x3,
{IR::Value(1u), IR::Value(2u), IR::Value(3u)});
IR::IREmitter use(program.values->blocks[1]);
use.Emit(IR::ValueOpcode::CompositeExtractU32x3,
{vector, IR::Value(2u)});
use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)});
Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str());
std::vector<uint32_t> spirv;
Check(Spirv::EmitProgram(program, result.resources, options.input_info,
spirv, &error),
Check(Spirv::EmitProgram(program, result.resources, options.input_info, spirv,
&error),
error.c_str());
CheckSpirvBinaryValidates(spirv);
const auto before = MeasureSpirv(result.spirv);
@@ -8533,7 +8579,8 @@ void TestWqmConstantPropagation() {
"WqmU64 constant propagation crossed a quad or dword boundary");
}
const auto check_invalid_signature = [](ValueOpcode opcode, Value valid, Value invalid) {
const auto check_invalid_signature = [](ValueOpcode opcode, Value valid,
Value invalid) {
ValueProgram malformed;
malformed.block_storage.push_back(std::make_unique<Block>());
malformed.blocks.push_back(malformed.block_storage.back().get());
@@ -8547,7 +8594,8 @@ void TestWqmConstantPropagation() {
"value IR accepted a WQM operand that violates opcode metadata");
};
check_invalid_signature(ValueOpcode::WqmU64, Value(uint64_t{1}), Value(true));
check_invalid_signature(ValueOpcode::WqmMask, Value(true), Value(uint64_t{1}));
check_invalid_signature(ValueOpcode::WqmMask, Value(true),
Value(uint64_t{1}));
}
void TestNativeWideValueValidation() {
@@ -8568,8 +8616,8 @@ void TestNativeWideValueValidation() {
IREmitter ir(program.blocks.front());
const auto resource = ir.Emit(ValueOpcode::GetBufferResource,
{Value(0u), Value(0u), Value(0u), Value(0u)});
ir.Emit(opcode,
{resource, Value(0u), Value(0u), Value(0u), Value(true)}, flags);
ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(0u), Value(true)},
flags);
};
const auto check_rejected = [](const ValueProgram &program,
const char *expected) {
@@ -8585,8 +8633,7 @@ void TestNativeWideValueValidation() {
memory.kind = ResourceKind::Buffer;
memory.data_dwords = 3;
program.memory_info.push_back(memory);
append_load(program, ValueOpcode::LoadBufferU32x3,
MemoryFlags{.index = 1});
append_load(program, ValueOpcode::LoadBufferU32x3, MemoryFlags{.index = 1});
check_rejected(program, "invalid memory-info index");
}
{
@@ -8624,6 +8671,43 @@ void TestNativeWideValueValidation() {
ir.Emit(ValueOpcode::CompositeExtractU32x3, {vector, Value(3u)});
check_rejected(program, "invalid component index");
}
const auto append_shared = [](ValueProgram &program, ValueOpcode opcode,
MemoryFlags flags) {
IREmitter ir(program.blocks.front());
ir.Emit(opcode, {Value(0u), Value(true)}, flags);
};
{
auto program = make_program();
append_shared(program, ValueOpcode::LoadSharedU32, MemoryFlags{.index = 1});
check_rejected(program, "invalid memory-info index");
}
{
auto program = make_program();
MemoryInfo memory;
memory.kind = ResourceKind::Buffer;
program.memory_info.push_back(memory);
append_shared(program, ValueOpcode::LoadSharedU32, {});
check_rejected(program, "invalid shared-memory metadata");
}
{
auto program = make_program();
MemoryInfo memory;
memory.kind = ResourceKind::Gds;
memory.resource = 1;
program.memory_info.push_back(memory);
append_shared(program, ValueOpcode::LoadSharedU32, {});
check_rejected(program, "invalid shared-memory metadata");
}
{
auto program = make_program();
MemoryInfo memory;
memory.kind = ResourceKind::Lds;
memory.data_dwords = 2;
memory.component_count = 2;
program.memory_info.push_back(memory);
append_shared(program, ValueOpcode::LoadSharedU32x3, {});
check_rejected(program, "inconsistent shared-memory width");
}
}
void TestNewShaderRecompilerZeroInitialRegisterState() {
@@ -8800,7 +8884,7 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
const uint32_t native_wqm_shader[] = {
EncodeSop1(0x0a, 2, 0), // s_wqm_b64 s[2:3], s[0:1]
EncodeVop1(0x01, 0, 2), // v_mov_b32 v0, s2
EncodeExp0(0x0c, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01),
EncodeExp0(0x0c, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01),
};
auto native_options = options;
native_options.lane_mask_mode = ShaderLaneMaskMode::NativeWave;
@@ -8814,7 +8898,8 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
"native scalar WQM retained frontend expansion or subgroup machinery");
Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 2u &&
SpirvInstructionOpcodeCount(result.spirv, 194u) == 4u,
"native scalar WQM emitter was folded away or did not use the compact path");
"native scalar WQM emitter was folded away or did not use the compact "
"path");
const uint32_t native_special_wqm_shader[] = {
EncodeSop1(0x04, 126, 0), // s_mov_b64 exec, s[0:1]
@@ -8823,7 +8908,7 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
EncodeSop1(0x0a, 106, 106), // s_wqm_b64 vcc, vcc
EncodeVop1(0x01, 0, 126), // v_mov_b32 v0, exec_lo
EncodeVop1(0x01, 1, 106), // v_mov_b32 v1, vcc_lo
EncodeExp0(0x0c, 0x3), EncodeExp1(0, 1, 0, 0), EncodeSopp(0x01),
EncodeExp0(0x0c, 0x3), EncodeExp1(0, 1, 0, 0), EncodeSopp(0x01),
};
Check(ShaderRecompiler::TryRecompile(native_special_wqm_shader,
native_options, result, &error),
@@ -10406,6 +10491,19 @@ void TestNewShaderRecompilerSpirvSizeBaselines() {
empty_metrics.type_pointers == 0u,
"empty shader emitted unused core types or pointers");
const uint32_t dead_gds[] = {
EncodeDs0(0x36, 0) | (1u << 17u),
EncodeDs1(0, 0, 1),
EncodeSopp(0x01),
};
const auto dead_gds_result =
compile("dead-gds", dead_gds,
{.words = 64, .instructions = 20, .labels = 4, .branches = 3});
Check(ShaderRecompiler::IR::FindBinding(
dead_gds_result.program.bindings,
ShaderRecompiler::IR::DescriptorBindingKind::Gds) == nullptr,
"dead GDS load retained a descriptor binding");
const uint32_t structured_phi[] = {
EncodeSMovB32(0, 128), // s0 = 0
EncodeSopc(0x0a, 0, 129), // loop: s_cmp_lt_u32 s0, 1
@@ -10494,6 +10592,83 @@ void TestNewShaderRecompilerSpirvSizeBaselines() {
scalar_stores == 0u,
"wide buffer fixture retained scalar sibling memory operations");
const uint32_t wide_lds[] = {
EncodeDs0(0xff, 0), EncodeDs1(0, 0, 4), // ds_read_b128 v[0:3], v4
EncodeDs0(0xdf, 16), EncodeDs1(0, 0, 4), // ds_write_b128 v[0:3], v4
EncodeSopp(0x01),
};
const auto wide_lds_result = compile("wide-lds", wide_lds,
{.words = 548,
.instructions = 156,
.variables = 1,
.workgroup_variables = 1,
.loads = 4,
.stores = 4,
.phis = 5,
.labels = 34,
.selection_merges = 10,
.branches = 23,
.conditional_branches = 10});
const auto wide_lds_metrics = MeasureSpirv(wide_lds_result.spirv);
Check(wide_lds_metrics.workgroup_variables == 1u &&
wide_lds_metrics.runtime_arrays == 0u &&
wide_lds_metrics.array_lengths == 0u,
"native wide LDS fixture emitted the wrong storage topology");
const uint32_t wide_gds[] = {
EncodeDs0(0xff, 0) | (1u << 17u),
EncodeDs1(0, 0, 4),
EncodeDs0(0xdf, 16) | (1u << 17u),
EncodeDs1(0, 0, 4),
EncodeSopp(0x01),
};
const auto wide_gds_result = compile("wide-gds", wide_gds,
{.words = 577,
.instructions = 162,
.runtime_arrays = 1,
.variables = 1,
.loads = 4,
.stores = 4,
.array_lengths = 1,
.phis = 5,
.labels = 34,
.selection_merges = 10,
.branches = 23,
.conditional_branches = 10});
const auto wide_gds_metrics = MeasureSpirv(wide_gds_result.spirv);
Check(wide_gds_metrics.runtime_arrays == 1u &&
wide_gds_metrics.workgroup_variables == 0u &&
wide_gds_metrics.array_lengths == 1u,
"native wide GDS fixture did not share one entry-dominating length");
Check(ShaderRecompiler::IR::FindBinding(
wide_gds_result.program.bindings,
ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr,
"live native wide GDS access did not allocate its descriptor binding");
const auto count_shared = [](const ShaderRecompiler::CompileResult &result,
ShaderRecompiler::IR::ValueOpcode opcode) {
uint32_t count = 0;
for (const auto *block : result.program.values->blocks) {
count += std::ranges::count(*block, opcode,
&ShaderRecompiler::IR::Inst::GetOpcode);
}
return count;
};
for (const auto *result : {&wide_lds_result, &wide_gds_result}) {
Check(count_shared(*result,
ShaderRecompiler::IR::ValueOpcode::LoadSharedU32x4) ==
1u &&
count_shared(
*result,
ShaderRecompiler::IR::ValueOpcode::WriteSharedU32x4) == 1u &&
count_shared(*result,
ShaderRecompiler::IR::ValueOpcode::LoadSharedU32) ==
0u &&
count_shared(*result,
ShaderRecompiler::IR::ValueOpcode::WriteSharedU32) ==
0u,
"native wide shared fixture retained scalar sibling operations");
}
const uint32_t wqm[] = {
EncodeVopc(0xc1, 5 + 256, 8), // v_cmp_lt_u32 vcc, v5, v8
EncodeSop1(0x0a, 2, 106), // s_wqm_b64 s[2:3], vcc
@@ -10563,6 +10738,8 @@ int main() {
TestNewShaderRecompilerNativeWideBufferIr();
TestNewShaderRecompilerMubufFormatLowering();
TestNewShaderRecompilerTypedBufferLowering();
TestNewShaderRecompilerDsReadWrite2Lowering();
TestNewShaderRecompilerDsWideAndAtomicLowering();
TestNewShaderRecompilerCapturedVop1SdwaByteConvert();
TestNewShaderRecompilerScalarMemoryBindingDomains();
// Opcode semantics and optimized direct SPIR-V are exercised by