raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-27 05:16:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor(shader): preserve native wide buffer operations

Browse Source
This commit is contained in:
nmzik
2026-08-19 07:51:11 +02:00
parent 3ea2099245
commit 5383972e96
25 changed files with 1100 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/ShaderRecompiler.cpp
+11 -9
View File
@@ -632,7 +632,9 @@ uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputIn
continue;
}
const auto* load = FindEmbeddedFetchLoad(loads, inst.pc);
if (load == nullptr || inst.memory.component_index >= load->components) {
if (load == nullptr || inst.memory.data_dwords != load->components ||
inst.dst.kind != IR::OperandKind::Register ||
inst.dst.reg.file != IR::RegisterFile::Vector) {
continue;
}
@@ -644,16 +646,16 @@ uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputIn
continue;
}
inst.op = IR::Opcode::LoadInputF32;
inst.input_info.attr = static_cast<uint32_t>(resource_id);
inst.input_info.chan = inst.memory.component_index;
inst.memory = {};
inst.src_count = 0;
inst.op = IR::Opcode::LoadInputF32;
inst.input_info.attr = static_cast<uint32_t>(resource_id);
inst.input_info.chan = 0;
inst.input_info.component_count = load->components;
inst.memory = {};
inst.src_count = 0;
mutable_input_info->resource_fetch_components[resource_id] =
std::max(mutable_input_info->resource_fetch_components[resource_id],
static_cast<int>(inst.input_info.chan + 1u));
rewritten++;
static_cast<int>(load->components));
rewritten += load->components;
}
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp
+5
View File
@@ -323,6 +323,10 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true;
case IR::ValueOpcode::CompositeConstructU32x3:
ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x3,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true;
case IR::ValueOpcode::CompositeConstructF32x2:
ctx.Emit(inst, OpCompositeConstruct, IR::Type::F32x2,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
@@ -332,6 +336,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
return true;
case IR::ValueOpcode::CompositeExtractU32x2:
case IR::ValueOpcode::CompositeExtractU32x3:
case IR::ValueOpcode::CompositeExtractU32x4:
ctx.Emit(inst, OpCompositeExtract, IR::Type::U32,
{ctx.Arg(inst, 0), inst.Arg(1).U32()});
src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp
+1 -1
View File
@@ -392,7 +392,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define(
inst,
EmitValueOrDefaultIfCondition(
state, condition, TypeU32Vector(state, 4), ConstantU32x4Zero(state), [&]() {
state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4), [&]() {
const auto image = LoadSampledImageDescriptor(state, mem, pc, view);
const auto color = state.builder.AllocateId();
const auto coord = CoordU32(ctx, mem, *address, view);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+3 -1
View File
@@ -342,6 +342,8 @@ uint32_t TypeI32(EmitterState& state);
uint32_t TypeI32Pair(EmitterState& state);
uint32_t TypeF32(EmitterState& state);
uint32_t TypeU32Vector(EmitterState& state, uint32_t components);
uint32_t TypeU32Composite(EmitterState& state, uint32_t components);
uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t pointee);
@@ -615,7 +617,7 @@ uint32_t ConstantBool(EmitterState& state, bool value);
uint32_t ConstantU64(EmitterState& state, uint64_t value);
uint32_t ConstantU32x4Zero(EmitterState& state);
uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components);
uint32_t GlslStd450(EmitterState& state);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+415 -117
View File
@@ -163,47 +163,78 @@ PreparedMemoryElement PrepareMemoryElement(ValueEmitContext& ctx, const IR::Memo
return {.resource = resource, .index = index};
}
PreparedMemoryElement PrepareMemoryElement(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource,
uint32_t raw_index) {
return {.resource = resource, .index = EmitMemoryElementIndex(ctx.state, resource, raw_index)};
}
uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
uint32_t index);
uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
uint32_t address, uint32_t index, uint32_t bits, bool sign_extend);
uint32_t LoadWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource) {
const auto access = PrepareMemoryElement(ctx, mem, resource, DwordIndex(ctx, inst, mem));
return EmitValueOrZeroIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index),
[&]() { return LoadWordInBounds(ctx, access.resource, access.index); });
}
uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) {
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
return EmitValueOrZeroIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
const auto value = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, TypeU32(ctx.state), value,
EmitMemoryElementPointer(ctx.state, access.resource, access.index)});
return value;
});
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
return LoadWordPrepared(ctx, inst, mem, resource);
});
}
uint32_t LoadSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t bits,
bool sign_extend) {
const auto address = ByteAddress(ctx, inst, mem);
const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 2));
const auto access = PrepareMemoryElement(ctx, mem, resource, raw_index);
return EmitValueOrZeroIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
return LoadSubwordInBounds(ctx, access.resource, address, access.index, bits,
sign_extend);
});
}
uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
uint32_t index) {
const auto value = ctx.state.builder.AllocateId();
const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index);
ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), value, pointer});
return value;
}
uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
uint32_t address, uint32_t index, uint32_t bits, bool sign_extend) {
const auto word = LoadWordInBounds(ctx, resource, index);
const auto byte =
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address, ConstantU32(ctx.state, 3));
const auto shift =
Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), byte, ConstantU32(ctx.state, 3));
const auto value =
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state),
Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), word, shift),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
if (!sign_extend) return value;
const auto left = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), value,
ConstantU32(ctx.state, 32u - bits));
return Binary(ctx.state, OpShiftRightArithmetic, TypeU32(ctx.state), left,
ConstantU32(ctx.state, 32u - bits));
}
uint32_t LoadSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits,
bool sign_extend) {
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto address = ByteAddress(ctx, inst, mem);
const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 2));
const auto access = PrepareMemoryElement(ctx, mem, raw_index);
return EmitValueOrZeroIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
const auto word = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, TypeU32(ctx.state), word,
EmitMemoryElementPointer(ctx.state, access.resource, access.index)});
const auto byte = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 3));
const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), byte,
ConstantU32(ctx.state, 3));
const auto value =
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state),
Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), word, shift),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
if (!sign_extend) return value;
const auto left = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), value,
ConstantU32(ctx.state, 32u - bits));
return Binary(ctx.state, OpShiftRightArithmetic, TypeU32(ctx.state), left,
ConstantU32(ctx.state, 32u - bits));
});
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
return LoadSubwordPrepared(ctx, inst, mem, resource, bits, sign_extend);
});
}
@@ -215,60 +246,117 @@ Prospero::BufferFormat BufferFormat(const ValueEmitContext& ctx, const IR::Inst&
IR::MemoryInfo RebaseFormattedComponent(IR::MemoryInfo mem, Prospero::BufferFormat format,
uint32_t component) {
if (!mem.typed) {
const auto component_slot = mem.component_index * 4u;
mem.offset = mem.offset >= component_slot ? mem.offset - component_slot : 0u;
mem.offset += Format::GetFormatComponentByteOffset(format, component);
}
mem.offset += Format::GetFormatComponentByteOffset(format, component);
mem.data_dwords = 1u;
mem.component_index = component;
return mem;
}
uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
const auto format = BufferFormat(ctx, inst, mem);
if (!Format::IsKnownFormat(format)) return LoadWord(ctx, inst, mem);
const auto info = Format::GetFormatInfo(format);
auto component = mem.component_index;
if (!mem.typed) {
const auto selector =
GetDstSel(ctx.state.program.info.buffers[mem.resource].descriptor_swizzle, component);
if (selector == 0u) return ConstantU32(ctx.state, 0);
if (selector == 1u) {
const auto integer = info.type == Format::ComponentType::Uint ||
info.type == Format::ComponentType::Sint;
return ConstantU32(ctx.state, integer ? 1u : 0x3f800000u);
}
if (selector < 4u) {
ExitDescriptorBindingFailure(ctx.state, IR::DescriptorBindingKind::Buffers,
mem.resource, "buffer descriptor has reserved dst_sel");
}
component = selector - 4u;
IR::MemoryInfo RebaseRawComponent(IR::MemoryInfo mem, uint32_t component) {
mem.offset += component * 4u;
mem.data_dwords = 1u;
mem.component_index = component;
return mem;
}
enum class FormattedSourceKind { Memory, Zero, One };
struct FormattedSource {
FormattedSourceKind kind = FormattedSourceKind::Zero;
uint32_t component = 0;
};
FormattedSource ResolveFormattedSource(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
const Format::BufferFormatInfo& info,
uint32_t output_component) {
if (mem.typed) {
return output_component < info.component_count
? FormattedSource {FormattedSourceKind::Memory, output_component}
: FormattedSource {};
}
if (component >= info.component_count) return ConstantU32(ctx.state, 0);
uint32_t raw = 0;
const auto bits = info.component_bits[component];
const auto selector = GetDstSel(ctx.state.program.info.buffers[mem.resource].descriptor_swizzle,
output_component);
if (selector == 0u) return {};
if (selector == 1u) return {FormattedSourceKind::One, 0};
if (selector < 4u) {
ExitDescriptorBindingFailure(ctx.state, IR::DescriptorBindingKind::Buffers, mem.resource,
"buffer descriptor has reserved dst_sel");
}
const auto component = selector - 4u;
return component < info.component_count
? FormattedSource {FormattedSourceKind::Memory, component}
: FormattedSource {};
}
uint32_t FormattedConstant(ValueEmitContext& ctx, const Format::BufferFormatInfo& info,
FormattedSourceKind kind) {
if (kind != FormattedSourceKind::One) return ConstantU32(ctx.state, 0);
const auto integer =
info.type == Format::ComponentType::Uint || info.type == Format::ComponentType::Sint;
return ConstantU32(ctx.state, integer ? 1u : 0x3f800000u);
}
template <typename LoadWordFn, typename LoadSubwordFn>
uint32_t LoadFormattedComponent(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
Prospero::BufferFormat format, const Format::BufferFormatInfo& info,
uint32_t output_component, LoadWordFn&& load_word,
LoadSubwordFn&& load_subword) {
const auto source = ResolveFormattedSource(ctx, mem, info, output_component);
if (source.kind != FormattedSourceKind::Memory) {
return FormattedConstant(ctx, info, source.kind);
}
const auto component = source.component;
const auto bits = info.component_bits[component];
uint32_t raw = 0;
if (info.packed_bitfield) {
raw = LoadWord(ctx, inst, RebaseFormattedComponent(mem, format, component));
raw = load_word(component);
const auto type =
IsSignedFormatComponent(info.type) ? TypeI32(ctx.state) : TypeU32(ctx.state);
const auto source =
const auto source_value =
type == TypeI32(ctx.state) ? Unary(ctx.state, OpBitcast, type, raw) : raw;
const auto extracted = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{IsSignedFormatComponent(info.type) ? OpBitFieldSExtract : OpBitFieldUExtract, type,
extracted, source, ConstantU32(ctx.state, info.component_bit_offset[component]),
extracted, source_value, ConstantU32(ctx.state, info.component_bit_offset[component]),
ConstantU32(ctx.state, bits)});
raw = type == TypeI32(ctx.state)
? Unary(ctx.state, OpBitcast, TypeU32(ctx.state), extracted)
: extracted;
} else if (bits == 32u) {
raw = LoadWord(ctx, inst, RebaseFormattedComponent(mem, format, component));
raw = load_word(component);
} else {
raw = LoadSubword(ctx, inst, RebaseFormattedComponent(mem, format, component), bits,
IsSignedFormatComponent(info.type));
raw = load_subword(component, bits, IsSignedFormatComponent(info.type));
}
return NormalizeFormatComponent(ctx.state, info, component, raw);
}
uint32_t FormattedLoadPrepared(ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem, uint32_t output_component,
const MemoryResourceAccess& resource) {
const auto format = BufferFormat(ctx, inst, mem);
if (!Format::IsKnownFormat(format)) {
return LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, output_component), resource);
}
const auto info = Format::GetFormatInfo(format);
return LoadFormattedComponent(
ctx, mem, format, info, output_component,
[&](uint32_t component) {
return LoadWordPrepared(ctx, inst, RebaseFormattedComponent(mem, format, component),
resource);
},
[&](uint32_t component, uint32_t bits, bool sign_extend) {
return LoadSubwordPrepared(ctx, inst, RebaseFormattedComponent(mem, format, component),
resource, bits, sign_extend);
});
}
uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
return FormattedLoadPrepared(ctx, inst, mem, 0u, resource);
});
}
template <typename Fn>
uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) {
const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
@@ -304,75 +392,104 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki
return observed;
}
void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t address, uint32_t index,
uint32_t bits, uint32_t data) {
const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index);
const auto shift = Binary(
ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address, ConstantU32(ctx.state, 3)),
ConstantU32(ctx.state, 3));
const auto mask = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift);
const auto value = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), data,
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)),
shift);
const auto merge = [&](uint32_t old) {
return Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), old,
Unary(ctx.state, OpNot, TypeU32(ctx.state), mask)),
value);
};
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
AtomicUpdate(ctx.state, pointer, mem.kind, merge);
} else {
const auto old = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), old, pointer});
ctx.state.builder.AddFunction({OpStore, pointer, merge(old)});
}
}
void StoreSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t bits, uint32_t data) {
const auto address = ByteAddress(ctx, inst, mem);
const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 2));
const auto access = PrepareMemoryElement(ctx, mem, resource, raw_index);
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
StoreSubwordInBounds(ctx, mem, access.resource, address, access.index, bits, data);
});
}
void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits) {
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto address = ByteAddress(ctx, inst, mem);
const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 2));
const auto access = PrepareMemoryElement(ctx, mem, raw_index);
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
const auto pointer =
EmitMemoryElementPointer(ctx.state, access.resource, access.index);
const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state),
address, ConstantU32(ctx.state, 3)),
ConstantU32(ctx.state, 3));
const auto mask =
Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift);
const auto value =
Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state),
ctx.Arg(inst, inst.NumArgs() - 2),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)),
shift);
const auto merge = [&](uint32_t old) {
return Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), old,
Unary(ctx.state, OpNot, TypeU32(ctx.state), mask)),
value);
};
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
AtomicUpdate(ctx.state, pointer, mem.kind, merge);
} else {
const auto old = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), old, pointer});
ctx.state.builder.AddFunction({OpStore, pointer, merge(old)});
}
});
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
StoreSubwordPrepared(ctx, inst, mem, resource, bits, ctx.Arg(inst, inst.NumArgs() - 2));
});
}
void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t data) {
const auto access = PrepareMemoryElement(ctx, mem, resource, DwordIndex(ctx, inst, mem));
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
ctx.state.builder.AddFunction(
{OpStore, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
data});
});
}
void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index,
uint32_t data) {
ctx.state.builder.AddFunction(
{OpStore, EmitMemoryElementPointer(ctx.state, resource, index), data});
}
void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) {
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
ctx.state.builder.AddFunction(
{OpStore, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
ctx.Arg(inst, inst.NumArgs() - 2)});
});
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
StoreWordPrepared(ctx, inst, mem, resource, ctx.Arg(inst, inst.NumArgs() - 2));
});
}
void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
void FormattedStorePrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
uint32_t component, const MemoryResourceAccess& resource,
uint32_t data) {
const auto format = BufferFormat(ctx, inst, mem);
if (!Format::IsKnownFormat(format)) {
StoreWord(ctx, inst, mem);
StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data);
return;
}
const auto info = Format::GetFormatInfo(format);
if (mem.component_index >= info.component_count) return;
const auto bits = info.component_bits[mem.component_index];
const auto component_mem = RebaseFormattedComponent(mem, format, mem.component_index);
if (component >= info.component_count) return;
const auto bits = info.component_bits[component];
const auto component_mem = RebaseFormattedComponent(mem, format, component);
if (bits == 8u || bits == 16u) {
StoreSubword(ctx, inst, component_mem, bits);
StoreSubwordPrepared(ctx, inst, component_mem, resource, bits, data);
} else {
StoreWord(ctx, inst, component_mem);
StoreWordPrepared(ctx, inst, component_mem, resource, data);
}
}
void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
FormattedStorePrepared(ctx, inst, mem, 0u, resource, ctx.Arg(inst, inst.NumArgs() - 2));
});
}
uint32_t AtomicOpcode(IR::ValueOpcode opcode) {
switch (opcode) {
case IR::ValueOpcode::BufferAtomicSwap32:
@@ -573,11 +690,192 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo
return result;
}
struct PreparedFormattedMemory {
Prospero::BufferFormat format = Prospero::BufferFormat::kInvalid;
Format::BufferFormatInfo info;
MemoryResourceAccess resource;
std::array<uint32_t, 4> addresses {};
std::array<uint32_t, 4> indices {};
uint32_t in_bounds = 0;
};
enum class FormattedAccess { Load, Store };
PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource,
uint32_t components, FormattedAccess access) {
PreparedFormattedMemory plan;
plan.format = BufferFormat(ctx, inst, mem);
plan.info = Format::GetFormatInfo(plan.format);
plan.resource = resource;
std::array<bool, 4> required_components {};
if (access == FormattedAccess::Load) {
for (uint32_t output = 0; output < components; output++) {
const auto source = ResolveFormattedSource(ctx, mem, plan.info, output);
if (source.kind == FormattedSourceKind::Memory) {
required_components[source.component] = true;
}
}
} else {
for (uint32_t component = 0;
component < std::min(components, plan.info.component_count); component++) {
required_components[component] = true;
}
}
bool first_bound = true;
for (uint32_t component = 0; component < plan.info.component_count; component++) {
if (!required_components[component]) continue;
const auto byte_offset = Format::GetFormatComponentByteOffset(plan.format, component);
bool reused = false;
for (uint32_t previous = 0; previous < component; previous++) {
if (required_components[previous] &&
Format::GetFormatComponentByteOffset(plan.format, previous) == byte_offset) {
plan.addresses[component] = plan.addresses[previous];
plan.indices[component] = plan.indices[previous];
reused = true;
break;
}
}
if (reused) continue;
const auto component_mem = RebaseFormattedComponent(mem, plan.format, component);
plan.addresses[component] = ByteAddress(ctx, inst, component_mem);
const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state),
plan.addresses[component], ConstantU32(ctx.state, 2));
plan.indices[component] = EmitMemoryElementIndex(ctx.state, resource, raw_index);
const auto component_bound =
EmitMemoryElementInBounds(ctx.state, resource, plan.indices[component]);
if (first_bound) {
plan.in_bounds = component_bound;
first_bound = false;
} else {
plan.in_bounds = AndCondition(ctx.state, plan.in_bounds, component_bound);
}
}
if (first_bound) plan.in_bounds = ConstantBool(ctx.state, true);
return plan;
}
uint32_t LoadFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
const PreparedFormattedMemory& plan, uint32_t output_component) {
return LoadFormattedComponent(
ctx, mem, plan.format, plan.info, output_component,
[&](uint32_t component) {
return LoadWordInBounds(ctx, plan.resource, plan.indices[component]);
},
[&](uint32_t component, uint32_t bits, bool sign_extend) {
return LoadSubwordInBounds(ctx, plan.resource, plan.addresses[component],
plan.indices[component], bits, sign_extend);
});
}
uint32_t ConstructU32Composite(EmitterState& state, uint32_t components,
const std::array<uint32_t, 4>& values) {
const auto result = state.builder.AllocateId();
std::vector<uint32_t> words {OpCompositeConstruct, TypeU32Composite(state, components), result};
words.insert(words.end(), values.begin(), values.begin() + components);
state.builder.AddFunction(words);
return result;
}
uint32_t FormattedOutOfBoundsValue(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
const PreparedFormattedMemory& plan, uint32_t components) {
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; component++) {
const auto source = ResolveFormattedSource(ctx, mem, plan.info, component);
values[component] = FormattedConstant(ctx, plan.info, source.kind);
}
return ConstructU32Composite(ctx.state, components, values);
}
void StoreFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
const PreparedFormattedMemory& plan, uint32_t component,
uint32_t data) {
if (component >= plan.info.component_count) return;
const auto bits = plan.info.component_bits[component];
if (bits == 8u || bits == 16u) {
StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses[component],
plan.indices[component], bits, data);
} else {
StoreWordInBounds(ctx, plan.resource, plan.indices[component], data);
}
}
uint32_t LoadWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
auto& state = ctx.state;
return EmitValueOrDefaultIfCondition(
state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU32Composite(state, components),
ConstantU32CompositeZero(state, components), [&]() {
const auto mem = ctx.Memory(inst);
const auto resource = PrepareMemoryResourceAccess(state, mem);
const auto format =
mem.formatted ? BufferFormat(ctx, inst, mem) : Prospero::BufferFormat::kInvalid;
if (Format::IsKnownFormat(format)) {
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, components,
FormattedAccess::Load);
return EmitValueOrDefaultIfCondition(
state, plan.in_bounds, TypeU32Composite(state, components),
FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() {
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; component++) {
values[component] = LoadFormattedInBounds(ctx, mem, plan, component);
}
return ConstructU32Composite(state, components, values);
});
}
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; component++) {
values[component] =
LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource);
}
return ConstructU32Composite(state, components, values);
});
}
void StoreWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
auto& state = ctx.state;
EmitIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto mem = ctx.Memory(inst);
const auto resource = PrepareMemoryResourceAccess(state, mem);
const auto composite = ctx.Arg(inst, inst.NumArgs() - 2);
const auto format =
mem.formatted ? BufferFormat(ctx, inst, mem) : Prospero::BufferFormat::kInvalid;
if (Format::IsKnownFormat(format)) {
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, components,
FormattedAccess::Store);
EmitIfCondition(state, plan.in_bounds, [&]() {
for (uint32_t component = 0; component < components; component++) {
const auto data = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeExtract, TypeU32(state), data, composite, component});
StoreFormattedInBounds(ctx, mem, plan, component, data);
}
});
return;
}
for (uint32_t component = 0; component < components; component++) {
const auto data = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeExtract, TypeU32(state), data, composite, component});
StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data);
}
});
}
} // namespace
bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto op = inst.GetOpcode();
auto& state = ctx.state;
const auto op = inst.GetOpcode();
const auto buffer_components = IR::BufferComponentCount(op);
if (buffer_components > 1u) {
if (IR::BufferAccessOf(op) == IR::BufferAccess::Read) {
ctx.Define(inst, LoadWideBuffer(ctx, inst, buffer_components));
} else {
StoreWideBuffer(ctx, inst, buffer_components);
}
return true;
}
if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) &&
ctx.Memory(inst).planning_only) {
return true;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp
+10 -4
View File
@@ -42,6 +42,11 @@ uint32_t TypeU32Vector(EmitterState& state, uint32_t components) {
return state.builder.Type(OpTypeVector, {TypeU32(state), components});
}
uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
EXIT_IF(components < 2u || components > 4u);
return components == 2u ? TypeU32Pair(state) : TypeU32Vector(state, components);
}
uint32_t TypeI32Vector(EmitterState& state, uint32_t components) {
return state.builder.Type(OpTypeVector, {TypeI32(state), components});
}
@@ -241,10 +246,11 @@ uint32_t ConstantU64(EmitterState& state, uint64_t value) {
ConstantU32(state, static_cast<uint32_t>(value >> 32u))});
}
uint32_t ConstantU32x4Zero(EmitterState& state) {
const auto zero = ConstantU32(state, 0);
return state.builder.Constant(OpConstantComposite, TypeU32Vector(state, 4),
{zero, zero, zero, zero});
uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components) {
EXIT_IF(components < 2u || components > 4u);
const auto zero = ConstantU32(state, 0);
std::vector<uint32_t> values(components, zero);
return state.builder.Constant(OpConstantComposite, TypeU32Composite(state, components), values);
}
uint32_t GlslStd450(EmitterState& state) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
+3
View File
@@ -46,6 +46,8 @@ uint32_t SpillPointerType(EmitterState& state, IR::Type type) {
case IR::Type::F32: return TypePointer(state, StorageClassFunction, TypeF32(state));
case IR::Type::U64:
case IR::Type::U32x2: return TypePointer(state, StorageClassFunction, TypeU32Pair(state));
case IR::Type::U32x3:
return TypePointer(state, StorageClassFunction, TypeU32Vector(state, 3));
case IR::Type::U32x4:
return TypePointer(state, StorageClassFunction, TypeU32Vector(state, 4));
case IR::Type::F32x2:
@@ -405,6 +407,7 @@ uint32_t ValueEmitContext::TypeId(IR::Type type) const {
case IR::Type::U64:
case IR::Type::U32x2: return TypeU32Pair(state);
case IR::Type::F32: return TypeF32(state);
case IR::Type::U32x3: return TypeU32Vector(state, 3);
case IR::Type::U32x4: return TypeU32Vector(state, 4);
case IR::Type::F32x2: return TypeF32Vector(state, 2);
default: return 0;
src/graphics/shader/recompiler/frontend/translate/Attribute.cpp
+6 -3
View File
@@ -13,9 +13,12 @@ IR::ExportFlags Translator::AddExportInfo(const IR::Instruction& inst) {
bool Translator::TranslateAttributeOperation(const IR::Instruction& inst) {
if (inst.op == IR::Opcode::LoadInputF32) {
WriteOperand(inst.dst,
ir.Emit(IR::ValueOpcode::GetAttribute,
{IR::Value(inst.input_info.attr), IR::Value(inst.input_info.chan)}));
for (uint32_t component = 0; component < inst.input_info.component_count; component++) {
WriteOperand(OffsetOperand(inst.dst, component),
ir.Emit(IR::ValueOpcode::GetAttribute,
{IR::Value(inst.input_info.attr),
IR::Value(inst.input_info.chan + component)}));
}
return true;
}
if (inst.op == IR::Opcode::Export) {
src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp
+1 -2
View File
@@ -12,7 +12,6 @@ bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
}
bool Translator::TranslateInstruction(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot,
const ScalarMemorySourceValues* scalar_source_snapshot) {
switch (IR::GetOpcodeInfo(inst.op).lowering_class) {
case IR::LoweringClass::Control: return TranslateControlOperation(inst);
@@ -20,7 +19,7 @@ bool Translator::TranslateInstruction(const IR::Instruction& inst,
case IR::LoweringClass::Lane: return TranslateLaneOperation(inst);
case IR::LoweringClass::State: return TranslateStateOperation(inst);
case IR::LoweringClass::Memory:
return TranslateMemoryOperation(inst, address_snapshot, scalar_source_snapshot);
return TranslateMemoryOperation(inst, scalar_source_snapshot);
case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst);
case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst);
case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst);
src/graphics/shader/recompiler/frontend/translate/Memory.cpp
+49 -20
View File
@@ -125,8 +125,8 @@ IR::ValueOpcode Translator::ImageAtomicOpcode(IR::Opcode opcode) {
}
}
BufferAddressValues Translator::ReadBufferAddress(const IR::Instruction& inst,
uint32_t first_source) {
Translator::BufferAddress Translator::ReadBufferAddress(const IR::Instruction& inst,
uint32_t first_source) {
uint32_t cursor = first_source;
const auto next = [&]() {
return cursor < inst.src_count ? ReadU32(inst.src[cursor++]) : IR::U32(IR::Value(0u));
@@ -201,12 +201,6 @@ IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) {
}
}
bool Translator::IsBufferLoadOperation(IR::Opcode opcode) {
return opcode == IR::Opcode::BufferLoadUbyte || opcode == IR::Opcode::BufferLoadSbyte ||
opcode == IR::Opcode::BufferLoadUshort || opcode == IR::Opcode::BufferLoadSshort ||
opcode == IR::Opcode::BufferLoadDword;
}
bool Translator::IsScalarMemoryLoadOperation(IR::Opcode opcode) {
return opcode == IR::Opcode::SLoadDword || opcode == IR::Opcode::SBufferLoadDword;
}
@@ -247,8 +241,7 @@ bool Translator::TranslateScalarMemory(const IR::Instruction& inst,
}
}
bool Translator::TranslateBufferLoad(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot) {
bool Translator::TranslateBufferLoad(const IR::Instruction& inst) {
IR::ValueOpcode opcode;
uint32_t bits;
bool sign;
@@ -266,19 +259,33 @@ bool Translator::TranslateBufferLoad(const IR::Instruction& inst,
sign = inst.op == IR::Opcode::BufferLoadSshort;
break;
case IR::Opcode::BufferLoadDword:
opcode = IR::ValueOpcode::LoadBufferU32;
bits = 32u;
sign = false;
switch (inst.memory.data_dwords) {
case 1u: opcode = IR::ValueOpcode::LoadBufferU32; break;
case 2u: opcode = IR::ValueOpcode::LoadBufferU32x2; break;
case 3u: opcode = IR::ValueOpcode::LoadBufferU32x3; break;
case 4u: opcode = IR::ValueOpcode::LoadBufferU32x4; break;
default: return false;
}
bits = 32u;
sign = false;
break;
default: return false;
}
const auto resource = GetBufferResource(inst.memory);
const auto address =
address_snapshot != nullptr ? *address_snapshot : ReadBufferAddress(inst, 0);
const auto address = ReadBufferAddress(inst, 0);
const auto loaded =
ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, ir.GetExec()},
AddMemoryInfo(inst));
WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign));
if (bits != 32u) {
WriteOperand(inst.dst, WidenSubdword(loaded, bits, sign));
} else if (inst.memory.data_dwords == 1u) {
WriteOperand(inst.dst, loaded);
} else {
for (uint32_t component = 0; component < inst.memory.data_dwords; component++) {
WriteOperand(OffsetOperand(inst.dst, component),
ir.CompositeExtract(loaded, component));
}
}
return true;
}
@@ -298,8 +305,31 @@ bool Translator::TranslateBufferStore(const IR::Instruction& inst) {
value = NarrowSubdword(data, 16u);
break;
case IR::Opcode::BufferStoreDword:
opcode = IR::ValueOpcode::StoreBufferU32;
value = data;
switch (inst.memory.data_dwords) {
case 1u:
opcode = IR::ValueOpcode::StoreBufferU32;
value = data;
break;
case 2u:
opcode = IR::ValueOpcode::StoreBufferU32x2;
value = ir.Emit(IR::ValueOpcode::CompositeConstructU32x2,
{data, ReadU32(OffsetOperand(inst.src[0], 1u))});
break;
case 3u:
opcode = IR::ValueOpcode::StoreBufferU32x3;
value = ir.Emit(IR::ValueOpcode::CompositeConstructU32x3,
{data, ReadU32(OffsetOperand(inst.src[0], 1u)),
ReadU32(OffsetOperand(inst.src[0], 2u))});
break;
case 4u:
opcode = IR::ValueOpcode::StoreBufferU32x4;
value = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4,
{data, ReadU32(OffsetOperand(inst.src[0], 1u)),
ReadU32(OffsetOperand(inst.src[0], 2u)),
ReadU32(OffsetOperand(inst.src[0], 3u))});
break;
default: return false;
}
break;
default: return false;
}
@@ -594,7 +624,6 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
}
bool Translator::TranslateMemoryOperation(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot,
const ScalarMemorySourceValues* scalar_source_snapshot) {
switch (inst.op) {
case IR::Opcode::LoadSrtDword:
@@ -606,7 +635,7 @@ bool Translator::TranslateMemoryOperation(const IR::Instruction& inst,
case IR::Opcode::BufferLoadSbyte:
case IR::Opcode::BufferLoadUshort:
case IR::Opcode::BufferLoadSshort:
case IR::Opcode::BufferLoadDword: return TranslateBufferLoad(inst, address_snapshot);
case IR::Opcode::BufferLoadDword: return TranslateBufferLoad(inst);
case IR::Opcode::BufferStoreByte:
case IR::Opcode::BufferStoreShort:
src/graphics/shader/recompiler/frontend/translate/Translate.cpp
+13 -21
View File
@@ -668,26 +668,12 @@ void Translator::WriteCompareResult(const IR::Operand& operand, IR::U1 value) {
}
bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error) {
BufferAddressValues buffer_address_snapshot {};
ScalarMemorySourceValues scalar_source_snapshot {};
uint32_t buffer_address_pc = 0;
uint32_t scalar_source_pc = 0;
bool has_buffer_address_snapshot = false;
bool has_scalar_source_snapshot = false;
uint32_t scalar_source_pc = 0;
bool has_scalar_source_snapshot = false;
for (const auto& source_inst: source.instructions) {
current_opcode = source_inst.op;
current_pc = source_inst.pc;
const bool grouped_buffer_load =
IsBufferLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u;
if (grouped_buffer_load &&
(!has_buffer_address_snapshot || buffer_address_pc != source_inst.pc ||
source_inst.memory.component_index == 0u)) {
buffer_address_snapshot = ReadBufferAddress(source_inst, 0);
buffer_address_pc = source_inst.pc;
has_buffer_address_snapshot = true;
} else if (!grouped_buffer_load) {
has_buffer_address_snapshot = false;
}
const bool grouped_scalar_load =
IsScalarMemoryLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u;
if (grouped_scalar_load &&
@@ -703,7 +689,6 @@ bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error
return Fail(error, "value IR input opcode is out of range");
}
if (TranslateInstruction(source_inst,
grouped_buffer_load ? &buffer_address_snapshot : nullptr,
grouped_scalar_load ? &scalar_source_snapshot : nullptr)) {
continue;
}
@@ -780,11 +765,11 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
const uint32_t wave_size = source.wave_size;
const bool per_invocation_masks = source.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
uint32_t vector_limit = 1u;
const auto include_vector = [&](const IR::Operand& operand) {
const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) {
if (operand.kind == IR::OperandKind::Register &&
operand.reg.file == IR::RegisterFile::Vector) {
vector_limit =
std::min(IR::NumVectorRegs, std::max(vector_limit, operand.reg.index + 1u));
std::min(IR::NumVectorRegs, std::max(vector_limit, operand.reg.index + count));
}
};
for (const auto& block: source.blocks) {
@@ -794,6 +779,13 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
for (uint32_t index = 0; index < inst.src_count; index++) {
include_vector(inst.src[index]);
}
if (inst.op == IR::Opcode::BufferLoadDword) {
include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u));
} else if (inst.op == IR::Opcode::BufferStoreDword) {
include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u));
} else if (inst.op == IR::Opcode::LoadInputF32) {
include_vector(inst.dst, std::max(inst.input_info.component_count, 1u));
}
}
}
result.block_storage.reserve(source.blocks.size() + 1u);
@@ -846,8 +838,8 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
}
entry_ir.SetExec(IR::U1(IR::Value(true)));
entry_ir.SetExecLo(IR::U32(IR::Value(per_invocation_masks ? 1u : 0xffffffffu)));
entry_ir.SetExecHi(IR::U32(IR::Value(
!per_invocation_masks && source.wave_size > 32u ? 0xffffffffu : 0u)));
entry_ir.SetExecHi(
IR::U32(IR::Value(!per_invocation_masks && source.wave_size > 32u ? 0xffffffffu : 0u)));
if (source.stage == ShaderType::Compute) {
const auto* cs = compute_input_info;
const auto thread_ids =
src/graphics/shader/recompiler/frontend/translate/Translator.h
+12 -17
View File
@@ -7,12 +7,6 @@
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
struct BufferAddressValues {
IR::U32 index;
IR::U32 offset;
IR::U32 soffset;
};
struct ScalarMemorySourceValues {
IR::Value resource;
IR::U32 offset;
@@ -70,6 +64,11 @@ private:
IR::Value low;
IR::Value high;
};
struct BufferAddress {
IR::U32 index;
IR::U32 offset;
IR::U32 soffset;
};
AddressOperands ReadAddressOperands(const IR::Instruction& inst, uint32_t first_source);
IR::U32 GetResourceDword(uint32_t index, uint32_t dword);
IR::Value GetBufferResource(const IR::MemoryInfo& memory);
@@ -83,18 +82,16 @@ private:
void WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask,
uint32_t component_limit);
IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode);
BufferAddressValues ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign);
IR::Value NarrowSubdword(IR::U32 value, uint32_t bits);
IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode);
IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode, bool gds);
bool IsBufferLoadOperation(IR::Opcode opcode);
bool IsScalarMemoryLoadOperation(IR::Opcode opcode);
BufferAddress ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign);
IR::Value NarrowSubdword(IR::U32 value, uint32_t bits);
IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode);
IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode, bool gds);
bool IsScalarMemoryLoadOperation(IR::Opcode opcode);
ScalarMemorySourceValues ReadScalarMemorySource(const IR::Instruction& inst);
bool TranslateScalarMemory(const IR::Instruction& inst,
const ScalarMemorySourceValues* source_snapshot);
bool TranslateBufferLoad(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot);
bool TranslateBufferLoad(const IR::Instruction& inst);
bool TranslateBufferStore(const IR::Instruction& inst);
bool TranslateAtomicMemory(const IR::Instruction& inst);
bool TranslateFlatLoad(const IR::Instruction& inst);
@@ -111,7 +108,6 @@ private:
IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
bool TranslateInstruction(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot,
const ScalarMemorySourceValues* scalar_source_snapshot);
bool TranslateStateOperation(const IR::Instruction& inst);
bool TranslateControlOperation(const IR::Instruction& inst);
@@ -119,7 +115,6 @@ private:
bool TranslateLaneOperation(const IR::Instruction& inst);
bool TranslateAttributeOperation(const IR::Instruction& inst);
bool TranslateMemoryOperation(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot,
const ScalarMemorySourceValues* scalar_source_snapshot);
bool TranslateIntegerCompare(const IR::Instruction& inst);
bool TranslateInteger16Compare(const IR::Instruction& inst);
src/graphics/shader/recompiler/ir/IREmitter.cpp
+8 -1
View File
@@ -140,7 +140,14 @@ U64 IREmitter::PackUint2x32(U32 low, U32 high) {
}
U32 IREmitter::CompositeExtract(Value composite, uint32_t index) {
return U32(Emit(ValueOpcode::CompositeExtractU32x2, {composite, Value(index)}));
ValueOpcode opcode;
switch (composite.GetType()) {
case Type::U32x2: opcode = ValueOpcode::CompositeExtractU32x2; break;
case Type::U32x3: opcode = ValueOpcode::CompositeExtractU32x3; break;
case Type::U32x4: opcode = ValueOpcode::CompositeExtractU32x4; break;
default: EXIT("invalid U32 composite type\n");
}
return U32(Emit(opcode, {composite, Value(index)}));
}
U32 IREmitter::IAdd(U32 lhs, U32 rhs) {
src/graphics/shader/recompiler/ir/ShaderIR.cpp
-48
View File
@@ -328,54 +328,6 @@ MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind ki
return mem;
}
uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded,
uint32_t component_index) {
if (!decoded.typed || !decoded.formatted) {
return component_index * 4u;
}
const auto format = Format::DecodeTBufferFormat(decoded.data_format, decoded.number_format);
if (!Format::IsKnownFormat(format)) {
return component_index * 4u;
}
return Format::GetFormatComponentByteOffset(format, component_index);
}
uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded) {
if (!decoded.typed || !decoded.formatted) {
return decoded.data_dwords;
}
const auto format = Format::DecodeTBufferFormat(decoded.data_format, decoded.number_format);
if (!Format::IsKnownFormat(format)) {
return decoded.data_dwords;
}
return Format::GetFormatComponentCount(format);
}
bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value,
BasicBlock& block, std::string* error) {
Instruction inst;
inst.pc = pc;
inst.op = Opcode::MoveU32;
inst.src[0] = MakeImmediateU32(value);
inst.src_count = 1;
if (!LowerRegisterOperand(dst, inst.dst, error)) {
return false;
}
block.instructions.push_back(inst);
return true;
}
MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, uint32_t component_index) {
auto mem = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
mem.offset += TypedBufferComponentOffsetBytes(decoded, component_index);
mem.data_dwords = 1;
mem.component_index = component_index;
mem.component_count = decoded.data_dwords;
return mem;
}
MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t byte_offset) {
auto mem = MemoryInfoFromDecoded(decoded, kind);
src/graphics/shader/recompiler/ir/ShaderIR.h
+3 -2
View File
@@ -123,8 +123,9 @@ struct ExportInfo {
};
struct InputInfo {
uint32_t attr = 0;
uint32_t chan = 0;
uint32_t attr = 0;
uint32_t chan = 0;
uint32_t component_count = 1;
bool operator==(const InputInfo& other) const = default;
};
src/graphics/shader/recompiler/ir/ShaderIRInternal.h
+3 -9
View File
@@ -36,15 +36,9 @@ bool TryOffsetScalarDestination(const Decoder::Operand& operand, uin
Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index);
MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t dword_index);
uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded,
uint32_t component_index);
uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded);
bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value,
BasicBlock& block, std::string* error);
MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, uint32_t component_index);
MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t byte_offset);
ResourceKind FlatSegmentResourceKind(uint32_t segment);
MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t byte_offset);
ResourceKind FlatSegmentResourceKind(uint32_t segment);
const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index);
Decoder::Operand M0Operand();
src/graphics/shader/recompiler/ir/ShaderIRLog.cpp
+2 -1
View File
@@ -130,7 +130,8 @@ std::string InputInfoToString(Opcode op, const InputInfo& input) {
if (op != Opcode::LoadInputF32) {
return "";
}
return fmt::format(" ; input_attr={} input_chan={}", input.attr, input.chan);
return fmt::format(" ; input_attr={} input_chan={} input_components={}", input.attr, input.chan,
input.component_count);
}
std::string TerminatorToString(const CFG::Terminator& term) {
src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp
+17 -40
View File
@@ -121,52 +121,29 @@ bool LowerBufferAddressSources(const Decoder::Instruction& decoded, Instruction&
}
bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto ir_op = BufferLoadOpcode(decoded);
const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
const auto typed_format_components = TypedBufferFormatComponentCount(decoded);
for (uint32_t i = 0; i < count; i++) {
if (i >= typed_format_components) {
if (!LowerMoveImmediateU32(decoded.pc, OffsetDecodedRegister(decoded.dst, i), 0, block,
error)) {
return false;
}
continue;
}
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.memory = OffsetBufferMemoryInfo(decoded, i);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
!LowerBufferAddressSources(decoded, inst, 0, error)) {
return false;
}
block.instructions.push_back(inst);
Instruction inst;
inst.pc = decoded.pc;
inst.op = BufferLoadOpcode(decoded);
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerBufferAddressSources(decoded, inst, 0, error)) {
return false;
}
block.instructions.push_back(inst);
return true;
}
bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto ir_op = BufferStoreOpcode(decoded.data_bits);
auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
if (decoded.data_bits == 32u) {
const auto typed_format_components = TypedBufferFormatComponentCount(decoded);
if (typed_format_components < count) {
count = typed_format_components;
}
}
for (uint32_t i = 0; i < count; i++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.memory = OffsetBufferMemoryInfo(decoded, i);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, i), inst.src[0], error) ||
!LowerBufferAddressSources(decoded, inst, 1, error)) {
return false;
}
block.instructions.push_back(inst);
Instruction inst;
inst.pc = decoded.pc;
inst.op = BufferStoreOpcode(decoded.data_bits);
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(decoded.dst, inst.src[0], error) ||
!LowerBufferAddressSources(decoded, inst, 1, error)) {
return false;
}
block.instructions.push_back(inst);
return true;
}
src/graphics/shader/recompiler/ir/ValueProgram.cpp
+40
View File
@@ -304,6 +304,46 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st
return Fail(error, fmt::format("untyped opcode {} survived translation",
ValueOpcodeName(inst.GetOpcode())));
}
const auto buffer_components = BufferComponentCount(inst.GetOpcode());
if (buffer_components != 0u) {
const auto memory_index = inst.Flags<MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) {
return Fail(error, fmt::format("{} has an invalid memory-info index",
ValueOpcodeName(inst.GetOpcode())));
}
const auto& memory = program.memory_info[memory_index];
if (memory.kind != ResourceKind::Buffer &&
memory.kind != ResourceKind::ScalarBuffer) {
return Fail(error, fmt::format("{} has a non-buffer resource kind",
ValueOpcodeName(inst.GetOpcode())));
}
if (buffer_components > 1u &&
(memory.kind != ResourceKind::Buffer || memory.data_bits != 32u ||
memory.data_dwords != buffer_components || memory.component_index != 0u)) {
return Fail(error, fmt::format("{} has inconsistent native-wide metadata",
ValueOpcodeName(inst.GetOpcode())));
}
if (buffer_components == 1u &&
(inst.GetOpcode() == ValueOpcode::LoadBufferU32 ||
inst.GetOpcode() == ValueOpcode::StoreBufferU32) &&
memory.data_dwords != 1u) {
return Fail(error, fmt::format("{} retains scalar-sibling width metadata",
ValueOpcodeName(inst.GetOpcode())));
}
}
uint32_t composite_components = 0u;
switch (inst.GetOpcode()) {
case ValueOpcode::CompositeExtractU32x2: composite_components = 2u; break;
case ValueOpcode::CompositeExtractU32x3: composite_components = 3u; break;
case ValueOpcode::CompositeExtractU32x4: composite_components = 4u; break;
default: break;
}
if (composite_components != 0u &&
(!inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32 ||
inst.Arg(1).U32() >= composite_components)) {
return Fail(error, fmt::format("{} has an invalid component index",
ValueOpcodeName(inst.GetOpcode())));
}
for (size_t arg_index = 0; arg_index < inst.NumArgs(); arg_index++) {
const auto arg = inst.Arg(arg_index);
if (arg.IsEmpty()) {
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp
+21 -2
View File
@@ -34,6 +34,7 @@ constexpr Type U64 = Type::U64;
constexpr Type F16 = Type::F16;
constexpr Type F32 = Type::F32;
constexpr Type U32x2 = Type::U32x2;
constexpr Type U32x3 = Type::U32x3;
constexpr Type U32x4 = Type::U32x4;
constexpr Type F32x2 = Type::F32x2;
constexpr Type SrtResource = Type::SrtResource;
@@ -133,10 +134,16 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
case ValueOpcode::ReadConstBuffer:
case ValueOpcode::LoadBufferU8:
case ValueOpcode::LoadBufferU16:
case ValueOpcode::LoadBufferU32: return BufferAccess::Read;
case ValueOpcode::LoadBufferU32:
case ValueOpcode::LoadBufferU32x2:
case ValueOpcode::LoadBufferU32x3:
case ValueOpcode::LoadBufferU32x4: return BufferAccess::Read;
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32: return BufferAccess::Write;
case ValueOpcode::StoreBufferU32:
case ValueOpcode::StoreBufferU32x2:
case ValueOpcode::StoreBufferU32x3:
case ValueOpcode::StoreBufferU32x4: return BufferAccess::Write;
case ValueOpcode::BufferAtomicSwap32:
case ValueOpcode::BufferAtomicIAdd32:
case ValueOpcode::BufferAtomicISub32:
@@ -153,4 +160,16 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
}
}
uint32_t BufferComponentCount(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::LoadBufferU32x2:
case ValueOpcode::StoreBufferU32x2: return 2u;
case ValueOpcode::LoadBufferU32x3:
case ValueOpcode::StoreBufferU32x3: return 3u;
case ValueOpcode::LoadBufferU32x4:
case ValueOpcode::StoreBufferU32x4: return 4u;
default: return BufferAccessOf(opcode) == BufferAccess::None ? 0u : 1u;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h
+1
View File
@@ -55,6 +55,7 @@ static_assert(std::is_trivially_copyable_v<ExportFlags>);
[[nodiscard]] size_t NumArgsOf(ValueOpcode opcode);
[[nodiscard]] bool HasSideEffects(ValueOpcode opcode);
[[nodiscard]] BufferAccess BufferAccessOf(ValueOpcode opcode);
[[nodiscard]] uint32_t BufferComponentCount(ValueOpcode opcode);
[[nodiscard]] std::string_view ValueOpcodeName(ValueOpcode opcode);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc
+8
View File
@@ -57,9 +57,11 @@ VALUE_OPCODE(ConvertU32F32, U32, F32)
VALUE_OPCODE(ConvertF32S32, F32, U32)
VALUE_OPCODE(ConvertF32U32, F32, U32)
VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32)
VALUE_OPCODE(CompositeConstructU32x3, U32x3, U32, U32, U32)
VALUE_OPCODE(CompositeConstructF32x2, F32x2, F32, F32)
VALUE_OPCODE(CompositeConstructU32x4, U32x4, U32, U32, U32, U32)
VALUE_OPCODE(CompositeExtractU32x2, U32, U32x2, U32)
VALUE_OPCODE(CompositeExtractU32x3, U32, U32x3, U32)
VALUE_OPCODE(CompositeExtractU32x4, U32, U32x4, U32)
VALUE_OPCODE(PackUint2x32, U64, U32x2)
VALUE_OPCODE(UnpackUint2x32, U32x2, U64)
@@ -215,9 +217,15 @@ VALUE_OPCODE(StoreAddressU32, Void, AddressResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU8, U8, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU16, U16, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU32, U32, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU32x2, U32x2, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU32x3, U32x3, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU32x4, U32x4, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(StoreBufferU8, Void, BufferResource, U32, U32, U32, U8, U1)
VALUE_OPCODE(StoreBufferU16, Void, BufferResource, U32, U32, U32, U16, U1)
VALUE_OPCODE(StoreBufferU32, Void, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(StoreBufferU32x2, Void, BufferResource, U32, U32, U32, U32x2, U1)
VALUE_OPCODE(StoreBufferU32x3, Void, BufferResource, U32, U32, U32, U32x3, U1)
VALUE_OPCODE(StoreBufferU32x4, Void, BufferResource, U32, U32, U32, U32x4, U1)
VALUE_OPCODE(BufferAtomicSwap32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicIAdd32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1)
src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp
+3
View File
@@ -287,6 +287,9 @@ void FoldInstruction(Inst& inst) {
case ValueOpcode::CompositeExtractU32x2:
FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x2, 2);
return;
case ValueOpcode::CompositeExtractU32x3:
FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x3, 3);
return;
case ValueOpcode::CompositeExtractU32x4:
FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x4, 4);
return;