raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 02:47:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(shader): simplify buffer access emission

Browse Source
This commit is contained in:
nmzik
2026-08-19 06:32:47 +02:00
parent 49145c1902
commit 3ea2099245
12 changed files with 474 additions and 474 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/ShaderRecompiler.cpp
+3 -1
View File
@@ -942,7 +942,9 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
if (!IR::AllocateBindings(ir, layout_options, error)) {
return false;
}
Spirv::AnalyzeProgramRequirements(ir);
if (!Spirv::AnalyzeProgramRequirements(ir, error)) {
return false;
}
std::string ir_dump;
if (options.dump_ir) {
ir_dump = MakeIrDump(cfg, ir);
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
+35 -27
View File
@@ -6,8 +6,6 @@
#include <algorithm>
#include <array>
#include <cinttypes>
#include <cstdio>
namespace Libs::Graphics::ShaderRecompiler::Spirv {
@@ -219,23 +217,12 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
} // namespace
static constexpr size_t InitialEmitterVectorReserve = 4096;
static void ReportReserveExceeded(const IR::Program& program, const char* vector_name,
size_t size) {
if (size <= InitialEmitterVectorReserve) {
return;
}
std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64
" stage=%u vector=%s size=%zu reserve=%zu\n",
program.shader_hash, static_cast<unsigned>(program.stage), vector_name, size,
InitialEmitterVectorReserve);
}
void AnalyzeProgramRequirements(IR::Program& program) {
auto& requirements = program.spirv_requirements.emplace();
bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
program.spirv_requirements.reset();
IR::SpirvRequirements requirements {};
if (program.values == nullptr) {
return;
program.spirv_requirements.emplace(requirements);
return true;
}
const auto MarkExactSubgroup = [&] {
requirements.requires_exact_subgroup = true;
@@ -243,6 +230,25 @@ void AnalyzeProgramRequirements(IR::Program& program) {
};
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
if (memory_index >= program.values->memory_info.size()) {
return Fail(error, "buffer operation has invalid memory metadata");
}
const auto& memory = program.values->memory_info[memory_index];
if (memory.kind == IR::ResourceKind::Buffer) {
if (memory.resource >= program.info.buffers.size()) {
return Fail(error, "buffer operation has invalid resource metadata");
}
if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
if (program.stage != ShaderType::Compute) {
return Fail(error, "buffer ADD_TID is only valid for compute shaders");
}
requirements.requires_exact_subgroup = true;
requirements.subgroup_local_invocation_id = true;
}
}
}
switch (inst.GetOpcode()) {
case IR::ValueOpcode::Ballot: MarkExactSubgroup(); break;
case IR::ValueOpcode::DppMoveU32:
@@ -302,8 +308,10 @@ void AnalyzeProgramRequirements(IR::Program& program) {
case IR::ValueOpcode::SharedAtomicOr32:
case IR::ValueOpcode::SharedAtomicXor32: {
const auto index = inst.Flags<IR::MemoryFlags>().index;
if (index >= program.values->memory_info.size()) {
return Fail(error, "shared operation has invalid memory metadata");
}
if (program.stage != ShaderType::Compute &&
index < program.values->memory_info.size() &&
program.values->memory_info[index].kind == IR::ResourceKind::Lds) {
requirements.function_lds = true;
}
@@ -318,8 +326,10 @@ void AnalyzeProgramRequirements(IR::Program& program) {
break;
case IR::ValueOpcode::SetAttribute: {
const auto index = inst.Flags<IR::ExportFlags>().index;
if (index >= program.values->export_info.size()) {
return Fail(error, "attribute export has invalid metadata");
}
if (program.stage == ShaderType::Pixel &&
index < program.values->export_info.size() &&
program.values->export_info[index].vm) {
requirements.pixel_valid_mask = true;
}
@@ -329,6 +339,8 @@ void AnalyzeProgramRequirements(IR::Program& program) {
}
}
}
program.spirv_requirements.emplace(requirements);
return true;
}
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
@@ -368,9 +380,9 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
state.stage = program.stage;
state.wave_size = program.wave_size;
state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
state.inputs.reserve(InitialEmitterVectorReserve);
state.outputs.reserve(InitialEmitterVectorReserve);
state.interface_variables.reserve(InitialEmitterVectorReserve);
state.inputs.reserve(program.info.inputs.size());
state.outputs.reserve(program.info.outputs.size());
state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());
CopyProgramInputsAndOutputs(state, program);
AllocateInputVariables(state);
AllocateOutputVariables(state);
@@ -379,10 +391,6 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
return false;
}
ReportReserveExceeded(program, "inputs", state.inputs.size());
ReportReserveExceeded(program, "outputs", state.outputs.size());
ReportReserveExceeded(program, "interface_variables", state.interface_variables.size());
auto binary = state.builder.Build();
if (binary.empty()) {
SetError(error, "SPIR-V builder returned an empty module");
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h
+1 -1
View File
@@ -9,7 +9,7 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv {
void AnalyzeProgramRequirements(IR::Program& program);
bool AnalyzeProgramRequirements(IR::Program& program, std::string* error);
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv, std::string* error);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp
+1 -1
View File
@@ -115,7 +115,7 @@ DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
if (state.subgroup_local_invocation_id_variable == 0) {
return ConstantU32(state, 0);
EXIT("SubgroupLocalInvocationId was not declared before SPIR-V function emission\n");
}
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+17 -32
View File
@@ -672,38 +672,32 @@ uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint3
uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc);
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem);
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc);
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
void EmitStorageBufferOffsets(EmitterState& state);
DescriptorResourceBinding StorageBufferBindingForMemory(EmitterState& state,
const IR::MemoryInfo& mem, uint32_t use_pc);
uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc);
uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t index, uint32_t use_pc);
uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t index, uint32_t use_pc);
uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index);
uint32_t LdsDwordCount(const EmitterState& state);
uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index);
struct MemoryResourceAccess {
IR::ResourceKind kind = IR::ResourceKind::None;
uint32_t object_pointer = 0;
uint32_t length = 0;
uint32_t index_offset = 0;
bool add_index_offset = false;
};
uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index);
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index);
uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
uint32_t raw_index);
uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
uint32_t use_pc);
uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index);
uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index);
uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value);
@@ -807,11 +801,6 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std:
// These templates accept local lambdas from several emitter translation units.
template <typename Fn>
void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
if (condition == 0) {
fn();
return;
}
const auto then_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId();
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
@@ -825,10 +814,6 @@ void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
template <typename Fn>
uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition, uint32_t type,
uint32_t default_value, Fn&& fn) {
if (condition == 0) {
return fn();
}
const auto then_label = state.builder.AllocateId();
const auto then_exit = state.builder.AllocateId();
const auto else_label = state.builder.AllocateId();
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+211 -206
View File
@@ -25,68 +25,63 @@ uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t
}
uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) {
if (lhs == 0) return rhs;
if (rhs == 0) return lhs;
return Binary(state, OpLogicalAnd, TypeBool(state), lhs, rhs);
}
uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
uint32_t index, uint32_t offset, uint32_t soffset) {
auto& state = ctx.state;
const auto packed =
ConstantU32(state, StorageBufferPackedStride(state, mem, inst.Flags<IR::MemoryFlags>().pc));
if (mem.formatted && !mem.typed && inst.GetOpcode() == IR::ValueOpcode::StoreBufferU32) {
const auto add_tid_bit = Binary(
state, OpBitwiseAnd, TypeU32(state),
Binary(state, OpShiftRightLogical, TypeU32(state), packed, ConstantU32(state, 20)),
ConstantU32(state, 1));
const auto add_tid =
Binary(state, OpINotEqual, TypeBool(state), add_tid_bit, ConstantU32(state, 0));
auto& state = ctx.state;
const uint32_t packed = StorageBufferPackedStride(state, mem);
const uint32_t stride = packed & 0x3fffu;
const bool swizzle = stride != 0u && ((packed >> 14u) & 1u) != 0u;
if (((packed >> 20u) & 1u) != 0u) {
const auto lane = Binary(state, OpBitwiseAnd, TypeU32(state),
EmitLocalInvocationIndex(state), ConstantU32(state, 63));
index = Select(state, TypeU32(state), add_tid,
Binary(state, OpIAdd, TypeU32(state), index, lane), index);
EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63));
index = Binary(state, OpIAdd, TypeU32(state), index, lane);
}
if (mem.offset != 0u) {
offset = Binary(state, OpIAdd, TypeU32(state), offset, ConstantU32(state, mem.offset));
}
uint32_t address = 0;
if (!swizzle) {
if (stride == 0u) {
address = offset;
} else {
const auto indexed = stride == 1u ? index
: Binary(state, OpIMul, TypeU32(state), index,
ConstantU32(state, stride));
address = Binary(state, OpIAdd, TypeU32(state), indexed, offset);
}
} else {
const uint32_t stride_enum = (packed >> 16u) & 3u;
const uint32_t index_stride = 8u << stride_enum;
const auto index_msb = Binary(state, OpShiftRightLogical, TypeU32(state), index,
ConstantU32(state, stride_enum + 3u));
const auto index_lsb = Binary(state, OpBitwiseAnd, TypeU32(state), index,
ConstantU32(state, index_stride - 1u));
const auto offset_msb =
Binary(state, OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, ~3u));
const auto offset_lsb =
Binary(state, OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, 3u));
const auto indexed_msb = stride == 1u ? index_msb
: Binary(state, OpIMul, TypeU32(state), index_msb,
ConstantU32(state, stride));
const auto msb = Binary(state, OpIMul, TypeU32(state),
Binary(state, OpIAdd, TypeU32(state), indexed_msb, offset_msb),
ConstantU32(state, index_stride));
const auto lsb = Binary(
state, OpIAdd, TypeU32(state),
Binary(state, OpShiftLeftLogical, TypeU32(state), index_lsb, ConstantU32(state, 2u)),
offset_lsb);
address = Binary(state, OpIAdd, TypeU32(state), msb, lsb);
}
const auto soffset_value = inst.Arg(3).Resolve();
if (soffset_value.IsImmediate() && soffset_value.GetType() == IR::Type::U32 &&
soffset_value.U32() == 0u) {
return address;
}
const auto stride =
Binary(state, OpBitwiseAnd, TypeU32(state), packed, ConstantU32(state, 0x3fffu));
const auto swizzle =
Binary(state, OpBitwiseAnd, TypeU32(state),
Binary(state, OpShiftRightLogical, TypeU32(state), packed, ConstantU32(state, 14)),
ConstantU32(state, 1));
const auto use_swizzle =
Binary(state, OpLogicalAnd, TypeBool(state),
Binary(state, OpINotEqual, TypeBool(state), stride, ConstantU32(state, 0)),
Binary(state, OpINotEqual, TypeBool(state), swizzle, ConstantU32(state, 0)));
const auto stride_enum =
Binary(state, OpBitwiseAnd, TypeU32(state),
Binary(state, OpShiftRightLogical, TypeU32(state), packed, ConstantU32(state, 16)),
ConstantU32(state, 3));
const auto index_shift =
Binary(state, OpIAdd, TypeU32(state), stride_enum, ConstantU32(state, 3));
const auto index_stride =
Binary(state, OpShiftLeftLogical, TypeU32(state), ConstantU32(state, 8), stride_enum);
const auto index_mask =
Binary(state, OpISub, TypeU32(state), index_stride, ConstantU32(state, 1));
offset = Binary(state, OpIAdd, TypeU32(state), offset, ConstantU32(state, mem.offset));
const auto linear = Binary(state, OpIAdd, TypeU32(state),
Binary(state, OpIMul, TypeU32(state), index, stride), offset);
const auto index_msb = Binary(state, OpShiftRightLogical, TypeU32(state), index, index_shift);
const auto index_lsb = Binary(state, OpBitwiseAnd, TypeU32(state), index, index_mask);
const auto offset_msb =
Binary(state, OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, ~3u));
const auto offset_lsb =
Binary(state, OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, 3));
const auto msb =
Binary(state, OpIMul, TypeU32(state),
Binary(state, OpIAdd, TypeU32(state),
Binary(state, OpIMul, TypeU32(state), index_msb, stride), offset_msb),
index_stride);
const auto lsb =
Binary(state, OpIAdd, TypeU32(state),
Binary(state, OpShiftLeftLogical, TypeU32(state), index_lsb, ConstantU32(state, 2)),
offset_lsb);
const auto address = Select(state, TypeU32(state), use_swizzle,
Binary(state, OpIAdd, TypeU32(state), msb, lsb), linear);
return Binary(state, OpIAdd, TypeU32(state), address, soffset);
}
@@ -156,68 +151,66 @@ uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor
ConstantU32(ctx.state, 2));
}
uint32_t InBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) {
if (mem.kind == IR::ResourceKind::Lds) {
return EmitLdsElementInBounds(ctx.state, index);
}
if (mem.kind == IR::ResourceKind::Gds) {
return EmitGdsElementInBounds(ctx.state, index);
}
return EmitStorageBufferElementInBounds(ctx.state, mem, index, pc);
struct PreparedMemoryElement {
MemoryResourceAccess resource;
uint32_t index = 0;
};
PreparedMemoryElement PrepareMemoryElement(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
uint32_t raw_index) {
auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index);
return {.resource = resource, .index = index};
}
uint32_t Pointer(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) {
return EmitMemoryElementPointer(ctx.state, mem, index, pc);
}
uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem,
uint32_t extra_offset = 0) {
mem.offset += extra_offset;
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) {
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
const auto value = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, TypeU32(ctx.state), value, Pointer(ctx, mem, index, pc)});
return value;
});
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
return EmitValueOrZeroIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
const auto value = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, TypeU32(ctx.state), value,
EmitMemoryElementPointer(ctx.state, access.resource, access.index)});
return value;
});
});
}
uint32_t LoadSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits,
bool sign_extend, uint32_t extra_offset = 0) {
mem.offset += extra_offset;
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
bool sign_extend) {
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto address = ByteAddress(ctx, inst, mem);
const auto index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 2));
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
const auto word = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, TypeU32(ctx.state), word, Pointer(ctx, mem, index, pc)});
const auto byte = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 3));
const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), byte,
ConstantU32(ctx.state, 3));
const auto value =
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state),
Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), word, shift),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
if (!sign_extend) return value;
const auto left = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), value,
ConstantU32(ctx.state, 32u - bits));
return Binary(ctx.state, OpShiftRightArithmetic, TypeU32(ctx.state), left,
ConstantU32(ctx.state, 32u - bits));
});
const auto address = ByteAddress(ctx, inst, mem);
const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 2));
const auto access = PrepareMemoryElement(ctx, mem, raw_index);
return EmitValueOrZeroIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
const auto word = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, TypeU32(ctx.state), word,
EmitMemoryElementPointer(ctx.state, access.resource, access.index)});
const auto byte = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 3));
const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), byte,
ConstantU32(ctx.state, 3));
const auto value =
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state),
Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), word, shift),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
if (!sign_extend) return value;
const auto left = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), value,
ConstantU32(ctx.state, 32u - bits));
return Binary(ctx.state, OpShiftRightArithmetic, TypeU32(ctx.state), left,
ConstantU32(ctx.state, 32u - bits));
});
});
}
Prospero::BufferFormat BufferFormat(const ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem) {
return mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format)
: StorageBufferFormat(ctx.state, mem, inst.Flags<IR::MemoryFlags>().pc);
: StorageBufferFormat(ctx.state, mem);
}
IR::MemoryInfo RebaseFormattedComponent(IR::MemoryInfo mem, Prospero::BufferFormat format,
@@ -311,54 +304,55 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki
return observed;
}
void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits,
uint32_t extra_offset = 0) {
mem.offset += extra_offset;
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits) {
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto address = ByteAddress(ctx, inst, mem);
const auto index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 2));
EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
const auto pointer = Pointer(ctx, mem, index, pc);
const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 3)),
ConstantU32(ctx.state, 3));
const auto mask = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift);
const auto value = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state),
ctx.Arg(inst, inst.NumArgs() - 2),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)),
shift);
const auto merge = [&](uint32_t old) {
return Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), old,
Unary(ctx.state, OpNot, TypeU32(ctx.state), mask)),
value);
};
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
AtomicUpdate(ctx.state, pointer, mem.kind, merge);
} else {
const auto old = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), old, pointer});
ctx.state.builder.AddFunction({OpStore, pointer, merge(old)});
}
});
const auto address = ByteAddress(ctx, inst, mem);
const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 2));
const auto access = PrepareMemoryElement(ctx, mem, raw_index);
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
const auto pointer =
EmitMemoryElementPointer(ctx.state, access.resource, access.index);
const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state),
address, ConstantU32(ctx.state, 3)),
ConstantU32(ctx.state, 3));
const auto mask =
Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift);
const auto value =
Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state),
ctx.Arg(inst, inst.NumArgs() - 2),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)),
shift);
const auto merge = [&](uint32_t old) {
return Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), old,
Unary(ctx.state, OpNot, TypeU32(ctx.state), mask)),
value);
};
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
AtomicUpdate(ctx.state, pointer, mem.kind, merge);
} else {
const auto old = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), old, pointer});
ctx.state.builder.AddFunction({OpStore, pointer, merge(old)});
}
});
});
}
void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem,
uint32_t extra_offset = 0) {
mem.offset += extra_offset;
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) {
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
ctx.state.builder.AddFunction(
{OpStore, Pointer(ctx, mem, index, pc), ctx.Arg(inst, inst.NumArgs() - 2)});
});
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
ctx.state.builder.AddFunction(
{OpStore, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
ctx.Arg(inst, inst.NumArgs() - 2)});
});
});
}
@@ -417,29 +411,32 @@ uint32_t AtomicOpcode(IR::ValueOpcode opcode) {
uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
bool return_value) {
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
const auto result =
EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
const auto old = ctx.state.builder.AllocateId();
const auto scope = mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
ctx.state.builder.AddFunction({AtomicOpcode(inst.GetOpcode()), TypeU32(ctx.state),
old, Pointer(ctx, mem, index, pc),
ConstantU32(ctx.state, scope),
ConstantU32(ctx.state, MemorySemanticsNone), value});
if (mem.kind == IR::ResourceKind::Lds) {
const auto semantics =
MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
ctx.state.builder.AddFunction({OpMemoryBarrier,
ConstantU32(ctx.state, ScopeWorkgroup),
ConstantU32(ctx.state, semantics)});
} else {
EmitDeviceAtomicMemoryBarrier(ctx.state);
}
return old;
});
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
return EmitValueOrZeroIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index),
[&]() {
const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
const auto old = ctx.state.builder.AllocateId();
const auto scope =
mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
ctx.state.builder.AddFunction(
{AtomicOpcode(inst.GetOpcode()), TypeU32(ctx.state), old,
EmitMemoryElementPointer(ctx.state, access.resource, access.index),
ConstantU32(ctx.state, scope), ConstantU32(ctx.state, MemorySemanticsNone),
value});
if (mem.kind == IR::ResourceKind::Lds) {
const auto semantics =
MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
ctx.state.builder.AddFunction({OpMemoryBarrier,
ConstantU32(ctx.state, ScopeWorkgroup),
ConstantU32(ctx.state, semantics)});
} else {
EmitDeviceAtomicMemoryBarrier(ctx.state);
}
return old;
});
});
return return_value ? result : ConstantU32(ctx.state, 0);
}
@@ -479,41 +476,46 @@ uint32_t FloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t sour
uint32_t FloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
bool max_value) {
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
const auto pointer = Pointer(ctx, mem, index, pc);
const auto source = ctx.Arg(inst, inst.NumArgs() - 2);
return AtomicUpdate(ctx.state, pointer, mem.kind, [&](uint32_t old) {
return FloatAtomicReplacement(ctx.state, old, source, max_value);
});
});
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
return EmitValueOrZeroIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
const auto pointer =
EmitMemoryElementPointer(ctx.state, access.resource, access.index);
const auto source = ctx.Arg(inst, inst.NumArgs() - 2);
return AtomicUpdate(ctx.state, pointer, mem.kind, [&](uint32_t old) {
return FloatAtomicReplacement(ctx.state, old, source, max_value);
});
});
});
}
uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
bool max_value) {
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)});
const auto data = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable});
AtomicUpdate(ctx.state, Pointer(ctx, mem, index, pc), mem.kind, [&](uint32_t old) {
const auto old_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), old);
const auto compare_f =
Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 3));
const auto data_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), data);
const auto compare = Binary(
ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan, TypeBool(ctx.state),
max_value ? old_f : compare_f, max_value ? compare_f : old_f);
return Unary(ctx.state, OpBitcast, TypeU32(ctx.state),
Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f));
});
});
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
ctx.state.builder.AddFunction(
{OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)});
const auto data = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable});
AtomicUpdate(
ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
mem.kind, [&](uint32_t old) {
const auto old_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), old);
const auto compare_f =
Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 3));
const auto data_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), data);
const auto compare =
Binary(ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan,
TypeBool(ctx.state), max_value ? old_f : compare_f,
max_value ? compare_f : old_f);
return Unary(ctx.state, OpBitcast, TypeU32(ctx.state),
Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f));
});
});
});
return 0;
}
@@ -526,8 +528,12 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo
const auto size = Binary(state, OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
const auto address =
Binary(state, OpIAdd, TypeU32(state), base, ConstantU32(state, ctx.Memory(inst).offset));
const auto index =
const auto raw_index =
Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
auto mem = ctx.Memory(inst);
mem.kind = gds ? IR::ResourceKind::Gds : IR::ResourceKind::Lds;
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto index = EmitMemoryElementIndex(state, access, raw_index);
const auto exec = ctx.Arg(inst, 2);
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
@@ -546,8 +552,7 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo
ConstantU32(state, ScopeSubgroup), ballot});
const auto is_first =
Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), first);
const auto storage_bounds =
gds ? EmitGdsElementInBounds(state, index) : EmitLdsElementInBounds(state, index);
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
const auto m0_bounds =
gds ? Binary(state, OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
: Binary(state, OpULessThan, TypeBool(state),
@@ -556,11 +561,10 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo
state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds)));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value,
gds ? EmitGdsElementPointer(state, index) : EmitLdsElementPointer(state, index),
ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
ConstantU32(state, MemorySemanticsNone), count});
state.builder.AddFunction({append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value,
EmitMemoryElementPointer(state, access, index),
ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
ConstantU32(state, MemorySemanticsNone), count});
return value;
});
const auto result = state.builder.AllocateId();
@@ -597,13 +601,14 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
Binary(state, OpIAdd, TypeU32(state), ctx.Arg(inst, 1), ConstantU32(state, mem.offset));
const auto index =
Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
const auto condition = EmitStorageBufferElementInBounds(state, mem, index, pc);
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto element = EmitMemoryElementIndex(state, access, index);
const auto condition = EmitMemoryElementInBounds(state, access, element);
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpLoad, TypeU32(state), value,
EmitStorageBufferElementPointer(state, mem, index, pc)});
EmitMemoryElementPointer(state, access, element)});
return value;
}));
return true;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+77 -127
View File
@@ -36,9 +36,7 @@ uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint3
return ret;
}
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc) {
(void)use_pc;
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) {
if (mem.resource >= state.program.info.buffers.size()) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
"buffer specialization is missing");
@@ -46,9 +44,7 @@ uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryIn
return state.program.info.buffers[mem.resource].packed_stride;
}
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc) {
(void)use_pc;
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem) {
if (mem.resource >= state.program.info.buffers.size()) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
"buffer specialization is missing");
@@ -67,92 +63,6 @@ void EmitStorageBufferOffsets(EmitterState& state) {
}
}
bool IsAddressMemoryKind(IR::ResourceKind kind) {
return kind == IR::ResourceKind::ScalarAddress || kind == IR::ResourceKind::Flat ||
kind == IR::ResourceKind::Global || kind == IR::ResourceKind::Scratch;
}
DescriptorResourceBinding
StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) {
(void)use_pc;
const auto binding =
ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
if (state.storage_buffer_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
"storage buffer descriptor array was not emitted");
}
return binding;
}
uint32_t EmitStorageBufferIndex(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
uint32_t use_pc) {
if (IsAddressMemoryKind(mem.kind)) {
return index;
}
const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
return EmitAddU32(state, index, state.storage_buffer_offsets[binding.array_index]);
}
uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc) {
if (IsAddressMemoryKind(mem.kind)) {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
mem.resource, "address memory binding was not emitted");
}
const auto binding =
ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state), pointer,
state.address_memory_variable,
ConstantU32(state, binding.array_index)});
return pointer;
}
const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state), pointer,
state.storage_buffer_variable,
ConstantU32(state, binding.array_index)});
return pointer;
}
uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t index, uint32_t use_pc) {
index = EmitStorageBufferIndex(state, mem, index, use_pc);
const auto object = EmitStorageBufferObjectPointer(state, mem, use_pc);
const auto length = state.builder.AllocateId();
const auto in_bounds = state.builder.AllocateId();
state.builder.AddFunction({OpArrayLength, TypeU32(state), length, object, 0});
state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, length});
return in_bounds;
}
uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t index, uint32_t use_pc) {
index = EmitStorageBufferIndex(state, mem, index, use_pc);
if (IsAddressMemoryKind(mem.kind)) {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
mem.resource, "address memory binding was not emitted");
}
const auto binding =
ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
state.address_memory_variable,
ConstantU32(state, binding.array_index), ConstantU32(state, 0),
index});
return pointer;
}
const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
state.storage_buffer_variable,
ConstantU32(state, binding.array_index), ConstantU32(state, 0),
index});
return pointer;
}
uint32_t LdsDwordCount(const EmitterState& state) {
return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u;
}
@@ -169,34 +79,89 @@ static void EnsureLdsStorage(EmitterState& state) {
state.builder.AddName(state.lds_variable, "lds_dwords");
}
uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) {
EnsureLdsStorage(state);
const auto pointer = state.builder.AllocateId();
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem) {
MemoryResourceAccess access {.kind = mem.kind};
switch (mem.kind) {
case IR::ResourceKind::Lds:
EnsureLdsStorage(state);
access.object_pointer = state.lds_variable;
access.length = ConstantU32(state, LdsDwordCount(state));
return access;
case IR::ResourceKind::Gds:
if (state.gds_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Gds, mem.resource,
"GDS binding was not emitted");
}
access.object_pointer = state.gds_variable;
break;
case IR::ResourceKind::ScalarAddress:
case IR::ResourceKind::Flat:
case IR::ResourceKind::Global:
case IR::ResourceKind::Scratch: {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
mem.resource,
"address memory binding was not emitted");
}
const auto binding = ResourceForDescriptor(
state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
access.object_pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state),
access.object_pointer, state.address_memory_variable,
ConstantU32(state, binding.array_index)});
break;
}
case IR::ResourceKind::ScalarBuffer:
case IR::ResourceKind::Buffer: {
if (state.storage_buffer_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers,
mem.resource,
"storage buffer descriptor array was not emitted");
}
const auto binding =
ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
access.object_pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state),
access.object_pointer, state.storage_buffer_variable,
ConstantU32(state, binding.array_index)});
access.index_offset = state.storage_buffer_offsets[binding.array_index];
access.add_index_offset = true;
break;
}
default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
}
access.length = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain,
TypeLdsElementPointer(state, state.stage == ShaderType::Compute ? StorageClassWorkgroup
: StorageClassFunction),
pointer, state.lds_variable, index});
return pointer;
{OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
return access;
}
uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) {
EnsureLdsStorage(state);
uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
uint32_t raw_index) {
return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index;
}
uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index) {
const auto in_bounds = state.builder.AllocateId();
state.builder.AddFunction(
{OpULessThan, TypeBool(state), in_bounds, index, ConstantU32(state, LdsDwordCount(state))});
state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, access.length});
return in_bounds;
}
uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
uint32_t use_pc) {
if (mem.kind == IR::ResourceKind::Lds) {
return EmitLdsElementPointer(state, index);
uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index) {
const auto pointer = state.builder.AllocateId();
if (access.kind == IR::ResourceKind::Lds) {
state.builder.AddFunction({OpAccessChain,
TypeLdsElementPointer(state, state.stage == ShaderType::Compute
? StorageClassWorkgroup
: StorageClassFunction),
pointer, access.object_pointer, index});
} else {
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
access.object_pointer, ConstantU32(state, 0), index});
}
if (mem.kind == IR::ResourceKind::Gds) {
return EmitGdsElementPointer(state, index);
}
return EmitStorageBufferElementPointer(state, mem, index, use_pc);
return pointer;
}
uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) {
@@ -338,21 +303,6 @@ void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
{OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)});
}
uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) {
const auto length = state.builder.AllocateId();
const auto in_bounds = state.builder.AllocateId();
state.builder.AddFunction({OpArrayLength, TypeU32(state), length, state.gds_variable, 0});
state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, length});
return in_bounds;
}
uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
state.gds_variable, ConstantU32(state, 0), index});
return pointer;
}
uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
if ((control & 0xc000u) == 0xc000u) {
const uint32_t mask = control & 0x1fu;
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp
+29 -15
View File
@@ -71,15 +71,16 @@ std::string_view ValueOpcodeName(ValueOpcode opcode) {
}
bool HasSideEffects(ValueOpcode opcode) {
const auto buffer_access = BufferAccessOf(opcode);
if (buffer_access == BufferAccess::Write || buffer_access == BufferAccess::Atomic) {
return true;
}
switch (opcode) {
case ValueOpcode::Reference:
case ValueOpcode::ReferenceU32:
case ValueOpcode::StoreAddressU8:
case ValueOpcode::StoreAddressU16:
case ValueOpcode::StoreAddressU32:
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32:
case ValueOpcode::WriteSharedU8:
case ValueOpcode::WriteSharedU16:
case ValueOpcode::WriteSharedU32:
@@ -110,18 +111,6 @@ bool HasSideEffects(ValueOpcode opcode) {
case ValueOpcode::GdsAtomicAnd32:
case ValueOpcode::GdsAtomicOr32:
case ValueOpcode::GdsAtomicXor32:
case ValueOpcode::BufferAtomicSwap32:
case ValueOpcode::BufferAtomicIAdd32:
case ValueOpcode::BufferAtomicISub32:
case ValueOpcode::BufferAtomicSMin32:
case ValueOpcode::BufferAtomicUMin32:
case ValueOpcode::BufferAtomicSMax32:
case ValueOpcode::BufferAtomicUMax32:
case ValueOpcode::BufferAtomicAnd32:
case ValueOpcode::BufferAtomicOr32:
case ValueOpcode::BufferAtomicXor32:
case ValueOpcode::BufferAtomicFMin32:
case ValueOpcode::BufferAtomicFMax32:
case ValueOpcode::ImageAtomicIAdd32:
case ValueOpcode::ImageAtomicUMin32:
case ValueOpcode::ImageAtomicUMax32:
@@ -139,4 +128,29 @@ bool HasSideEffects(ValueOpcode opcode) {
}
}
BufferAccess BufferAccessOf(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::ReadConstBuffer:
case ValueOpcode::LoadBufferU8:
case ValueOpcode::LoadBufferU16:
case ValueOpcode::LoadBufferU32: return BufferAccess::Read;
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32: return BufferAccess::Write;
case ValueOpcode::BufferAtomicSwap32:
case ValueOpcode::BufferAtomicIAdd32:
case ValueOpcode::BufferAtomicISub32:
case ValueOpcode::BufferAtomicSMin32:
case ValueOpcode::BufferAtomicUMin32:
case ValueOpcode::BufferAtomicSMax32:
case ValueOpcode::BufferAtomicUMax32:
case ValueOpcode::BufferAtomicAnd32:
case ValueOpcode::BufferAtomicOr32:
case ValueOpcode::BufferAtomicXor32:
case ValueOpcode::BufferAtomicFMin32:
case ValueOpcode::BufferAtomicFMax32: return BufferAccess::Atomic;
default: return BufferAccess::None;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h
+3
View File
@@ -16,6 +16,8 @@ enum class ValueOpcode {
Count,
};
enum class BufferAccess { None, Read, Write, Atomic };
struct DppMoveFlags {
uint16_t control = 0;
uint8_t row_mask = 0xf;
@@ -52,6 +54,7 @@ static_assert(std::is_trivially_copyable_v<ExportFlags>);
[[nodiscard]] Type ArgTypeOf(ValueOpcode opcode, size_t index);
[[nodiscard]] size_t NumArgsOf(ValueOpcode opcode);
[[nodiscard]] bool HasSideEffects(ValueOpcode opcode);
[[nodiscard]] BufferAccess BufferAccessOf(ValueOpcode opcode);
[[nodiscard]] std::string_view ValueOpcodeName(ValueOpcode opcode);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp
+7 -45
View File
@@ -25,46 +25,6 @@ const char* StageName(ShaderType stage) {
}
}
bool IsBufferAtomic(ValueOpcode op) {
switch (op) {
case ValueOpcode::BufferAtomicSwap32:
case ValueOpcode::BufferAtomicIAdd32:
case ValueOpcode::BufferAtomicISub32:
case ValueOpcode::BufferAtomicSMin32:
case ValueOpcode::BufferAtomicUMin32:
case ValueOpcode::BufferAtomicSMax32:
case ValueOpcode::BufferAtomicUMax32:
case ValueOpcode::BufferAtomicAnd32:
case ValueOpcode::BufferAtomicOr32:
case ValueOpcode::BufferAtomicXor32:
case ValueOpcode::BufferAtomicFMin32:
case ValueOpcode::BufferAtomicFMax32: return true;
default: return false;
}
}
bool IsBufferStore(ValueOpcode op) {
switch (op) {
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32: return true;
default: return IsBufferAtomic(op);
}
}
bool IsBuffer(ValueOpcode op) {
switch (op) {
case ValueOpcode::ReadConstBuffer:
case ValueOpcode::LoadBufferU8:
case ValueOpcode::LoadBufferU16:
case ValueOpcode::LoadBufferU32:
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32: return true;
default: return IsBufferAtomic(op);
}
}
bool IsAddressStore(ValueOpcode op) {
switch (op) {
case ValueOpcode::StoreAddressU8:
@@ -393,7 +353,8 @@ private:
for (const auto* block: m_values.blocks) {
for (const auto& inst: *block) {
const auto op = inst.GetOpcode();
if ((!IsBuffer(op) && !IsAddress(op) && !IsImage(op)) || &inst == &owner) {
if ((BufferAccessOf(op) == BufferAccess::None && !IsAddress(op) && !IsImage(op)) ||
&inst == &owner) {
continue;
}
if (inst.Flags<MemoryFlags>().index == index) {
@@ -669,8 +630,9 @@ private:
static void Merge(BufferResource& resource, const MemoryInfo& memory, ValueOpcode op,
uint32_t pc) {
const bool atomic = IsBufferAtomic(op);
const bool write = IsBufferStore(op);
const auto access = BufferAccessOf(op);
const bool atomic = access == BufferAccess::Atomic;
const bool write = access == BufferAccess::Write || atomic;
resource.first_use_pc = std::min(resource.first_use_pc, pc);
resource.max_byte_extent = std::max(resource.max_byte_extent, ByteExtent(memory));
resource.read = resource.read || !write || atomic;
@@ -813,7 +775,7 @@ private:
bool Collect(Inst& inst, std::string* error) {
const auto op = inst.GetOpcode();
if (!IsBuffer(op) && !IsAddress(op) && !IsImage(op)) {
if (BufferAccessOf(op) == BufferAccess::None && !IsAddress(op) && !IsImage(op)) {
return true;
}
const auto flags = inst.Flags<MemoryFlags>();
@@ -832,7 +794,7 @@ private:
uint32_t source = 0;
uint32_t resource = 0;
if (IsBuffer(op)) {
if (BufferAccessOf(op) != BufferAccess::None) {
if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle, source,
error)) {
return false;