raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 02:47:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

shader: share bounded record descriptors for packed loads and byte stores

Browse Source 
Co-authored-by: fxpw <fxpw@outlook.com>
This commit is contained in:
nmzik
2026-10-14 05:07:24 +02:00
parent 4cf5a96b79
commit 26d326d1b0
12 changed files with 581 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
+5
View File
@@ -105,6 +105,11 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
Fail(program, "typed buffer handle has an invalid dense resource");
}
break;
case IR::ValueOpcode::GetBufferResourceIndex:
if (dense >= program.info.buffers.size()) {
Fail(program, "typed buffer selector has an invalid dense resource");
}
break;
case IR::ValueOpcode::GetAddressResource:
if (planning_only_handle(inst)) {
break;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h
+1
View File
@@ -246,6 +246,7 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& in
uint32_t EmitGetShaderBase(ValueEmitContext& ctx);
inline constexpr auto EmitGetSrtResource = EmitVoid;
inline constexpr auto EmitGetBufferResource = EmitGetSrtResource;
uint32_t EmitGetBufferResourceIndex(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitGetAddressResource = EmitGetSrtResource;
inline constexpr auto EmitGetScratchResource = EmitGetSrtResource;
inline constexpr auto EmitGetImageResource = EmitGetSrtResource;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+3 -2
View File
@@ -399,7 +399,7 @@ uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryIn
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
bool UsesSharedIndirectBufferStore(const EmitterState& state, uint32_t resource);
bool UsesSharedIndirectBufferAccess(const EmitterState& state, uint32_t resource);
void EmitMemoryOffsets(EmitterState& state);
@@ -418,7 +418,8 @@ struct MemoryResourceAccess {
uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
uint32_t search_iterations, uint32_t default_resource);
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t array_index = 0u);
MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
const IR::MemoryInfo& mem,
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+122 -85
View File
@@ -286,6 +286,85 @@ uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& res
uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
uint32_t address, uint32_t index, uint32_t bits, bool sign_extend);
template <typename Fn>
auto EmitBufferAccess(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem,
uint32_t components, Fn&& emit) {
auto& state = ctx.state;
constexpr bool has_result = !std::is_void_v<
std::invoke_result_t<Fn, const IR::MemoryInfo&, const MemoryResourceAccess&>>;
const auto access = [&](uint32_t resource) {
mem.resource = resource;
if (mem.kind == IR::ResourceKind::Buffer) {
const auto& buffer = state.program.info.buffers[resource];
if (buffer.packed_stride == 0u &&
buffer.source < state.program.descriptor_sources.size() &&
state.program.descriptor_sources[buffer.source].indirect_descriptor) {
// Empty candidates still carry the format/swizzle needed for OOB loads.
return emit(mem, MemoryResourceAccess{});
}
}
return emit(mem, PrepareMemoryResourceAccess(state, mem));
};
if (mem.kind != IR::ResourceKind::Buffer ||
state.program.info.buffers[mem.resource].indirect_root != mem.resource) {
return access(mem.resource);
}
const auto& buffer = state.program.info.buffers[mem.resource];
const auto& handle = *inst.Arg(0).ResolveInstruction();
const auto selected = ctx.Arg(handle, 0);
const auto shared = UsesSharedIndirectBufferAccess(state, mem.resource);
const auto selected_access = [&]() {
if (shared) {
const auto root_slot = buffer.descriptor_index;
const auto child_slot = state.program.info.buffers[buffer.indirect_resources[1]].descriptor_index;
auto slot = EmitAddU32(state, selected, ConstantU32(state, child_slot - 1u));
if (child_slot != root_slot + 1u) {
slot = Select(state, TypeU32(state),
Binary(state, spv::OpIEqual, TypeBool(state), selected, ConstantU32(state, 0u)),
ConstantU32(state, root_slot), slot);
}
return emit(mem, PrepareMemoryResourceAccess(state, mem, slot));
}
return EmitIndexSwitch(state, selected, buffer.indirect_resources.size(),
has_result ? TypeU32Composite(state, components) : 0u,
[&](uint32_t item) { return access(buffer.indirect_resources[item]); });
};
if constexpr (!has_result) {
if (!shared) return selected_access();
}
const auto valid = Binary(state, spv::OpULessThan, TypeBool(state), selected,
ConstantU32(state, buffer.indirect_resources.size()));
if constexpr (has_result) {
return EmitValueOrDefaultIfCondition(state, valid, TypeU32Composite(state, components),
ConstantU32CompositeZero(state, components), selected_access);
} else {
EmitIfCondition(state, valid, selected_access);
}
}
uint32_t BufferIndexInBounds(ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem, const MemoryResourceAccess& resource,
uint32_t payload_in_bounds) {
if (mem.kind != IR::ResourceKind::Buffer) return payload_in_bounds;
auto& state = ctx.state;
const auto& buffer = state.program.info.buffers[mem.resource];
const auto& sources = state.program.descriptor_sources;
if (buffer.source >= sources.size() || !sources[buffer.source].indirect_descriptor ||
!sources[buffer.source].indirect_descriptor->indexed) return payload_in_bounds;
const auto bytes = Binary(state, spv::OpIMul, TypeU32(state), resource.length,
ConstantU32(state, resource.element_bits / 8u));
const auto available = Binary(state, spv::OpISub, TypeU32(state), bytes, resource.byte_offset);
// Test the original index before multiplication can wrap. A truncated binding
// may omit final-record padding; the payload check below still checks every byte.
const auto last_record = Binary(state, spv::OpUDiv, TypeU32(state),
Binary(state, spv::OpISub, TypeU32(state), available, ConstantU32(state, 1u)),
ConstantU32(state, buffer.packed_stride & 0x3fffu));
return AndCondition(state, payload_in_bounds,
AndCondition(state,
Binary(state, spv::OpUGreaterThan, TypeBool(state), bytes, resource.byte_offset),
Binary(state, spv::OpULessThanEqual, TypeBool(state), ctx.Arg(inst, 1), last_record)));
}
uint32_t LoadWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource) {
const auto index = DwordIndex(ctx, inst, mem, resource.byte_offset);
@@ -466,20 +545,23 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
void StoreSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t bits, uint32_t data) {
if (resource.object_pointer == 0u) return;
const auto address = ByteAddress(ctx, inst, mem, resource.byte_offset);
const auto index = Binary(
ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
ctx.state, BufferIndexInBounds(ctx, inst, mem, resource,
EmitMemoryElementInBounds(ctx.state, resource, index)), [&]() {
StoreSubwordInBounds(ctx, mem, resource, address, index, bits, data);
});
}
void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits) {
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
StoreSubwordPrepared(ctx, inst, mem, resource, bits, ctx.Arg(inst, inst.NumArgs() - 2));
EmitBufferAccess(ctx, inst, mem, 0u, [&](const auto& selected, const auto& resource) {
StoreSubwordPrepared(ctx, inst, selected, resource, bits, ctx.Arg(inst, inst.NumArgs() - 2));
});
});
}
@@ -536,75 +618,20 @@ void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t data) {
auto& state = ctx.state;
const auto source = mem.kind == IR::ResourceKind::Buffer
? state.program.info.buffers[mem.resource].source : UINT32_MAX;
const auto& sources = state.program.descriptor_sources;
const auto* indirect = source < sources.size() && sources[source].indirect_descriptor
? &*sources[source].indirect_descriptor : nullptr;
uint32_t index = 0;
uint32_t in_bounds = 0;
if (indirect != nullptr && indirect->indexed_store) {
// The resource plan proves stride-four, index-only stores. Check the original
// index before any byte multiplication can wrap it into the buffer.
const auto prefix = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
resource.byte_offset, ConstantU32(state, 2));
const auto guest_index = ctx.Arg(inst, 1);
const auto records = Binary(state, spv::OpISub, TypeU32(state), resource.length, prefix);
in_bounds = AndCondition(state,
Binary(state, spv::OpULessThanEqual, TypeBool(state), prefix, resource.length),
Binary(state, spv::OpULessThan, TypeBool(state), guest_index, records));
index = EmitAddU32(state, guest_index, prefix);
} else {
index = DwordIndex(ctx, inst, mem, resource.byte_offset);
in_bounds = EmitMemoryElementInBounds(state, resource, index);
}
EmitIfCondition(state, in_bounds, [&]() {
if (resource.object_pointer == 0u) return;
const auto index = DwordIndex(ctx, inst, mem, resource.byte_offset);
const auto in_bounds = BufferIndexInBounds(ctx, inst, mem, resource,
EmitMemoryElementInBounds(ctx.state, resource, index));
EmitIfCondition(ctx.state, in_bounds, [&]() {
StoreWordInBounds(ctx, resource, index, data);
});
}
void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) {
auto& state = ctx.state;
EmitIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto data = ctx.Arg(inst, inst.NumArgs() - 2);
const auto store = [&](uint32_t resource) {
if (mem.kind == IR::ResourceKind::Buffer) {
const auto& buffer = state.program.info.buffers[resource];
if (buffer.packed_stride == 0u &&
buffer.source < state.program.descriptor_sources.size() &&
state.program.descriptor_sources[buffer.source].indirect_descriptor) return;
}
mem.resource = resource;
const auto access = PrepareMemoryResourceAccess(state, mem);
StoreWordPrepared(ctx, inst, mem, access, data);
};
if (mem.kind != IR::ResourceKind::Buffer ||
state.program.info.buffers[mem.resource].indirect_root != mem.resource) {
store(mem.resource);
return;
}
const auto& buffer = state.program.info.buffers[mem.resource];
const auto* handle = inst.Arg(0).ResolveInstruction();
if (handle == nullptr || buffer.indirect_resources.empty() ||
state.flattened_srt_variable == 0 || buffer.indirect_search_iterations == 0u) {
ctx.Fail(inst, "has no indirect buffer runtime mapping");
return;
}
const auto selected = EmitIndirectResourceIndex(
state, ctx.Arg(*handle, 0), buffer.indirect_mapping_offset,
buffer.indirect_search_iterations, UINT32_MAX);
if (UsesSharedIndirectBufferStore(state, mem.resource)) {
EmitIfCondition(state, Binary(state, spv::OpULessThan, TypeBool(state), selected,
ConstantU32(state, buffer.indirect_resources.size())), [&]() {
const auto slot = EmitAddU32(state, selected, ConstantU32(state, buffer.descriptor_index));
const auto access = PrepareStorageBufferResourceAccess(state, mem, state.storage_buffers[2], slot);
StoreWordPrepared(ctx, inst, mem, access, data);
});
return;
}
EmitIndexSwitch(state, selected, buffer.indirect_resources.size(), 0u,
[&](uint32_t item) { store(buffer.indirect_resources[item]); });
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
EmitBufferAccess(ctx, inst, mem, 0u, [&](const auto& selected, const auto& resource) {
StoreWordPrepared(ctx, inst, selected, resource, ctx.Arg(inst, inst.NumArgs() - 2));
});
});
}
@@ -673,9 +700,9 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::
ConstantU32(ctx.state, info.byte_size - 1u));
const auto last_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), end,
ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
plan.in_bounds = AndCondition(
plan.in_bounds = BufferIndexInBounds(ctx, inst, mem, resource, AndCondition(
ctx.state, Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), end, start),
EmitMemoryElementInBounds(ctx.state, plan.resource, last_index));
EmitMemoryElementInBounds(ctx.state, plan.resource, last_index)));
plan.base = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), start,
ConstantU32(ctx.state, ~(std::min(4u, info.byte_size) - 1u)));
return plan;
@@ -732,10 +759,11 @@ uint32_t LoadFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst,
const Format::BufferFormatInfo& info, uint32_t components) {
auto& state = ctx.state;
const auto count = mem.data_bits == 16u ? mem.component_count : components;
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info);
const auto value = EmitValueOrDefaultIfCondition(
state, plan.in_bounds, TypeU32Composite(state, count),
FormattedOutOfBoundsValue(ctx, mem, plan, count), [&]() {
const auto plan = resource.object_pointer == 0u ? PreparedFormattedMemory{.info = info}
: PrepareFormattedMemory(ctx, inst, mem, resource, info);
const auto out_of_bounds = FormattedOutOfBoundsValue(ctx, mem, plan, count);
const auto value = resource.object_pointer == 0u ? out_of_bounds : EmitValueOrDefaultIfCondition(
state, plan.in_bounds, TypeU32Composite(state, count), out_of_bounds, [&]() {
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < count; component++) {
values[component] = LoadFormattedInBounds(ctx, mem, plan, component);
@@ -1078,18 +1106,20 @@ uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compon
if (mem.kind == IR::ResourceKind::IndirectBuffer) {
return LoadIndirectBuffer(ctx, inst, components);
}
const auto resource = PrepareMemoryResourceAccess(state, mem);
const auto info = Format::GetFormatInfo(
mem.formatted ? BufferFormat(ctx, mem) : Prospero::BufferFormat::kInvalid);
if (info.type != Format::ComponentType::Unknown) {
return LoadFormattedPrepared(ctx, inst, mem, resource, info, components);
}
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; component++) {
values[component] =
LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource);
}
return ConstructU32Composite(state, components, values);
return EmitBufferAccess(ctx, inst, mem, components, [&](const auto& selected, const auto& resource) {
const auto info = Format::GetFormatInfo(
selected.formatted ? BufferFormat(ctx, selected) : Prospero::BufferFormat::kInvalid);
if (info.type != Format::ComponentType::Unknown) {
return LoadFormattedPrepared(ctx, inst, selected, resource, info, components);
}
if (resource.object_pointer == 0u) return ConstantU32CompositeZero(state, components);
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; component++) {
values[component] =
LoadWordPrepared(ctx, inst, RebaseRawComponent(selected, component), resource);
}
return ConstructU32Composite(state, components, values);
});
});
}
@@ -1431,6 +1461,13 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
}));
}
uint32_t EmitGetBufferResourceIndex(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto& buffer = ctx.state.program.info.buffers.at(inst.Flags<uint32_t>());
if (buffer.indirect_resources.empty()) return ConstantU32(ctx.state, 0u);
return EmitIndirectResourceIndex(ctx.state, ctx.Arg(inst, 0), buffer.indirect_mapping_offset,
buffer.indirect_search_iterations, UINT32_MAX);
}
void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto op = inst.GetOpcode();
const auto& mem = ctx.Memory(inst);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+15 -9
View File
@@ -108,17 +108,22 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
return state.program.info.buffers[mem.resource].descriptor_format;
}
bool UsesSharedIndirectBufferStore(const EmitterState& state, uint32_t resource) {
bool UsesSharedIndirectBufferAccess(const EmitterState& state, uint32_t resource) {
const auto& buffers = state.program.info.buffers;
const auto& root = buffers[resource];
if (root.indirect_root != resource || root.descriptor_index == UINT32_MAX ||
root.packed_stride == 0u || root.indirect_resources.empty()) {
root.packed_stride == 0u || root.indirect_resources.size() < 2u) {
return false;
}
uint32_t slot = root.descriptor_index;
for (const auto candidate: root.indirect_resources) {
if (buffers[candidate].packed_stride != root.packed_stride ||
buffers[candidate].descriptor_index != slot++) return false;
uint32_t next_child_slot = UINT32_MAX;
for (uint32_t ordinal = 0; ordinal < root.indirect_resources.size(); ++ordinal) {
const auto& candidate = buffers[root.indirect_resources[ordinal]];
if (candidate.packed_stride != root.packed_stride ||
candidate.descriptor_format != root.descriptor_format ||
candidate.descriptor_swizzle != root.descriptor_swizzle ||
candidate.descriptor_index == UINT32_MAX ||
(ordinal > 1u && candidate.descriptor_index != next_child_slot)) return false;
if (ordinal != 0u) next_child_slot = candidate.descriptor_index + 1u;
}
return true;
}
@@ -126,7 +131,7 @@ bool UsesSharedIndirectBufferStore(const EmitterState& state, uint32_t resource)
void EmitMemoryOffsets(EmitterState& state) {
std::array<bool, IR::ShaderInfo::MaxBuffers> dynamic_offsets {};
for (uint32_t resource = 0; resource < state.program.info.buffers.size(); ++resource) {
if (!UsesSharedIndirectBufferStore(state, resource)) continue;
if (!UsesSharedIndirectBufferAccess(state, resource)) continue;
for (const auto candidate: state.program.info.buffers[resource].indirect_resources) {
dynamic_offsets[state.program.info.buffers[candidate].descriptor_index] = true;
}
@@ -227,7 +232,8 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
return access;
}
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem) {
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t array_index) {
MemoryResourceAccess access {.kind = mem.kind};
switch (mem.kind) {
case IR::ResourceKind::Lds:
@@ -267,7 +273,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
} else if (bits == 16u) {
buffer = &state.storage_buffers[1];
}
access = PrepareStorageBufferResourceAccess(state, mem, *buffer);
access = PrepareStorageBufferResourceAccess(state, mem, *buffer, array_index);
access.element_bits = bits;
return access;
}
src/graphics/shader/recompiler/ir/ShaderIR.h
+1 -1
View File
@@ -537,7 +537,7 @@ struct DescriptorSource {
uint32_t table_stride = 0;
uint32_t table_record_bytes = 0;
bool table_scalar = false;
bool indexed_store = false;
bool indexed = false;
bool signed_count = false;
uint32_t workgroup_axis = UINT32_MAX;
Value key_count;
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc
+1
View File
@@ -235,6 +235,7 @@ VALUE_OPCODE(BpermuteU32, U32, U32, U32, U1)
// dense bindings are assigned by resource tracking.
VALUE_OPCODE(GetSrtResource, SrtResource)
VALUE_OPCODE(GetBufferResource, BufferResource, U32, U32, U32, U32)
VALUE_OPCODE(GetBufferResourceIndex, U32, U32)
VALUE_OPCODE(GetAddressResource, AddressResource, U32, U32)
VALUE_OPCODE(GetScratchResource, AddressResource)
VALUE_OPCODE(GetImageResource, ImageResource, U32, U32, U32, U32, U32, U32, U32, U32)
src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp
+34 -10
View File
@@ -227,16 +227,38 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t offset,
return ReadTable(base & AddressMask & ~uint64_t {3}, size, offset & ~uint64_t {3}, runtime, words);
}
bool NormalizeIndirectStoreBuffer(DescriptorValue& value, bool indexed, bool formatted) {
bool NormalizeIndirectBuffer(DescriptorValue& value, bool indexed, const BufferResource& resource) {
ShaderBufferResource descriptor;
if (!DecodeBufferDescriptor(value, descriptor)) return false;
const bool formatted_read = resource.formatted && resource.read && !resource.written;
if (descriptor.Type() != 0u || descriptor.Format() == Prospero::BufferFormat::kInvalid ||
descriptor.Base48() == 0u || descriptor.GetSize() == 0u) {
(!formatted_read && (descriptor.Base48() == 0u || descriptor.GetSize() == 0u))) {
value.dwords.fill(0u);
} else if (descriptor.Stride() != (indexed ? 4u : 1u) || descriptor.SwizzleEnabled() ||
descriptor.AddTid() || descriptor.OutOfBounds() != (indexed ? 0u : 3u)) {
return true;
}
if (descriptor.SwizzleEnabled() || descriptor.AddTid() ||
descriptor.OutOfBounds() != (indexed ? 0u : 3u)) {
return SpecializationFail("indirect buffers require exact linear index or byte bounds");
}
if (formatted_read) {
const auto format = Format::GetFormatInfo(descriptor.Format());
if (!indexed || format.type == Format::ComponentType::Unknown)
return SpecializationFail("indirect formatted loads require a supported indexed format");
if (descriptor.Base48() == 0u || descriptor.Stride() == 0u || descriptor.NumRecords() == 0u) {
// Every index is out of bounds. Preserve the format and swizzle constants.
value.dwords[1] &= ~(0x3fffu << 16u);
value.dwords[2] = 0u;
return true;
}
const auto alignment = std::min(4u, format.byte_size);
if (descriptor.Stride() < format.byte_size ||
(descriptor.Stride() & (alignment - 1u)) != 0u ||
(descriptor.Base48() & (alignment - 1u)) != 0u)
return SpecializationFail("indirect formatted loads require aligned full-format records");
} else if ((!indexed && descriptor.Stride() != 1u) ||
(indexed && resource.max_byte_extent != 1u && descriptor.Stride() != 4u)) {
return SpecializationFail("indirect buffer stores require exact linear DWORD or byte bounds");
} else if (formatted && (descriptor.Format() != Prospero::BufferFormat::k32UInt ||
} else if (resource.formatted && (descriptor.Format() != Prospero::BufferFormat::k32UInt ||
descriptor.DstSelXYZW() != DstSel(4, 0, 0, 0) ||
(descriptor.Base48() & 3u) != 0u)) {
return SpecializationFail("indirect formatted X stores require an aligned 32-bit UInt identity mapping");
@@ -248,7 +270,8 @@ bool IsBoundedDescriptorTable(const ResourcePlan& program,
const DescriptorSource::IndirectDescriptor& indirect) {
const auto* table = Source(program, indirect.table_source);
return indirect.sources.empty() && !indirect.selector &&
indirect.workgroup_axis == UINT32_MAX && table != nullptr && table->dword_count == 4u;
indirect.key_count.IsEmpty() && indirect.workgroup_axis == UINT32_MAX &&
table != nullptr && table->dword_count == 4u;
}
template <typename Specialization, typename Normalize>
@@ -372,7 +395,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
key_count = runtime.workgroup_counts[indirect.workgroup_axis];
if (key_count == 0u) return false;
} else {
if (table_value.dword_count != 2u || !evaluate_count(key_count))
if (!evaluate_count(key_count))
return false;
}
if (!indirect.selector_first.IsEmpty() && !clean.Evaluate(indirect.selector_first, first_key))
@@ -1308,8 +1331,8 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
program, *source->indirect_descriptor, i, 4u, observed, clean, snapshot,
snapshot.buffers, specialization.buffers, ShaderInfo::MaxBuffers,
[&](DescriptorValue& value) {
return NormalizeIndirectStoreBuffer(
value, source->indirect_descriptor->indexed_store, base.formatted);
return NormalizeIndirectBuffer(
value, source->indirect_descriptor->indexed, base);
}))
return false;
}
@@ -1526,7 +1549,8 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
if (BufferAccessOf(inst.GetOpcode()) == BufferAccess::Read) {
const auto& memory = memory_info[inst.Flags<MemoryFlags>().index];
if (memory.kind == ResourceKind::Buffer &&
specialization.buffers[memory.resource].zero_stride_oob) {
specialization.buffers[memory.resource].zero_stride_oob &&
buffers[memory.resource].indirect_root == BufferResource::NoIndirectBuffer) {
// Bounds mode 0 checks offset >= stride, so zero stride
// makes every vector read out of bounds regardless of its address.
const auto count = BufferComponentCount(inst.GetOpcode());
src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp
+80 -26
View File
@@ -394,6 +394,7 @@ public:
memory.sampler = patch.sampler;
}
}
std::vector<std::pair<const IndirectDescriptorPlan*, Value>> buffer_selections;
for (const auto& plan: m_indirect_descriptors) {
auto key = plan.key;
if (plan.table_read != nullptr) {
@@ -452,8 +453,32 @@ public:
plan.handle->SetArg(word, Value(0u));
continue;
}
if (plan.handle->NumArgs() == 4u) {
plan.handle->SetArg(0, plan.key);
if (plan.handle->GetOpcode() == ValueOpcode::GetBufferResource) {
Value selected;
for (const auto& [previous, value]: buffer_selections) {
if (plan.selection_anchor != nullptr && previous->reads == plan.reads &&
previous->source == plan.source &&
previous->handle->Flags<uint32_t>() == plan.handle->Flags<uint32_t>() &&
EquivalentValue(m_program, previous->key, plan.key) &&
AvailableBefore(*value.Instruction(), *plan.handle)) {
selected = value;
break;
}
}
if (selected.IsEmpty()) {
const auto* anchor = plan.selection_anchor != nullptr ? plan.selection_anchor : plan.handle;
auto* block = anchor->Parent();
const auto where = std::ranges::find_if(block->Instructions(),
[&](const Inst& inst) { return &inst == anchor; });
auto& selection = *block->PrependNewInst(where, ValueOpcode::GetBufferResourceIndex, {plan.key});
selection.SetFlags(plan.handle->Flags<uint32_t>());
selected = Value(&selection);
buffer_selections.emplace_back(&plan, selected);
}
plan.handle->SetArg(0, selected);
for (uint32_t word = 1; word < 4u; ++word) plan.handle->SetArg(word, Value(0u));
} else if (plan.handle->NumArgs() == 4u) {
plan.handle->SetArg(0, key);
for (uint32_t word = 1; word < 4u; ++word) plan.handle->SetArg(word, Value(0u));
} else {
if (plan.reads[0] == nullptr) plan.handle->SetArg(0, key);
@@ -519,6 +544,7 @@ private:
bool retain_reads = false;
bool table_indexed = false;
const Inst* table_read = nullptr;
const Inst* selection_anchor = nullptr;
};
[[noreturn]] void Fail(uint32_t pc, const std::string& reason) const {
@@ -931,7 +957,7 @@ private:
const auto& b = *descriptor.indirect_descriptor;
if (a.selector != b.selector || a.table_source != b.table_source ||
a.table_offset != b.table_offset || a.table_immediate != b.table_immediate ||
a.table_stride != b.table_stride || a.indexed_store != b.indexed_store ||
a.table_stride != b.table_stride || a.indexed != b.indexed ||
a.workgroup_axis != b.workgroup_axis || a.sources != b.sources ||
a.signed_count != b.signed_count ||
!EquivalentValue(m_program, a.key_count, b.key_count) ||
@@ -2522,6 +2548,25 @@ private:
return true;
}
bool IndirectBufferAccess(const Inst& inst) const {
const auto opcode = inst.GetOpcode();
if (opcode != ValueOpcode::StoreBufferU32 && opcode != ValueOpcode::StoreBufferU8 &&
opcode != ValueOpcode::LoadBufferU32x2) return false;
const auto index = inst.Flags<MemoryFlags>().index;
if (index >= m_program.memory_info.size()) return false;
const auto& memory = m_program.memory_info[index];
if (memory.kind != ResourceKind::Buffer || memory.typed || memory.idxen == memory.offen ||
(memory.idxen && (memory.offset != 0u || inst.Arg(2).Resolve() != Value(0u))) ||
inst.Arg(3).Resolve() != Value(0u)) return false;
if (opcode == ValueOpcode::StoreBufferU32)
return !memory.formatted || (memory.idxen && memory.data_bits == 32u && memory.data_dwords == 1u);
if (!memory.idxen) return false;
if (opcode == ValueOpcode::StoreBufferU8)
return !memory.formatted && memory.data_bits == 8u;
return memory.formatted && memory.data_bits == 16u && memory.data_dwords == 2u &&
memory.component_count == 4u;
}
bool MatchDescriptorTable(Inst& handle, const DescriptorSource& descriptor,
IndirectDescriptorPlan& plan,
DescriptorSource& table_source, Value& key, uint32_t& table_offset,
@@ -2599,10 +2644,10 @@ private:
if (handle.NumArgs() == 8u) {
if (memory->kind == ResourceKind::ScalarAddress)
plan.retain_reads |= !UsesOnlyImageDescriptors(*read);
} else if (std::ranges::any_of(read->Uses(), [](const Use& use) {
} else if (std::ranges::any_of(read->Uses(), [&](const Use& use) {
return use.user->GetOpcode() != ValueOpcode::GetBufferResource ||
std::ranges::any_of(use.user->Uses(), [](const Use& consumer) {
return consumer.user->GetOpcode() != ValueOpcode::StoreBufferU32;
std::ranges::any_of(use.user->Uses(), [&](const Use& consumer) {
return !IndirectBufferAccess(*consumer.user);
});
})) {
return false;
@@ -2780,19 +2825,14 @@ private:
IndirectDescriptorPlan& plan) {
if (handle.GetOpcode() != ValueOpcode::GetBufferResource || handle.NumArgs() != 4u ||
handle.Uses().empty()) return false;
if (handle.Uses().front().user->GetOpcode() != ValueOpcode::StoreBufferU32) return false;
if (!IndirectBufferAccess(*handle.Uses().front().user)) return false;
const bool legacy_store = handle.Uses().front().user->GetOpcode() == ValueOpcode::StoreBufferU32;
const auto indexed = m_program.memory_info[handle.Uses().front().user->Flags<MemoryFlags>().index].idxen;
for (const auto& use: handle.Uses()) {
const auto& inst = *use.user;
if (inst.GetOpcode() != ValueOpcode::StoreBufferU32) return false;
if (!IndirectBufferAccess(inst)) return false;
const auto& memory = m_program.memory_info[inst.Flags<MemoryFlags>().index];
uint32_t scalar = 1u;
if (memory.kind != ResourceKind::Buffer || memory.typed ||
(memory.formatted && (!indexed || memory.data_bits != 32u || memory.data_dwords != 1u)) ||
memory.idxen != indexed || memory.offen == indexed ||
(indexed && (memory.offset != 0u || inst.Arg(2).Resolve() != Value(0u))) ||
!ImmediateU32(inst.Arg(3), scalar) || scalar != 0u)
return false;
if (memory.idxen != indexed) return false;
}
DescriptorSource table_source;
Value key;
@@ -2800,12 +2840,25 @@ private:
uint32_t table_stride = 0;
uint32_t table_immediate = 0;
if (!MatchDescriptorTable(handle, descriptor, plan, table_source, key, table_offset,
table_stride, table_immediate) || table_stride != 16u) return false;
table_stride, table_immediate) ||
table_stride < 16u || (table_stride & 3u) != 0u) return false;
DescriptorSource::IndirectDescriptor indirect;
indirect.table_stride = table_stride;
indirect.table_offset = table_offset;
indirect.table_immediate = table_immediate;
indirect.indexed_store = indexed;
indirect.indexed = indexed;
for (uint32_t word = 0; word < 4u; ++word) {
const auto* anchor = plan.reads[word];
const auto* key_inst = key.Resolve().TryInstruction();
if (AvailableBefore(*anchor, handle) &&
(key_inst == nullptr || (key_inst != anchor && AvailableBefore(*key_inst, *anchor))) &&
std::all_of(plan.reads.begin(), plan.reads.begin() + 4u,
[&](const Inst* read) { return AvailableBefore(*read, *anchor); })) {
plan.selection_anchor = anchor;
break;
}
}
if (!legacy_store && plan.selection_anchor == nullptr) return false;
const auto finish = [&] {
indirect.table_source = InternSource(table_source);
DescriptorSource source;
@@ -2817,19 +2870,20 @@ private:
plan.key = key;
return true;
};
if (table_source.dword_count == 2u && MatchPackedScalarKey(key, indirect)) return finish();
if (const auto* bound = BoundedLoop(key, handle.Parent()); bound != nullptr) {
indirect.key_count = bound->Arg(1);
indirect.signed_count = bound->GetOpcode() == ValueOpcode::SLessThan32;
return finish();
}
if (!legacy_store || table_stride != 16u) return false;
if (table_source.dword_count == 2u) {
const auto* bound = BoundedLoop(key, handle.Parent());
if (bound != nullptr) {
indirect.key_count = bound->Arg(1);
indirect.signed_count = bound->GetOpcode() == ValueOpcode::SLessThan32;
} else {
if (!MatchPackedScalarKey(key, indirect)) {
// EXEC implications must describe the same iteration as the selected value.
if (m_native_cfg.irreducible || !m_native_cfg.back_edges.empty()) return false;
const auto* lane = key.Resolve().TryInstruction();
if (lane == nullptr || lane->GetOpcode() != ValueOpcode::ReadFirstLane ||
!std::ranges::all_of(handle.Uses(), [&](const Use& use) {
return Implies(use.user->Arg(5), lane->Arg(1));
return Implies(use.user->Arg(use.user->NumArgs() - 1u), lane->Arg(1));
})) return false;
const auto maximum = SelectorMaximum(lane->Arg(0), {}, 0u, lane->Arg(1));
if (maximum >= MaxIndirectDescriptorProbes) return false;
@@ -3112,7 +3166,7 @@ private:
for (auto* block: m_program.blocks) {
for (auto& inst: *block) {
if ((ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None &&
inst.GetOpcode() != ValueOpcode::StoreBufferU32) ||
!IndirectBufferAccess(inst)) ||
inst.NumArgs() == 0u) {
continue;
}
@@ -3138,7 +3192,7 @@ private:
m_indirect_descriptors.push_back(std::move(plan));
}
};
plan_handle(inst.Arg(0), inst.GetOpcode() == ValueOpcode::StoreBufferU32 ? 4u : 8u,
plan_handle(inst.Arg(0), IndirectBufferAccess(inst) ? 4u : 8u,
memory.resource * 4u, false);
if (inst.GetOpcode() == ValueOpcode::ImageSampleRaw &&
(memory.image_sample_flags & Decoder::ImageSampleFlagAdjust) == 0u)
src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp
+10 -3
View File
@@ -343,9 +343,16 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
if (memory.resource >= info.buffers.size()) {
return Fail("buffer operation has invalid resource metadata");
}
const auto bits = StorageBufferElementBits(program, memory);
info.buffer_u8 |= bits == 8u;
info.buffer_u16 |= bits == 16u;
const auto collect_element = [&](uint32_t resource) {
auto selected = memory;
selected.resource = resource;
const auto bits = StorageBufferElementBits(program, selected);
info.buffer_u8 |= bits == 8u;
info.buffer_u16 |= bits == 16u;
};
collect_element(memory.resource);
for (const auto child: info.buffers[memory.resource].indirect_resources)
collect_element(child);
if ((info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
if (program.stage != ShaderType::Compute) {
return Fail("buffer ADD_TID is only valid for compute shaders");
tests/ResourceTrackingTests.cpp
+174 -4
View File
@@ -4293,7 +4293,7 @@ void TestBoundedFormattedBufferStore() {
auto fixture = make();
auto plan = ExtractResourcePlan(fixture->program);
const auto& source = plan.descriptor_sources[plan.info.buffers[0].source];
Check(source.indirect_descriptor && source.indirect_descriptor->indexed_store &&
Check(source.indirect_descriptor && source.indirect_descriptor->indexed &&
!source.indirect_descriptor->signed_count &&
!source.indirect_descriptor->selector &&
!source.indirect_descriptor->key_count.IsEmpty() &&
@@ -4393,6 +4393,175 @@ void TestBoundedFormattedBufferStore() {
"finite store accepted an unproved induction or indexed byte offset");
}
void TestBoundedBufferRecords() {
namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
namespace Prospero = Libs::Graphics::Prospero;
using Libs::Graphics::DstSel;
const auto make = [](uint32_t step = 1u, bool bypass = false) {
auto fixture = std::make_unique<Fixture>();
auto& f = *fixture;
auto* entry = f.block;
auto* header = f.AddBlock();
auto* body = f.AddBlock();
auto* latch = f.AddBlock();
auto* exit = f.AddBlock();
const auto branch = [](Block* from, Block* to) {
from->AddBranch(to);
from->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = to};
};
branch(entry, header);
branch(body, latch);
branch(latch, header);
header->AddBranch(body);
header->AddBranch(bypass ? latch : exit);
header->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = body, .false_block = bypass ? latch : exit};
exit->terminator.kind = CFG::TerminatorKind::Return;
std::array<Value, 4> table_words;
for (uint32_t word = 0; word < 4u; ++word) table_words[word] = f.UserData(word);
const auto table = f.Buffer(table_words);
const auto count = f.Emit(ValueOpcode::LoadAddressU32,
{f.Address(f.UserData(4u), f.UserData(5u)), Value(0u), Value(0u), Value(true)},
f.AddMemory({.kind = ResourceKind::ScalarAddress}, 0x424u));
const auto lane = f.Emit(ValueOpcode::LaneId);
auto& phi = header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
phi.AddPhiOperand(entry, Value(0u));
phi.AddPhiOperand(latch, f.Emit(ValueOpcode::IAdd32, {Value(&phi), Value(step)}, 0, latch));
header->condition = f.Emit(ValueOpcode::SLessThan32, {Value(&phi), count}, 0, header);
f.block = body;
const auto offset = f.Emit(ValueOpcode::IMul32, {Value(&phi), Value(192u)});
for (const bool formatted : {true, false}) {
std::array<Value, 4> words;
for (uint32_t word = 0; word < 4u; ++word)
words[word] = f.Emit(ValueOpcode::ReadConstBuffer, {table, offset},
f.AddMemory({.kind = ResourceKind::ScalarBuffer,
.offset = (formatted ? 160u : 176u) + word * 4u,
.component_index = word, .component_count = 4u}, 0x1808u));
for (uint32_t use = 0; use < (formatted ? 112u : 16u); ++use) {
const auto handle = f.Buffer(words);
if (formatted) {
const auto loaded = f.Emit(ValueOpcode::LoadBufferU32x2,
{handle, lane, Value(0u), Value(0u), Value(true)},
f.AddMemory({.kind = ResourceKind::Buffer, .data_dwords = 2u, .data_bits = 16u,
.component_count = 4u, .formatted = true, .idxen = true}, 0x1914u));
f.Emit(ValueOpcode::ReferenceU32,
{f.Emit(ValueOpcode::CompositeExtractU32x2, {loaded, Value(0u)})});
} else {
f.Emit(ValueOpcode::StoreBufferU8,
{handle, lane, Value(0u), Value(0u), Value(uint8_t{42u}), Value(true)},
f.AddMemory({.kind = ResourceKind::Buffer, .data_bits = 8u, .idxen = true}, 0x190cu));
}
}
}
f.PlanAndTrack();
return fixture;
};
auto fixture = make();
const auto plan = ExtractResourcePlan(fixture->program);
Check(plan.info.buffers.size() == 2u, "record descriptors did not share their resource roots");
uint32_t selections = 0u;
std::array<const Inst*, 2> selected{};
for (const auto* block : fixture->program.blocks) {
for (const auto& inst : *block) {
if (inst.GetOpcode() == ValueOpcode::GetBufferResourceIndex) {
++selections;
Check(block->id == 2u && inst.Flags<uint32_t>() < 2u,
"record selection moved outside its loop or lost its buffer root");
selected[inst.Flags<uint32_t>()] = &inst;
}
if (inst.GetOpcode() == ValueOpcode::LoadBufferU32x2 || inst.GetOpcode() == ValueOpcode::StoreBufferU8) {
const auto root = fixture->program.memory_info[inst.Flags<MemoryFlags>().index].resource;
Check(inst.Arg(0).Instruction()->Arg(0).Instruction() == selected[root],
"record operation did not use its dominating shared selection");
}
}
}
Check(selections == 2u, "repeated record operations duplicated candidate mapping searches");
for (uint32_t root = 0; root < 2u; ++root) {
const auto& source = plan.descriptor_sources[plan.info.buffers[root].source];
Check(source.indirect_descriptor && source.indirect_descriptor->indexed &&
source.indirect_descriptor->signed_count &&
source.indirect_descriptor->table_stride == 192u &&
source.indirect_descriptor->table_immediate == (root == 0u ? 160u : 176u) &&
!source.indirect_descriptor->key_count.IsEmpty(),
"record descriptor lost its bounded loop or member offset");
}
LinearTestMemory memory;
constexpr uint32_t table_offset = 256u;
const auto format = static_cast<uint32_t>(Prospero::BufferFormat::k16_16_16_16Float);
for (uint32_t row = 0; row < 4u; ++row) {
const auto first = (table_offset + row * 192u) / 4u;
std::fill_n(memory.words.begin() + first, 48u, 0xdeadbeefu);
for (const bool formatted : {true, false}) {
const auto word = first + (formatted ? 40u : 44u);
memory.words[word] = 0x100000u + row * 0x1000u + (formatted ? 0u : 0x8000u);
memory.words[word + 1u] = (formatted ? 8u : 3u) << 16u;
memory.words[word + 2u] = 16u;
memory.words[word + 3u] = formatted ? (format << 12u) | DstSel(4, 5, 6, 1) : 0x5204u;
}
}
std::array<uint32_t, 6> user_data{uint32_t(memory.base) + table_offset, 8u << 16u,
4u * 192u / 8u, 0x5204u, uint32_t(memory.base), 0u};
SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory,
.userdata = &memory, .read_specialization_memory = ReadLinearTestMemory};
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
const auto materialize = [&] { return MaterializeResources(plan, runtime, snapshot, specialization); };
for (const uint32_t count : {0u, 1u, 4u, 2u, UINT32_MAX, 0u, 4u}) {
memory.words[0] = count;
const uint32_t rows = count == UINT32_MAX ? 0u : count;
user_data[2] = rows == 0u ? 0u : 4u * 192u / 8u;
memory.reads = 0u;
Check(materialize() && snapshot.buffers.size() == 2u * std::max(rows, 1u) &&
memory.reads == 1u + rows * 2u,
"bounded record enumeration read unrelated fields or retained stale candidates");
for (uint32_t row = 0; row < rows; ++row)
for (const auto member : {160u, 176u})
Check(std::ranges::find(snapshot.specialization_reads,
std::pair{memory.base + table_offset + row * 192u + member, uint64_t{16u}}) !=
snapshot.specialization_reads.end(),
"record descriptor escaped same-dispatch overlap capture");
}
const auto original = specialization;
memory.words[(table_offset + 160u) / 4u] += 0x100u;
Check(materialize() && specialization == original,
"record address refresh changed the shader specialization");
user_data[2] = 4u;
memory.reads = 0u;
Check(materialize() && snapshot.buffers.size() == 2u && memory.reads == 1u,
"record enumeration read beyond the scalar buffer table bounds");
user_data[2] = 4u * 192u / 8u;
const auto first = (table_offset + 160u) / 4u;
const auto first_address = memory.words[first];
memory.words[first] = 0u;
memory.words[first + 2u] = 0u;
Check(materialize() && specialization.buffers[0].zero_stride_oob &&
specialization.buffers[0].descriptor_swizzle == DstSel(4, 5, 6, 1) &&
specialization.buffers[0].indirect_root == 0u,
"empty formatted candidate lost its OOB constants or other candidates");
ApplyResourceSpecialization(fixture->program, specialization);
uint32_t loads = 0u;
for (const auto* block : fixture->program.blocks)
for (const auto& inst : *block) loads += inst.GetOpcode() == ValueOpcode::LoadBufferU32x2;
Check(loads == 112u, "empty root incorrectly folded nonempty formatted candidates");
memory.words[first] = first_address;
memory.words[first + 2u] = 16u;
for (const auto [word, replacement] : std::array<std::pair<uint32_t, uint32_t>, 5>{
{{first + 1u, 4u << 16u}, {first + 1u, 9u << 16u},
{first, memory.words[first] + 1u}, {first + 3u, memory.words[first + 3u] | (1u << 28u)},
{first + 1u, (8u << 16u) | (1u << 31u)}}}) {
const auto saved = memory.words[word];
memory.words[word] = replacement;
Check(!materialize(), "formatted record accepted unproved alignment or addressing");
memory.words[word] = saved;
}
memory.words[0] = MaxIndirectDescriptorProbes + 1u;
Check(!materialize(), "record table accepted an unbounded candidate count");
for (const auto [step, bypass] : {std::pair{2u, false}, std::pair{1u, true}})
CheckFatal([&] { make(step, bypass); }, "not a valid runtime value",
"record table accepted an unproved loop bound");
}
void TestFiniteBufferSelection() {
enum class Variant { Valid, WiderExec, Unbounded, Cycle, TooMany };
const auto make = [](Variant variant) {
@@ -4516,7 +4685,7 @@ void TestPackedScalarBufferSelection() {
fixture->PlanAndTrack();
auto plan = ExtractResourcePlan(fixture->program);
const auto& source = plan.descriptor_sources[plan.info.buffers[0].source];
Check(source.indirect_descriptor && source.indirect_descriptor->indexed_store &&
Check(source.indirect_descriptor && source.indirect_descriptor->indexed &&
source.indirect_descriptor->selector->byte_ranges ==
std::vector<std::pair<uint32_t, uint32_t>>{{0x1c0u, 1024u}},
"packed scalar lookup lost a Phi arm or independent DWORD alignment");
@@ -4571,8 +4740,8 @@ void TestPackedScalarBufferSelection() {
mixed->PlanAndTrack();
const auto& buffers = mixed->program.info.buffers;
Check(buffers.size() == 2u && buffers[0].source != buffers[1].source &&
mixed->program.descriptor_sources[buffers[0].source].indirect_descriptor->indexed_store &&
!mixed->program.descriptor_sources[buffers[1].source].indirect_descriptor->indexed_store,
mixed->program.descriptor_sources[buffers[0].source].indirect_descriptor->indexed &&
!mixed->program.descriptor_sources[buffers[1].source].indirect_descriptor->indexed,
"indexed and byte-addressed stores incorrectly shared their descriptor bounds contract");
}
@@ -5888,6 +6057,7 @@ int main() {
Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit);
Run("writable descriptor phi", TestWritableDescriptorPhi);
Run("bounded formatted buffer store", TestBoundedFormattedBufferStore);
Run("bounded buffer records", TestBoundedBufferRecords);
Run("finite buffer selection", TestFiniteBufferSelection);
Run("packed scalar buffer selection", TestPackedScalarBufferSelection);
Run("conditional sampler phi", TestConditionalSamplerPhi);
tests/ShaderRecompilerComputeTests.cpp
+135 -2
View File
@@ -36575,6 +36575,135 @@ void CheckBoundedFormattedBufferCopy(VulkanHarness &vulkan) {
std::printf("[compute] %-32s ok\n", "BoundedFormattedBufferCopy");
}
void CheckBoundedRecordBufferAccess(VulkanHarness &vulkan) {
using namespace ShaderRecompiler::IR;
using F = Prospero::BufferFormat;
const u32 alignment = vulkan.RuntimeContext().GetPhysicalDeviceProperties()
.limits.minStorageBufferOffsetAlignment;
const u32 slot = (4096u + alignment - 1u) / alignment * alignment;
for (const u32 wave : {32u, 64u}) {
for (const u32 count : {0u, 2u, 6u}) {
TestCase test;
test.name = wave == 32 ? "BoundedRecordBufferAccess32" : "BoundedRecordBufferAccess64";
constexpr u32 table = 256u;
test.initial.assign((slot * 13u + 6u * wave * 16u) / 4u, 0xdeadbeefu);
test.has_user_data = test.has_compute_info = true;
test.user_data[0] = table;
test.user_data[1] = 8u << 16u;
test.user_data[2] = 6u * 192u / 8u;
test.user_data[3] = 0x5204u;
test.user_data[4] = count;
const std::array<u32, 4> output{slot * 13u, 16u << 16u, 6u * wave, 0x5204u};
std::copy(output.begin(), output.end(), test.user_data.begin() + 16u);
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1;
test.compute_info.lds_size_dwords = 0;
auto &code = test.code;
code.insert(code.end(), {EncodeSMovB32(32, InlineU32(0)), EncodeSop1(0x04, 36, 126)});
AppendVMovLiteral(&code, 13, 0x80000000u);
code.insert(code.end(), {EncodeVopc(0xc2, InlineU32(6), 0),
EncodeVop2(0x01, 12, Vgpr(0), 13)});
AppendVMovLiteral(&code, 14, 0x55555556u);
code.insert(code.end(), {EncodeVopc(0xc2, InlineU32(6), 0),
EncodeVop2(0x01, 15, Vgpr(0), 14)});
AppendVMovLiteral(&code, 10, 0xa5c39b72u);
const auto header = code.size();
code.push_back(EncodeSopc(0x04, 32, 4));
const auto exit_branch = code.size();
code.push_back(0);
code.insert(code.end(), {EncodeSop2(0x26, 33, 32, 255), 192u,
EncodeSmem0(0x0a, 8, 0), EncodeSmem1(160u, 33),
EncodeSmem0(0x0a, 12, 0), EncodeSmem1(176u, 33), EncodeSopp(0x0c, 0),
EncodeMubuf0(0x83, 0, true, false), EncodeMubuf1(4, 2, 12),
EncodeMubuf0(0x83, 0, true, false, true), EncodeMubuf1(6, 2, 12),
EncodeSopp(0x0c, 0), EncodeSop2(0x1e, 34, 32, InlineU32(std::countr_zero(wave))),
EncodeVop2(0x25, 20, 34, 0),
EncodeMubuf0(0x1e, 0, true, false), EncodeMubuf1(4, 4, 20)});
AppendSMovLiteral(&code, 126, 0x55555555u);
AppendSMovLiteral(&code, 127, wave == 64 ? 0x55555555u : 0u);
code.insert(code.end(), {EncodeMubuf0(0x19, 0, true, false, true), EncodeMubuf1(10, 3, 15),
EncodeSop1(0x04, 126, 36), EncodeSop2(0x02, 32, 32, InlineU32(1))});
code.push_back(EncodeSopp(0x02, header - code.size() - 1u));
code[exit_branch] = EncodeSopp(0x04, code.size() - exit_branch - 1u);
AppendEnd(&code);
for (u32 row = 0; row < 6u; ++row) {
const u32 src = slot * (row + 1u) + (row + 1u) * 4u;
const u32 dst = slot * (row + 7u) + row + 1u;
const auto format = count == 6u && row == 0u ? F::k32_32_32_32Float :
count == 6u && row == 1u ? F::k8_8_8_8UNorm : F::k16_16_16_16Float;
const u32 swizzle = row == 2u ? DstSel(1, 4, 1, 4) : DstSel(4, 5, 6, 7);
std::array<u32, 4> input{row == 2u ? 0u : src, 16u << 16u,
row == 2u ? 0u : 2u, static_cast<u32>(format) << 12u | swizzle};
if (row == 3u) input.fill(0u);
const std::array<u32, 4> destination{dst, 3u << 16u, 16u, 0x5204u};
std::copy(input.begin(), input.end(), test.initial.begin() + table / 4u + row * 48u + 40u);
std::copy(destination.begin(), destination.end(), test.initial.begin() + table / 4u + row * 48u + 44u);
for (u32 record = 0; record < 2u; ++record) {
const auto word = src / 4u + record * 4u;
if (format == F::k32_32_32_32Float) {
const std::array<u32, 4> data{0x3f800000u, 0x3f000000u, 0u, 0xc0000000u};
std::copy(data.begin(), data.end(), test.initial.begin() + word);
} else if (format == F::k8_8_8_8UNorm) test.initial[word] = 0xffffffffu;
else {
test.initial[word] = 0x38003c00u;
test.initial[word + 1u] = 0xc0000000u;
}
}
}
test.expected = test.initial;
auto expected_bytes = std::as_writable_bytes(std::span(test.expected));
for (u32 row = 0; row < count; ++row) {
for (u32 lane = 0; lane < wave; ++lane) {
std::array<u32, 2> value{};
if (row == 2u) value = {0x3c00u, 0x3c00u};
else if (row != 3u && lane < 2u && !(row == 5u && lane == 1u))
value = count == 6u && row == 1u ? std::array<u32, 2>{0x3c003c00u, 0x3c003c00u}
: std::array<u32, 2>{0x38003c00u, 0xc0000000u};
const u32 out = slot * 13u / 4u + (row * wave + lane) * 4u;
for (u32 word = 0; word < 4u; ++word) test.expected[out + word] = value[word % 2u];
if (lane < 16u && lane != 6u && (lane & 1u) == 0u)
expected_bytes[slot * (row + 7u) + row + 1u + lane * 3u] = std::byte{0xc3};
}
}
if (count == 2u) {
test.required_spirv = {"StorageBufferArrayNonUniformIndexing"};
test.forbidden_spirv = {"OpSwitch", "PhysicalStorageBuffer"};
} else if (count == 6u) test.required_spirv = {"OpSwitch"};
auto compiled = CompileCase(test, vulkan.SubgroupSize());
Require(test.name, "mixed native aliases", count != 6u ||
(compiled.program.info.buffer_u8 && compiled.program.info.buffer_u16),
"mixed candidates lost an 8-bit or 16-bit native binding");
u32 selections = 0;
for (auto *block : compiled.program.blocks)
for (const auto &inst : *block) selections += inst.GetOpcode() == ValueOpcode::GetBufferResourceIndex;
Require(test.name, "shared loop selection", count == 0u ? selections <= 2u : selections == 2u,
"repeated formatted loads repeated the descriptor lookup");
for (u32 resource = 0; resource < compiled.resources.buffers.size(); ++resource) {
ShaderBufferResource descriptor{};
std::copy_n(compiled.resources.buffers[resource].dwords.begin(), 4u, descriptor.fields);
const u32 address = descriptor.Base48();
const u32 prefix = address % alignment;
u32 bytes = std::max(4u, static_cast<u32>(descriptor.GetSize()));
if (address == slot * 5u + 20u) bytes = 24u;
if (address == slot * 6u + 24u) bytes = 23u;
test.storage_buffer_ranges.emplace_back(address - prefix, bytes + prefix);
const u32 binding = compiled.program.info.buffers[resource].descriptor_index;
if (binding != UINT32_MAX)
compiled.packed_user_data[compiled.program.bindings.memory_offset_dword + binding / 4u] |=
prefix << ((binding % 4u) * 8u);
}
auto output_buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output_buffer);
CompareWords(test, "row keys, mixed formats, empty-One, partial payloads, prefixes and byte canaries",
test.expected, vulkan.ReadBuffer(test.name, output_buffer, test.expected.size()));
vulkan.DestroyBuffer(&output_buffer);
}
}
std::printf("[compute] %-32s ok\n", "BoundedRecordBufferAccess");
}
void CheckFiniteBufferSelection(VulkanHarness &vulkan) {
for (const u32 wave : {32u, 64u}) {
TestCase test;
@@ -36663,9 +36792,11 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) {
{Value(&lane), Value(7u)});
auto &key = block->AppendNewInst(ValueOpcode::IAdd32,
{Value(&selector), Value(8u)});
auto &selection = block->AppendNewInst(ValueOpcode::GetBufferResourceIndex, {Value(&key)});
selection.SetFlags<uint32_t>(0u);
auto &handle = block->AppendNewInst(
ValueOpcode::GetBufferResource,
{Value(&key), Value(0u), Value(0u), Value(0u)});
{Value(&selection), Value(0u), Value(0u), Value(0u)});
handle.SetFlags<uint32_t>(0u);
auto &lane_offset = block->AppendNewInst(ValueOpcode::IMul32,
{Value(&lane), Value(4u)});
@@ -36715,7 +36846,7 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) {
if (indexed) {
for (auto &buffer : program.info.buffers) buffer.packed_stride = 4;
}
program.descriptor_sources[0].indirect_descriptor->indexed_store = indexed;
program.descriptor_sources[0].indirect_descriptor->indexed = indexed;
program.memory_info[0].idxen = indexed;
program.memory_info[0].offen = !indexed;
store.SetArg(1, indexed ? Value(&index) : Value(0u));
@@ -45378,6 +45509,7 @@ int main(int argc, char **argv) {
CheckBoundedFormattedBufferCopy(vulkan);
CheckFiniteBufferSelection(vulkan);
CheckIndirectBufferStore(vulkan);
CheckBoundedRecordBufferAccess(vulkan);
CheckRuntimeBufferRecords(vulkan);
RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors());
RunCase(&vulkan, BufferLoadDwordGpuSelectedDescriptors());
@@ -46079,6 +46211,7 @@ int main(int argc, char **argv) {
CheckRectListShaders();
CheckBoundedFormattedBufferCopy(vulkan);
CheckIndirectBufferStore(vulkan);
CheckBoundedRecordBufferAccess(vulkan);
CheckFiniteBufferSelection(vulkan);
CheckRuntimeBufferRecords(vulkan);
CheckComputeThreadDimensions(vulkan);