raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-19 18:36:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

shader: resolve packed buffer selectors with shared indexed stores

Browse Source 
Co-authored-by: Ali Almohaya <AliAlmohaya@gmail.com>
This commit is contained in:
nmzik
2026-10-12 17:06:20 +02:00
parent 8f7d745383
commit fd66f580b4
9 changed files with 400 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/presentation/window/vulkanWindow.cpp
+4
View File
@@ -63,6 +63,7 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
features.shaderBufferInt64Atomics = VK_TRUE;
features.storageBuffer8BitAccess = VK_TRUE;
features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
features.shaderStorageBufferArrayNonUniformIndexing = VK_TRUE;
return features;
}
@@ -284,6 +285,9 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
check_feature(features12.shaderSampledImageArrayNonUniformIndexing,
"shaderSampledImageArrayNonUniformIndexing",
required_features12.shaderSampledImageArrayNonUniformIndexing);
check_feature(features12.shaderStorageBufferArrayNonUniformIndexing,
"shaderStorageBufferArrayNonUniformIndexing",
required_features12.shaderStorageBufferArrayNonUniformIndexing);
check_feature(features13.robustImageAccess, "robustImageAccess");
check_feature(features13.dynamicRendering, "dynamicRendering",
required_features13.dynamicRendering);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+5 -1
View File
@@ -398,6 +398,8 @@ uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryIn
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
bool UsesSharedIndirectBufferStore(const EmitterState& state, uint32_t resource);
void EmitMemoryOffsets(EmitterState& state);
uint32_t LdsDwordCount(const EmitterState& state);
@@ -409,6 +411,7 @@ struct MemoryResourceAccess {
uint32_t length = 0;
uint32_t byte_offset = 0;
uint32_t element_bits = 32;
bool non_uniform = false;
spv::MemoryAccessMask memory_access = spv::MemoryAccessMaskNone;
};
@@ -418,7 +421,8 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
const IR::MemoryInfo& mem,
BufferDefinition& buffer);
BufferDefinition& buffer,
uint32_t array_index = 0);
uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+39 -6
View File
@@ -249,10 +249,11 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3
});
}
uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
uint32_t prefix = 0) {
if (mem.kind == IR::ResourceKind::Buffer) {
const auto resource = ctx.state.program.info.buffers.at(mem.resource).descriptor_index;
const auto prefix = ctx.state.memory_byte_offsets[resource];
if (prefix == 0u) prefix = ctx.state.memory_byte_offsets[resource];
const auto address = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state),
BufferByteAddress(ctx, inst, mem), prefix);
// The host binding prefix must not wrap an out-of-range guest address into the buffer.
@@ -273,8 +274,9 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo
return ScratchByteAddress(ctx, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2));
}
uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
auto address = ByteAddress(ctx, inst, mem);
uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
uint32_t prefix = 0) {
auto address = ByteAddress(ctx, inst, mem, prefix);
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
// RDNA2 DS region addresses retain bits [15:2] after adding the byte offset.
address = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
@@ -540,8 +542,30 @@ void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t data) {
const auto index = DwordIndex(ctx, inst, mem);
EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
auto& state = ctx.state;
const auto source = mem.kind == IR::ResourceKind::Buffer
? state.program.info.buffers[mem.resource].source : UINT32_MAX;
const auto& sources = state.program.descriptor_sources;
const auto* indirect = source < sources.size() && sources[source].indirect_descriptor
? &*sources[source].indirect_descriptor : nullptr;
uint32_t index = 0;
uint32_t in_bounds = 0;
if (indirect != nullptr && indirect->indexed_store) {
// The resource plan proves stride-four, index-only stores. Check the original
// index before any byte multiplication can wrap it into the buffer.
const auto prefix = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
resource.byte_offset, ConstantU32(state, 2));
const auto guest_index = ctx.Arg(inst, 1);
const auto records = Binary(state, spv::OpISub, TypeU32(state), resource.length, prefix);
in_bounds = AndCondition(state,
Binary(state, spv::OpULessThanEqual, TypeBool(state), prefix, resource.length),
Binary(state, spv::OpULessThan, TypeBool(state), guest_index, records));
index = EmitAddU32(state, guest_index, prefix);
} else {
index = DwordIndex(ctx, inst, mem, resource.byte_offset);
in_bounds = EmitMemoryElementInBounds(state, resource, index);
}
EmitIfCondition(state, in_bounds, [&]() {
StoreWordInBounds(ctx, resource, index, data);
});
}
@@ -576,6 +600,15 @@ void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem)
const auto selected = EmitIndirectResourceIndex(
state, ctx.Arg(*handle, 0), buffer.indirect_mapping_offset,
buffer.indirect_search_iterations, UINT32_MAX);
if (UsesSharedIndirectBufferStore(state, mem.resource)) {
EmitIfCondition(state, Binary(state, spv::OpULessThan, TypeBool(state), selected,
ConstantU32(state, buffer.indirect_resources.size())), [&]() {
const auto slot = EmitAddU32(state, selected, ConstantU32(state, buffer.descriptor_index));
const auto access = PrepareStorageBufferResourceAccess(state, mem, state.storage_buffers[2], slot);
StoreWordPrepared(ctx, inst, mem, access, data);
});
return;
}
const auto merge = state.builder.AllocateId();
std::vector<uint32_t> labels(buffer.indirect_resources.size());
std::vector<uint32_t> branches {spv::OpSwitch, selected, merge};
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+65 -10
View File
@@ -2,14 +2,13 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
static uint32_t EmitShaderDataDwordLoadAt(EmitterState& state, uint32_t dword_index) {
if (state.program.bindings.UsesPushData()) {
dword_index += state.program.bindings.push_data_start_dword;
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state),
pointer, state.push_constant_variable, ConstantU32(state, 0),
ConstantU32(state, dword_index));
dword_index);
state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
return value;
}
@@ -18,13 +17,20 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
pointer, state.shader_data_storage_variable,
ConstantU32(state, 0), ConstantU32(state, dword_index));
ConstantU32(state, 0), dword_index);
state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
return value;
}
return ConstantU32(state, 0);
}
uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
if (state.program.bindings.UsesPushData()) {
dword_index += state.program.bindings.push_data_start_dword;
}
return EmitShaderDataDwordLoadAt(state, ConstantU32(state, dword_index));
}
uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(opcode, TypeU32(state), ret, lhs, rhs);
@@ -102,12 +108,38 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
return state.program.info.buffers[mem.resource].descriptor_format;
}
bool UsesSharedIndirectBufferStore(const EmitterState& state, uint32_t resource) {
const auto& buffers = state.program.info.buffers;
const auto& root = buffers[resource];
if (root.indirect_root != resource || root.descriptor_index == UINT32_MAX ||
root.packed_stride == 0u || root.indirect_resources.empty()) {
return false;
}
uint32_t slot = root.descriptor_index;
for (const auto candidate: root.indirect_resources) {
if (buffers[candidate].packed_stride != root.packed_stride ||
buffers[candidate].descriptor_index != slot++) return false;
}
return true;
}
void EmitMemoryOffsets(EmitterState& state) {
std::array<bool, IR::ShaderInfo::MaxBuffers> dynamic_offsets {};
for (uint32_t resource = 0; resource < state.program.info.buffers.size(); ++resource) {
if (!UsesSharedIndirectBufferStore(state, resource)) continue;
for (const auto candidate: state.program.info.buffers[resource].indirect_resources) {
dynamic_offsets[state.program.info.buffers[candidate].descriptor_index] = true;
}
}
const auto count = state.program.bindings.descriptor_counts[static_cast<size_t>(IR::DescriptorBindingKind::Buffers)];
for (uint32_t i = 0; i < count; i += 4u) {
const auto lanes = std::min(4u, count - i);
if (std::all_of(dynamic_offsets.begin() + i, dynamic_offsets.begin() + i + lanes,
[](bool dynamic) { return dynamic; })) continue;
const auto word =
EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u);
for (uint32_t lane = 0; lane < std::min(4u, count - i); ++lane) {
for (uint32_t lane = 0; lane < lanes; ++lane) {
if (dynamic_offsets[i + lane]) continue;
state.memory_byte_offsets[i + lane] = EmitNative<spv::OpBitFieldUExtract, IR::Type::U32>(
state, word, ConstantU32(state, lane * 8u), ConstantU32(state, 8u));
}
@@ -152,7 +184,8 @@ void EnsureLdsStorage(EmitterState& state) {
MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
const IR::MemoryInfo& mem,
BufferDefinition& buffer) {
BufferDefinition& buffer,
uint32_t array_index) {
if (buffer.variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
"storage buffer descriptor array was not emitted");
@@ -160,15 +193,34 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
if (buffer.pointer_type == 0) {
buffer.pointer_type = TypePointer(state, spv::StorageClassStorageBuffer, buffer.type);
}
const auto array_index =
state.program.info.buffers.at(mem.resource).descriptor_index;
const bool non_uniform = array_index != 0u;
const auto static_index = state.program.info.buffers.at(mem.resource).descriptor_index;
if (!non_uniform) array_index = ConstantU32(state, static_index);
MemoryResourceAccess access {
.kind = mem.kind,
.non_uniform = non_uniform,
.memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone};
access.object_pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, buffer.pointer_type, access.object_pointer,
buffer.variable, ConstantU32(state, array_index));
access.byte_offset = state.memory_byte_offsets[array_index];
buffer.variable, array_index);
if (non_uniform) {
state.builder.RequireExtension("SPV_EXT_descriptor_indexing");
state.builder.RequireCapability(spv::CapabilityShaderNonUniform);
state.builder.RequireCapability(spv::CapabilityStorageBufferArrayNonUniformIndexing);
state.builder.AddAnnotation(spv::OpDecorate, access.object_pointer, spv::DecorationNonUniform);
const auto offsets_start = state.program.bindings.memory_offset_dword +
(state.program.bindings.UsesPushData() ? state.program.bindings.push_data_start_dword : 0u);
const auto word = EmitShaderDataDwordLoadAt(state, EmitAddU32(
state, ConstantU32(state, offsets_start),
Binary(state, spv::OpShiftRightLogical, TypeU32(state), array_index, ConstantU32(state, 2))));
const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
Binary(state, spv::OpBitwiseAnd, TypeU32(state), array_index, ConstantU32(state, 3)),
ConstantU32(state, 3));
access.byte_offset = EmitNative<spv::OpBitFieldUExtract, IR::Type::U32>(
state, word, shift, ConstantU32(state, 8));
} else {
access.byte_offset = state.memory_byte_offsets[static_index];
}
access.length = state.builder.AllocateId();
state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length,
access.object_pointer, 0);
@@ -260,6 +312,9 @@ uint32_t EmitStorageBufferElementPointer(EmitterState& state,
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, pointer_type, pointer, access.object_pointer,
ConstantU32(state, 0), index);
if (access.non_uniform) {
state.builder.AddAnnotation(spv::OpDecorate, pointer, spv::DecorationNonUniform);
}
return pointer;
}
src/graphics/shader/recompiler/ir/ShaderIR.h
+2
View File
@@ -504,6 +504,7 @@ struct DescriptorSource {
uint32_t source = UINT32_MAX;
uint32_t stride = 0;
uint32_t offset = 0;
std::vector<std::pair<uint32_t, uint32_t>> byte_ranges;
bool operator==(const SelectorRead& other) const = default;
};
@@ -514,6 +515,7 @@ struct DescriptorSource {
uint32_t table_stride = 0;
uint32_t table_record_bytes = 0;
bool table_scalar = false;
bool indexed_store = false;
uint32_t workgroup_axis = UINT32_MAX;
Value key_count;
Value selector_first;
src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp
+23 -6
View File
@@ -195,15 +195,15 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset,
runtime.read_specialization_memory(runtime.userdata, address, prefix);
}
bool NormalizeIndirectStoreBuffer(DescriptorValue& value) {
bool NormalizeIndirectStoreBuffer(DescriptorValue& value, bool indexed) {
ShaderBufferResource descriptor;
if (!DecodeBufferDescriptor(value, descriptor)) return false;
if (descriptor.Type() != 0u || descriptor.Format() == Prospero::BufferFormat::kInvalid ||
descriptor.Base48() == 0u || descriptor.GetSize() == 0u) {
value.dwords.fill(0u);
} else if (descriptor.Stride() != 1u || descriptor.SwizzleEnabled() || descriptor.AddTid() ||
descriptor.OutOfBounds() != 3u) {
return SpecializationFail("indirect buffer stores require linear byte bounds");
} else if (descriptor.Stride() != (indexed ? 4u : 1u) || descriptor.SwizzleEnabled() ||
descriptor.AddTid() || descriptor.OutOfBounds() != (indexed ? 0u : 3u)) {
return SpecializationFail("indirect buffer stores require exact linear DWORD or byte bounds");
}
return true;
}
@@ -337,6 +337,21 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
}
keys.resize(key_count);
std::iota(keys.begin(), keys.end(), 0u);
} else if (!selector->byte_ranges.empty()) {
if (material_value.dword_count != 2u) return false;
const auto base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0];
std::array<bool, 256> selected {};
for (const auto [offset, size]: selector->byte_ranges) {
if (size > MaxIndirectDescriptorProbes || ((offset | size) & 3u) != 0u) return false;
keys.resize(size / 4u);
if (!ReadScalarTable(base, UINT64_MAX, offset, runtime, keys)) return false;
for (const auto word: keys)
for (uint32_t shift = 0; shift < 32u; shift += 8u)
selected[(word >> shift) & 255u] = true;
}
keys.clear();
for (uint32_t key = 0; key < selected.size(); ++key)
if (selected[key]) keys.push_back(key);
} else if (!indirect.selector_first.IsEmpty()) {
ShaderBufferResource material;
uint32_t first = 0, count = 0;
@@ -380,7 +395,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
mask &= mask - 1u;
}
} else return false;
if (selector != nullptr) {
if (selector != nullptr && selector->byte_ranges.empty()) {
std::ranges::sort(keys);
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
}
@@ -1132,7 +1147,9 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
if (!MaterializeIndirectDescriptor(
program, *source->indirect_descriptor, i, 4u, observed, clean, snapshot,
snapshot.buffers, specialization.buffers, ShaderInfo::MaxBuffers,
NormalizeIndirectStoreBuffer))
[&](DescriptorValue& value) {
return NormalizeIndirectStoreBuffer(value, source->indirect_descriptor->indexed_store);
}))
return false;
}
} else if (!evaluate(base.source, snapshot.buffers[i], base.written)) {
src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp
+77 -15
View File
@@ -909,7 +909,7 @@ private:
const auto& b = *descriptor.indirect_descriptor;
if (a.selector != b.selector || a.table_source != b.table_source ||
a.table_offset != b.table_offset || a.table_immediate != b.table_immediate ||
a.table_stride != b.table_stride ||
a.table_stride != b.table_stride || a.indexed_store != b.indexed_store ||
a.workgroup_axis != b.workgroup_axis || a.sources != b.sources ||
!EquivalentValue(m_program, a.key_count, b.key_count) ||
a.selector_first.IsEmpty() != b.selector_first.IsEmpty() ||
@@ -1821,17 +1821,71 @@ private:
return true;
}
bool MatchPackedScalarKey(Value key, DescriptorSource::IndirectDescriptor& indirect) {
const auto* mask = key.Resolve().TryInstruction();
uint32_t bits = 0;
if (mask == nullptr || mask->GetOpcode() != ValueOpcode::BitwiseAnd32 ||
!ImmediateU32(mask->Arg(1), bits) || bits != 255u) return false;
const auto* shift = mask->Arg(0).Resolve().TryInstruction();
if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftRightLogical32 ||
(PossibleU32Bits(shift->Arg(1)) & ~24u) != 0u) return false;
DescriptorSource::IndirectDescriptor::SelectorRead selector;
std::vector<const Inst*> pending {shift->Arg(0).Resolve().TryInstruction()};
std::vector<const Inst*> visited;
while (!pending.empty()) {
const auto* read = pending.back();
pending.pop_back();
if (read == nullptr) return false;
if (std::ranges::find(visited, read) != visited.end()) continue;
visited.push_back(read);
if (read->GetOpcode() == ValueOpcode::Phi) {
for (size_t arm = 0; arm < read->NumArgs(); ++arm)
pending.push_back(read->Arg(arm).Resolve().TryInstruction());
continue;
}
uint32_t index = 0;
const auto* memory = ScalarReadMemory(*read, index);
DescriptorSource source;
if (memory == nullptr || memory->kind != ResourceKind::ScalarAddress ||
memory->offset > INT32_MAX || !MakeRuntimeTableSource(*read, source)) return false;
const auto maximum = PossibleU32Bits(read->Arg(1)) & ~3u;
const auto offset = memory->offset & ~3u;
if (uint64_t {offset} + maximum + 4u > UINT32_MAX) return false;
const auto source_index = InternSource(source);
if (selector.source != UINT32_MAX && selector.source != source_index) return false;
selector.source = source_index;
selector.byte_ranges.emplace_back(offset, maximum + 4u);
}
if (selector.byte_ranges.empty()) return false;
// Scalar reads align both offsets independently. Merge the possible read spans;
// the GPU retains the actual byte selection and the host only discovers V#s.
std::ranges::sort(selector.byte_ranges);
for (size_t i = 1; i < selector.byte_ranges.size();) {
auto& previous = selector.byte_ranges[i - 1u];
const auto current = selector.byte_ranges[i];
if (current.first > previous.first + previous.second) { ++i; continue; }
previous.second = std::max(previous.second, current.first + current.second - previous.first);
selector.byte_ranges.erase(selector.byte_ranges.begin() + i);
}
indirect.selector = std::move(selector);
return true;
}
bool TryMakeIndirectBuffer(Inst& handle, const DescriptorSource& descriptor,
IndirectDescriptorPlan& plan) {
if (handle.GetOpcode() != ValueOpcode::GetBufferResource || handle.NumArgs() != 4u ||
handle.Uses().empty()) return false;
if (handle.Uses().front().user->GetOpcode() != ValueOpcode::StoreBufferU32) return false;
const auto indexed = m_program.memory_info[handle.Uses().front().user->Flags<MemoryFlags>().index].idxen;
for (const auto& use: handle.Uses()) {
const auto& inst = *use.user;
if (inst.GetOpcode() != ValueOpcode::StoreBufferU32) return false;
const auto& memory = m_program.memory_info[inst.Flags<MemoryFlags>().index];
uint32_t scalar = 1u;
if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
memory.idxen || !memory.offen || !ImmediateU32(inst.Arg(3), scalar) || scalar != 0u)
memory.idxen != indexed || memory.offen == indexed ||
(indexed && (memory.offset != 0u || inst.Arg(2).Resolve() != Value(0u))) ||
!ImmediateU32(inst.Arg(3), scalar) || scalar != 0u)
return false;
}
DescriptorSource table_source;
@@ -1840,8 +1894,26 @@ private:
uint32_t table_stride = 0;
uint32_t table_immediate = 0;
if (!MatchDescriptorTable(handle, descriptor, plan, table_source, key, table_offset,
table_stride, table_immediate) || table_stride != 16u ||
table_source.dword_count != 4u || table_offset != 0u || table_immediate != 0u) return false;
table_stride, table_immediate) || table_stride != 16u) return false;
DescriptorSource::IndirectDescriptor indirect;
indirect.table_stride = table_stride;
indirect.table_offset = table_offset;
indirect.table_immediate = table_immediate;
indirect.indexed_store = indexed;
const auto finish = [&] {
indirect.table_source = InternSource(table_source);
DescriptorSource source;
source.dword_count = 4u;
source.dwords.fill(Value(0u));
source.indirect_descriptor = std::move(indirect);
plan.handle = &handle;
plan.source = InternSource(source);
plan.key = key;
return true;
};
if (table_source.dword_count == 2u && MatchPackedScalarKey(key, indirect)) return finish();
if (table_source.dword_count != 4u || table_offset != 0u || table_immediate != 0u || indexed)
return false;
const auto* lane_read = key.Resolve().TryInstruction();
if (lane_read == nullptr || lane_read->GetOpcode() != ValueOpcode::ReadFirstLane)
return false;
@@ -1869,21 +1941,11 @@ private:
!ImmediateU32(read->Arg(3), scalar) || scalar != 0u ||
!EquivalentValue(m_program, selected->Arg(0), read->Arg(4))) return false;
DescriptorSource material_source;
DescriptorSource::IndirectDescriptor indirect;
indirect.table_stride = table_stride;
if (!MakeRuntimeTableSource(*read, material_source) ||
!BoundedSelectedIndex(read->Arg(1), read->Arg(4), indirect)) return false;
indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead{.source = InternSource(material_source),
.offset = memory.offset + lane * 4u});
indirect.table_source = InternSource(table_source);
DescriptorSource source;
source.dword_count = 4u;
source.dwords.fill(Value(0u));
source.indirect_descriptor = indirect;
plan.handle = &handle;
plan.source = InternSource(source);
plan.key = key;
return true;
return finish();
}
using SelectorBounds = std::span<const std::pair<const Inst*, uint32_t>>;
tests/ResourceTrackingTests.cpp
+110
View File
@@ -2427,6 +2427,115 @@ void TestWritableDescriptorPhi() {
"control-dependent writable descriptor phi was not rejected transactionally");
}
void TestPackedScalarBufferSelection() {
const auto make = [](uint32_t shift_mask = 24u, bool cycle = false, uint32_t store_offset = 0u,
bool byte_store = false) {
auto fixture = std::make_unique<Fixture>();
auto& f = *fixture;
auto* entry = f.block;
auto* left = f.AddBlock();
auto* right = f.AddBlock();
auto* merge = f.AddBlock();
entry->AddBranch(left);
entry->AddBranch(right);
left->AddBranch(merge);
right->AddBranch(merge);
const auto base = f.Address(f.UserData(0u), f.UserData(1u));
const auto lane = f.Emit(ValueOpcode::LaneId);
const auto offset = f.Emit(ValueOpcode::BitwiseAnd32, {lane, Value(511u)});
const auto shift = f.Emit(ValueOpcode::BitwiseAnd32, {lane, Value(shift_mask)});
auto& phi = merge->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
for (const auto [block, immediate] : {std::pair{left, 0x1c1u}, std::pair{right, 0x3c2u}}) {
const auto read = f.Emit(ValueOpcode::LoadAddressU32,
{base, offset, Value(0u), Value(true)},
f.AddMemory({.kind = ResourceKind::ScalarAddress, .offset = immediate}, immediate), block);
phi.AddPhiOperand(block, cycle ? Value(&phi) : read);
}
f.block = merge;
const auto word = f.Emit(ValueOpcode::ShiftRightLogical32, {Value(&phi), shift});
const auto key = f.Emit(ValueOpcode::BitwiseAnd32, {word, Value(255u)});
const auto scaled = f.Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(4u)});
const auto table_offset = f.Emit(ValueOpcode::IAdd32, {scaled, Value(64u)});
std::array<Value, 4> descriptor;
for (uint32_t component = 0; component < descriptor.size(); ++component) {
descriptor[component] = f.Emit(ValueOpcode::LoadAddressU32,
{base, table_offset, Value(0u), Value(true)},
f.AddMemory({.kind = ResourceKind::ScalarAddress, .offset = component * 4u,
.component_index = component, .component_count = 4u}, 0x2b4u));
}
f.Emit(ValueOpcode::StoreBufferU32,
{f.Buffer(descriptor), lane, Value(0u), Value(0u), Value(42u), Value(true)},
f.AddMemory({.kind = ResourceKind::Buffer, .offset = store_offset, .idxen = true}, 0x2c8u));
if (byte_store)
f.Emit(ValueOpcode::StoreBufferU32,
{f.Buffer(descriptor), Value(0u), lane, Value(0u), Value(43u), Value(true)},
f.AddMemory({.kind = ResourceKind::Buffer, .offen = true}, 0x2d0u));
return fixture;
};
auto fixture = make();
fixture->PlanAndTrack();
auto plan = ExtractResourcePlan(fixture->program);
const auto& source = plan.descriptor_sources[plan.info.buffers[0].source];
Check(source.indirect_descriptor && source.indirect_descriptor->indexed_store &&
source.indirect_descriptor->selector->byte_ranges ==
std::vector<std::pair<uint32_t, uint32_t>>{{0x1c0u, 1024u}},
"packed scalar lookup lost a Phi arm or independent DWORD alignment");
LinearTestMemory memory;
memory.watched_address = memory.base + 0x1c0u;
for (uint32_t key = 0; key < 19u; ++key) {
const auto descriptor = 16u + key * 4u;
memory.words[descriptor] = 0x100000u + key * 0x10000u;
memory.words[descriptor + 1u] = 4u << 16u;
memory.words[descriptor + 2u] = 14400u;
memory.words[descriptor + 3u] = 0x5204u;
}
for (uint32_t i = 0; i < 256u; ++i) {
for (uint32_t byte = 0; byte < 4u; ++byte)
memory.words[0x1c0u / 4u + i] |= ((i * 4u + byte) % 19u) << (byte * 8u);
}
// No descriptor exists after key 18; broad enumeration would read this address.
memory.fail_address = memory.base + 0x170u;
const std::array<uint32_t, 2> user_data{static_cast<uint32_t>(memory.base), 0u};
const SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory,
.userdata = &memory, .read_specialization_memory = ReadLinearTestMemory};
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
const auto materialize = [&] { return MaterializeResources(plan, runtime, snapshot, specialization); };
Check(materialize() && snapshot.buffers.size() == 19u && memory.reads == 20u &&
memory.watched_reads == 1u && memory.watched_dwords == 256u,
"packed scalar lookup did not batch its read and deduplicate writable candidates");
Check(std::ranges::find(snapshot.specialization_reads,
std::pair{memory.base + 0x1c0u, uint64_t{1024u}}) != snapshot.specialization_reads.end() &&
std::ranges::all_of(snapshot.buffers, [&](const DescriptorValue& value) {
const auto key = (value.dwords[0] - 0x100000u) / 0x10000u;
return std::ranges::find(snapshot.specialization_reads,
std::pair{memory.base + 64u + key * 16u, uint64_t{16u}}) != snapshot.specialization_reads.end();
}),
"packed lookup or writable descriptors escaped specialization overlap tracking");
const auto initial = specialization;
memory.words[16] += 4096u;
memory.reads = memory.watched_reads = 0u;
Check(materialize() && specialization == initial && memory.reads == 20u,
"packed table refresh changed shader specialization or repeated scalar reads");
memory.words[17] = 8u << 16u;
Check(!materialize(), "indexed DWORD write accepted an incompatible descriptor stride");
memory.words[17] = 4u << 16u;
memory.words[19] |= 3u << 28u;
Check(!materialize(), "indexed DWORD write accepted raw byte bounds");
for (auto bad : {0u, 1u, 2u}) {
auto invalid = make(bad == 0u ? 28u : 24u, bad == 1u, bad == 2u ? 4u : 0u);
CheckFatal([&] { invalid->PlanAndTrack(); }, "not a valid runtime value",
"packed selector accepted a cross-byte shift, leafless cycle, or indexed byte offset");
}
auto mixed = make(24u, false, 0u, true);
mixed->PlanAndTrack();
const auto& buffers = mixed->program.info.buffers;
Check(buffers.size() == 2u && buffers[0].source != buffers[1].source &&
mixed->program.descriptor_sources[buffers[0].source].indirect_descriptor->indexed_store &&
!mixed->program.descriptor_sources[buffers[1].source].indirect_descriptor->indexed_store,
"indexed and byte-addressed stores incorrectly shared their descriptor bounds contract");
}
ResourcePlan ConditionalSamplerPlan(bool diamond, bool reverse, bool reverse_phi,
bool nonuniform = false,
bool writable = false) {
@@ -3732,6 +3841,7 @@ int main() {
Run("SRT runtime", TestSrtFlatteningAndRuntimeMemoization);
Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit);
Run("writable descriptor phi", TestWritableDescriptorPhi);
Run("packed scalar buffer selection", TestPackedScalarBufferSelection);
Run("conditional sampler phi", TestConditionalSamplerPhi);
Run("finite image phi cycle", TestFiniteImagePhiCycle);
Run("finite inline sampler", TestFiniteInlineSampler);
tests/ShaderRecompilerComputeTests.cpp
+75 -17
View File
@@ -18812,6 +18812,7 @@ private:
features12.shaderOutputViewportIndex != true ||
features12.shaderBufferInt64Atomics != true ||
features12.shaderSampledImageArrayNonUniformIndexing != true ||
features12.shaderStorageBufferArrayNonUniformIndexing != true ||
features12.shaderSharedInt64Atomics != true ||
image_atomic64.shaderImageInt64Atomics != true ||
workgroup_layout.workgroupMemoryExplicitLayout != true ||
@@ -18893,6 +18894,9 @@ private:
Require("VulkanHarness", "dispatch",
available_features12.shaderSampledImageArrayNonUniformIndexing == true,
"nonuniform sampled image indexing is not supported");
Require("VulkanHarness", "dispatch",
available_features12.shaderStorageBufferArrayNonUniformIndexing == true,
"nonuniform storage buffer indexing is not supported");
Require("VulkanHarness", "dispatch", available_min_lod.minLod == true,
"image view minimum LOD is not supported");
Require("VulkanHarness", "graphics", available_features12.shaderOutputLayer == true,
@@ -35580,8 +35584,6 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) {
test.initial.assign(64, 0xdeadbeefu);
test.storage_buffer_range_bytes = 64;
test.storage_buffer_offsets = {12, 108};
test.required_spirv = {"OpSwitch"};
test.forbidden_spirv = {"PhysicalStorageBuffer"};
CompiledShader compiled;
auto &program = compiled.program;
@@ -35603,13 +35605,19 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) {
ValueOpcode::GetBufferResource,
{Value(&key), Value(0u), Value(0u), Value(0u)});
handle.SetFlags<uint32_t>(0u);
auto &offset = block->AppendNewInst(ValueOpcode::IMul32,
{Value(&lane), Value(4u)});
auto &lane_offset = block->AppendNewInst(ValueOpcode::IMul32,
{Value(&lane), Value(4u)});
auto &overflow = block->AppendNewInst(ValueOpcode::IEqual32,
{Value(&lane), Value(6u)});
auto &offset = block->AppendNewInst(ValueOpcode::SelectU32,
{Value(&overflow), Value(UINT32_MAX), Value(&lane_offset)});
auto &index = block->AppendNewInst(ValueOpcode::SelectU32,
{Value(&overflow), Value(0x40000000u), Value(&lane)});
auto &data = block->AppendNewInst(ValueOpcode::IAdd32,
{Value(&lane), Value(0x1000u)});
auto &exec = block->AppendNewInst(ValueOpcode::INotEqual32,
{Value(&lane), Value(4u)});
block->AppendNewInst(ValueOpcode::StoreBufferU32,
auto &store = block->AppendNewInst(ValueOpcode::StoreBufferU32,
{Value(&handle), Value(0u), Value(&offset), Value(0u),
Value(&data), Value(&exec)});
program.memory_info.push_back({.kind = ResourceKind::Buffer, .offen = true});
@@ -35632,36 +35640,80 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) {
compute.host_subgroup_size = vulkan.SubgroupSize();
compute.threads_num[0] = 32;
compute.threads_num[1] = compute.threads_num[2] = 1;
for (const u32 null_buffers : {0u, 1u, 2u}) {
enum class Layout { Shared, NullChild, AllNull, DifferentStride, ShiftedPushData, StorageData,
Indexed, IndexedNullChild, IndexedSingle };
for (const auto layout : {Layout::Shared, Layout::NullChild, Layout::AllNull,
Layout::DifferentStride, Layout::ShiftedPushData, Layout::StorageData,
Layout::Indexed, Layout::IndexedNullChild, Layout::IndexedSingle}) {
const bool indexed = layout == Layout::Indexed || layout == Layout::IndexedNullChild ||
layout == Layout::IndexedSingle;
const bool null_child = layout == Layout::NullChild || layout == Layout::IndexedNullChild;
const bool single = layout == Layout::AllNull || layout == Layout::IndexedSingle;
program.info.buffers = {root, candidate};
if (null_buffers != 0u) program.info.buffers[1].packed_stride = 0;
if (null_buffers == 2u) {
if (indexed) {
for (auto &buffer : program.info.buffers) buffer.packed_stride = 4;
}
program.descriptor_sources[0].indirect_descriptor->indexed_store = indexed;
program.memory_info[0].idxen = indexed;
program.memory_info[0].offen = !indexed;
store.SetArg(1, indexed ? Value(&index) : Value(0u));
store.SetArg(2, indexed ? Value(0u) : Value(&offset));
if (null_child) program.info.buffers[1].packed_stride = 0;
if (layout == Layout::DifferentStride) program.info.buffers[1].packed_stride = 2;
if (single) {
// Materialization deduplicates all null destinations into one unmapped root.
program.info.buffers.resize(1);
program.info.buffers[0].packed_stride = 0;
if (layout == Layout::AllNull) program.info.buffers[0].packed_stride = 0;
program.info.buffers[0].indirect_root = BufferResource::NoIndirectBuffer;
program.info.buffers[0].indirect_search_iterations = 0;
program.info.buffers[0].indirect_resources.clear();
}
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
const auto push_start = layout == Layout::ShiftedPushData ? 3u :
layout == Layout::StorageData ? PushData::NoStart : 0u;
compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}, push_start);
Require(test.name, "candidate bindings",
program.bindings.descriptor_counts[0] == (null_buffers == 2u ? 1u : 2u) &&
program.bindings.descriptor_counts[0] == (single ? 1u : 2u) &&
program.info.buffers[0].descriptor_index == 0u &&
(null_buffers == 2u || program.info.buffers[1].descriptor_index == 1u),
(single || program.info.buffers[1].descriptor_index == 1u),
"indirect store lost a buffer binding");
compiled.packed_user_data.resize(program.bindings.ShaderDataDwords());
compiled.packed_user_data[program.bindings.memory_offset_dword] = 12u | (108u << 8u);
ValidateSpirv(test.name, compiled.spirv);
if (null_buffers == 2u) {
test.required_spirv.clear();
test.required_spirv.clear();
test.forbidden_spirv = {"PhysicalStorageBuffer"};
if (!single && !null_child && layout != Layout::DifferentStride) {
test.required_spirv = {"StorageBufferArrayNonUniformIndexing"};
test.forbidden_spirv.push_back("OpSwitch");
std::set<u32> nonuniform;
std::vector<u32> accessed_pointers;
u32 prefix_extracts = 0;
for (size_t offset = 5; offset < compiled.spirv.size();) {
const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
nonuniform.insert(words[1]);
if (opcode == spv::OpArrayLength) accessed_pointers.push_back(words[3]);
if (opcode == spv::OpStore) accessed_pointers.push_back(words[1]);
if (opcode == spv::OpBitFieldUExtract) ++prefix_extracts;
offset += words.size();
}
Require(test.name, "shared store and nonuniform pointers",
prefix_extracts == 1u && accessed_pointers.size() == 2u &&
std::ranges::all_of(accessed_pointers, [&](u32 pointer) {
return nonuniform.contains(pointer);
}),
"indirect store retained static prefixes, was duplicated or lost a NonUniform pointer decoration");
} else if (single) {
test.forbidden_spirv.push_back("OpSwitch");
} else {
test.required_spirv = {"OpSwitch"};
}
CheckSpirvText(test, compiled.spirv);
for (const bool swapped : {false, true}) {
if (null_buffers != 0u && swapped) continue;
if (layout != Layout::Shared && layout != Layout::IndexedNullChild && swapped) continue;
// Repeated keys select two distinct logical buffers within one native allocation.
// Keys below, between and above the live keys must not select either buffer.
compiled.resources.flattened_srt = {0u, 0u, 0u, 0u, 3u,
@@ -35669,9 +35721,15 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) {
12u, swapped ? 1u : 0u,
14u, swapped ? 0u : 1u};
test.expected = test.initial;
for (const u32 lane : {2u, 6u, 10u, 12u, 14u}) {
for (u32 lane = 0; lane < 16u; ++lane) {
if (lane == 4u || lane == 6u) continue;
if (layout == Layout::IndexedSingle) {
test.expected[test.storage_buffer_offsets[0] / 4u + lane] = 0x1000u + lane;
continue;
}
if (lane != 2u && lane != 10u && lane != 12u && lane != 14u) continue;
const u32 resource = (lane & 7u) == 4u ? swapped : !swapped;
if (null_buffers == 2u || (null_buffers == 1u && resource == 1u)) continue;
if (layout == Layout::AllNull || (null_child && resource == 1u)) continue;
test.expected[test.storage_buffer_offsets[resource] / 4u + lane] = 0x1000u + lane;
}
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());