diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index 5f23e283c..0d3fcbe55 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -63,6 +63,7 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe features.shaderBufferInt64Atomics = VK_TRUE; features.storageBuffer8BitAccess = VK_TRUE; features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE; + features.shaderStorageBufferArrayNonUniformIndexing = VK_TRUE; return features; } @@ -284,6 +285,9 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa check_feature(features12.shaderSampledImageArrayNonUniformIndexing, "shaderSampledImageArrayNonUniformIndexing", required_features12.shaderSampledImageArrayNonUniformIndexing); + check_feature(features12.shaderStorageBufferArrayNonUniformIndexing, + "shaderStorageBufferArrayNonUniformIndexing", + required_features12.shaderStorageBufferArrayNonUniformIndexing); check_feature(features13.robustImageAccess, "robustImageAccess"); check_feature(features13.dynamicRendering, "dynamicRendering", required_features13.dynamicRendering); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 57529c881..a8650235a 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -398,6 +398,8 @@ uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryIn Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem); +bool UsesSharedIndirectBufferStore(const EmitterState& state, uint32_t resource); + void EmitMemoryOffsets(EmitterState& state); uint32_t LdsDwordCount(const EmitterState& state); @@ -409,6 +411,7 @@ struct MemoryResourceAccess { uint32_t length = 0; uint32_t byte_offset = 0; uint32_t element_bits = 32; + bool non_uniform = false; spv::MemoryAccessMask memory_access = spv::MemoryAccessMaskNone; }; @@ -418,7 +421,8 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR:: MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, const IR::MemoryInfo& mem, - BufferDefinition& buffer); + BufferDefinition& buffer, + uint32_t array_index = 0); uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access, uint32_t index); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index d596c63b7..d18369ea6 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -249,10 +249,11 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3 }); } -uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { +uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + uint32_t prefix = 0) { if (mem.kind == IR::ResourceKind::Buffer) { const auto resource = ctx.state.program.info.buffers.at(mem.resource).descriptor_index; - const auto prefix = ctx.state.memory_byte_offsets[resource]; + if (prefix == 0u) prefix = ctx.state.memory_byte_offsets[resource]; const auto address = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), BufferByteAddress(ctx, inst, mem), prefix); // The host binding prefix must not wrap an out-of-range guest address into the buffer. @@ -273,8 +274,9 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo return ScratchByteAddress(ctx, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2)); } -uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { - auto address = ByteAddress(ctx, inst, mem); +uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + uint32_t prefix = 0) { + auto address = ByteAddress(ctx, inst, mem, prefix); if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) { // RDNA2 DS region addresses retain bits [15:2] after adding the byte offset. address = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address, @@ -540,8 +542,30 @@ void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) { void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t data) { - const auto index = DwordIndex(ctx, inst, mem); - EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { + auto& state = ctx.state; + const auto source = mem.kind == IR::ResourceKind::Buffer + ? state.program.info.buffers[mem.resource].source : UINT32_MAX; + const auto& sources = state.program.descriptor_sources; + const auto* indirect = source < sources.size() && sources[source].indirect_descriptor + ? &*sources[source].indirect_descriptor : nullptr; + uint32_t index = 0; + uint32_t in_bounds = 0; + if (indirect != nullptr && indirect->indexed_store) { + // The resource plan proves stride-four, index-only stores. Check the original + // index before any byte multiplication can wrap it into the buffer. + const auto prefix = Binary(state, spv::OpShiftRightLogical, TypeU32(state), + resource.byte_offset, ConstantU32(state, 2)); + const auto guest_index = ctx.Arg(inst, 1); + const auto records = Binary(state, spv::OpISub, TypeU32(state), resource.length, prefix); + in_bounds = AndCondition(state, + Binary(state, spv::OpULessThanEqual, TypeBool(state), prefix, resource.length), + Binary(state, spv::OpULessThan, TypeBool(state), guest_index, records)); + index = EmitAddU32(state, guest_index, prefix); + } else { + index = DwordIndex(ctx, inst, mem, resource.byte_offset); + in_bounds = EmitMemoryElementInBounds(state, resource, index); + } + EmitIfCondition(state, in_bounds, [&]() { StoreWordInBounds(ctx, resource, index, data); }); } @@ -576,6 +600,15 @@ void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) const auto selected = EmitIndirectResourceIndex( state, ctx.Arg(*handle, 0), buffer.indirect_mapping_offset, buffer.indirect_search_iterations, UINT32_MAX); + if (UsesSharedIndirectBufferStore(state, mem.resource)) { + EmitIfCondition(state, Binary(state, spv::OpULessThan, TypeBool(state), selected, + ConstantU32(state, buffer.indirect_resources.size())), [&]() { + const auto slot = EmitAddU32(state, selected, ConstantU32(state, buffer.descriptor_index)); + const auto access = PrepareStorageBufferResourceAccess(state, mem, state.storage_buffers[2], slot); + StoreWordPrepared(ctx, inst, mem, access, data); + }); + return; + } const auto merge = state.builder.AllocateId(); std::vector labels(buffer.indirect_resources.size()); std::vector branches {spv::OpSwitch, selected, merge}; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 1fbabf954..3e142eea7 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -2,14 +2,13 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { -uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) { +static uint32_t EmitShaderDataDwordLoadAt(EmitterState& state, uint32_t dword_index) { if (state.program.bindings.UsesPushData()) { - dword_index += state.program.bindings.push_data_start_dword; const auto pointer = state.builder.AllocateId(); const auto value = state.builder.AllocateId(); state.builder.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer, state.push_constant_variable, ConstantU32(state, 0), - ConstantU32(state, dword_index)); + dword_index); state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); return value; } @@ -18,13 +17,20 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) { const auto value = state.builder.AllocateId(); state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.shader_data_storage_variable, - ConstantU32(state, 0), ConstantU32(state, dword_index)); + ConstantU32(state, 0), dword_index); state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); return value; } return ConstantU32(state, 0); } +uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) { + if (state.program.bindings.UsesPushData()) { + dword_index += state.program.bindings.push_data_start_dword; + } + return EmitShaderDataDwordLoadAt(state, ConstantU32(state, dword_index)); +} + uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32_t rhs) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction(opcode, TypeU32(state), ret, lhs, rhs); @@ -102,12 +108,38 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR:: return state.program.info.buffers[mem.resource].descriptor_format; } +bool UsesSharedIndirectBufferStore(const EmitterState& state, uint32_t resource) { + const auto& buffers = state.program.info.buffers; + const auto& root = buffers[resource]; + if (root.indirect_root != resource || root.descriptor_index == UINT32_MAX || + root.packed_stride == 0u || root.indirect_resources.empty()) { + return false; + } + uint32_t slot = root.descriptor_index; + for (const auto candidate: root.indirect_resources) { + if (buffers[candidate].packed_stride != root.packed_stride || + buffers[candidate].descriptor_index != slot++) return false; + } + return true; +} + void EmitMemoryOffsets(EmitterState& state) { + std::array dynamic_offsets {}; + for (uint32_t resource = 0; resource < state.program.info.buffers.size(); ++resource) { + if (!UsesSharedIndirectBufferStore(state, resource)) continue; + for (const auto candidate: state.program.info.buffers[resource].indirect_resources) { + dynamic_offsets[state.program.info.buffers[candidate].descriptor_index] = true; + } + } const auto count = state.program.bindings.descriptor_counts[static_cast(IR::DescriptorBindingKind::Buffers)]; for (uint32_t i = 0; i < count; i += 4u) { + const auto lanes = std::min(4u, count - i); + if (std::all_of(dynamic_offsets.begin() + i, dynamic_offsets.begin() + i + lanes, + [](bool dynamic) { return dynamic; })) continue; const auto word = EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u); - for (uint32_t lane = 0; lane < std::min(4u, count - i); ++lane) { + for (uint32_t lane = 0; lane < lanes; ++lane) { + if (dynamic_offsets[i + lane]) continue; state.memory_byte_offsets[i + lane] = EmitNative( state, word, ConstantU32(state, lane * 8u), ConstantU32(state, 8u)); } @@ -152,7 +184,8 @@ void EnsureLdsStorage(EmitterState& state) { MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, const IR::MemoryInfo& mem, - BufferDefinition& buffer) { + BufferDefinition& buffer, + uint32_t array_index) { if (buffer.variable == 0) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, "storage buffer descriptor array was not emitted"); @@ -160,15 +193,34 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, if (buffer.pointer_type == 0) { buffer.pointer_type = TypePointer(state, spv::StorageClassStorageBuffer, buffer.type); } - const auto array_index = - state.program.info.buffers.at(mem.resource).descriptor_index; + const bool non_uniform = array_index != 0u; + const auto static_index = state.program.info.buffers.at(mem.resource).descriptor_index; + if (!non_uniform) array_index = ConstantU32(state, static_index); MemoryResourceAccess access { .kind = mem.kind, + .non_uniform = non_uniform, .memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone}; access.object_pointer = state.builder.AllocateId(); state.builder.AddFunction(spv::OpAccessChain, buffer.pointer_type, access.object_pointer, - buffer.variable, ConstantU32(state, array_index)); - access.byte_offset = state.memory_byte_offsets[array_index]; + buffer.variable, array_index); + if (non_uniform) { + state.builder.RequireExtension("SPV_EXT_descriptor_indexing"); + state.builder.RequireCapability(spv::CapabilityShaderNonUniform); + state.builder.RequireCapability(spv::CapabilityStorageBufferArrayNonUniformIndexing); + state.builder.AddAnnotation(spv::OpDecorate, access.object_pointer, spv::DecorationNonUniform); + const auto offsets_start = state.program.bindings.memory_offset_dword + + (state.program.bindings.UsesPushData() ? state.program.bindings.push_data_start_dword : 0u); + const auto word = EmitShaderDataDwordLoadAt(state, EmitAddU32( + state, ConstantU32(state, offsets_start), + Binary(state, spv::OpShiftRightLogical, TypeU32(state), array_index, ConstantU32(state, 2)))); + const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), + Binary(state, spv::OpBitwiseAnd, TypeU32(state), array_index, ConstantU32(state, 3)), + ConstantU32(state, 3)); + access.byte_offset = EmitNative( + state, word, shift, ConstantU32(state, 8)); + } else { + access.byte_offset = state.memory_byte_offsets[static_index]; + } access.length = state.builder.AllocateId(); state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0); @@ -260,6 +312,9 @@ uint32_t EmitStorageBufferElementPointer(EmitterState& state, const auto pointer = state.builder.AllocateId(); state.builder.AddFunction(spv::OpAccessChain, pointer_type, pointer, access.object_pointer, ConstantU32(state, 0), index); + if (access.non_uniform) { + state.builder.AddAnnotation(spv::OpDecorate, pointer, spv::DecorationNonUniform); + } return pointer; } diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index e0c7019d8..8b059e698 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -504,6 +504,7 @@ struct DescriptorSource { uint32_t source = UINT32_MAX; uint32_t stride = 0; uint32_t offset = 0; + std::vector> byte_ranges; bool operator==(const SelectorRead& other) const = default; }; @@ -514,6 +515,7 @@ struct DescriptorSource { uint32_t table_stride = 0; uint32_t table_record_bytes = 0; bool table_scalar = false; + bool indexed_store = false; uint32_t workgroup_axis = UINT32_MAX; Value key_count; Value selector_first; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 855494287..0ce82647e 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -195,15 +195,15 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset, runtime.read_specialization_memory(runtime.userdata, address, prefix); } -bool NormalizeIndirectStoreBuffer(DescriptorValue& value) { +bool NormalizeIndirectStoreBuffer(DescriptorValue& value, bool indexed) { ShaderBufferResource descriptor; if (!DecodeBufferDescriptor(value, descriptor)) return false; if (descriptor.Type() != 0u || descriptor.Format() == Prospero::BufferFormat::kInvalid || descriptor.Base48() == 0u || descriptor.GetSize() == 0u) { value.dwords.fill(0u); - } else if (descriptor.Stride() != 1u || descriptor.SwizzleEnabled() || descriptor.AddTid() || - descriptor.OutOfBounds() != 3u) { - return SpecializationFail("indirect buffer stores require linear byte bounds"); + } else if (descriptor.Stride() != (indexed ? 4u : 1u) || descriptor.SwizzleEnabled() || + descriptor.AddTid() || descriptor.OutOfBounds() != (indexed ? 0u : 3u)) { + return SpecializationFail("indirect buffer stores require exact linear DWORD or byte bounds"); } return true; } @@ -337,6 +337,21 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p } keys.resize(key_count); std::iota(keys.begin(), keys.end(), 0u); + } else if (!selector->byte_ranges.empty()) { + if (material_value.dword_count != 2u) return false; + const auto base = (uint64_t {material_value.dwords[1]} << 32u) | material_value.dwords[0]; + std::array selected {}; + for (const auto [offset, size]: selector->byte_ranges) { + if (size > MaxIndirectDescriptorProbes || ((offset | size) & 3u) != 0u) return false; + keys.resize(size / 4u); + if (!ReadScalarTable(base, UINT64_MAX, offset, runtime, keys)) return false; + for (const auto word: keys) + for (uint32_t shift = 0; shift < 32u; shift += 8u) + selected[(word >> shift) & 255u] = true; + } + keys.clear(); + for (uint32_t key = 0; key < selected.size(); ++key) + if (selected[key]) keys.push_back(key); } else if (!indirect.selector_first.IsEmpty()) { ShaderBufferResource material; uint32_t first = 0, count = 0; @@ -380,7 +395,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p mask &= mask - 1u; } } else return false; - if (selector != nullptr) { + if (selector != nullptr && selector->byte_ranges.empty()) { std::ranges::sort(keys); keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); } @@ -1132,7 +1147,9 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime if (!MaterializeIndirectDescriptor( program, *source->indirect_descriptor, i, 4u, observed, clean, snapshot, snapshot.buffers, specialization.buffers, ShaderInfo::MaxBuffers, - NormalizeIndirectStoreBuffer)) + [&](DescriptorValue& value) { + return NormalizeIndirectStoreBuffer(value, source->indirect_descriptor->indexed_store); + })) return false; } } else if (!evaluate(base.source, snapshot.buffers[i], base.written)) { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 7a14d5443..2504ea481 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -909,7 +909,7 @@ private: const auto& b = *descriptor.indirect_descriptor; if (a.selector != b.selector || a.table_source != b.table_source || a.table_offset != b.table_offset || a.table_immediate != b.table_immediate || - a.table_stride != b.table_stride || + a.table_stride != b.table_stride || a.indexed_store != b.indexed_store || a.workgroup_axis != b.workgroup_axis || a.sources != b.sources || !EquivalentValue(m_program, a.key_count, b.key_count) || a.selector_first.IsEmpty() != b.selector_first.IsEmpty() || @@ -1821,17 +1821,71 @@ private: return true; } + bool MatchPackedScalarKey(Value key, DescriptorSource::IndirectDescriptor& indirect) { + const auto* mask = key.Resolve().TryInstruction(); + uint32_t bits = 0; + if (mask == nullptr || mask->GetOpcode() != ValueOpcode::BitwiseAnd32 || + !ImmediateU32(mask->Arg(1), bits) || bits != 255u) return false; + const auto* shift = mask->Arg(0).Resolve().TryInstruction(); + if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftRightLogical32 || + (PossibleU32Bits(shift->Arg(1)) & ~24u) != 0u) return false; + DescriptorSource::IndirectDescriptor::SelectorRead selector; + std::vector pending {shift->Arg(0).Resolve().TryInstruction()}; + std::vector visited; + while (!pending.empty()) { + const auto* read = pending.back(); + pending.pop_back(); + if (read == nullptr) return false; + if (std::ranges::find(visited, read) != visited.end()) continue; + visited.push_back(read); + if (read->GetOpcode() == ValueOpcode::Phi) { + for (size_t arm = 0; arm < read->NumArgs(); ++arm) + pending.push_back(read->Arg(arm).Resolve().TryInstruction()); + continue; + } + uint32_t index = 0; + const auto* memory = ScalarReadMemory(*read, index); + DescriptorSource source; + if (memory == nullptr || memory->kind != ResourceKind::ScalarAddress || + memory->offset > INT32_MAX || !MakeRuntimeTableSource(*read, source)) return false; + const auto maximum = PossibleU32Bits(read->Arg(1)) & ~3u; + const auto offset = memory->offset & ~3u; + if (uint64_t {offset} + maximum + 4u > UINT32_MAX) return false; + const auto source_index = InternSource(source); + if (selector.source != UINT32_MAX && selector.source != source_index) return false; + selector.source = source_index; + selector.byte_ranges.emplace_back(offset, maximum + 4u); + } + if (selector.byte_ranges.empty()) return false; + // Scalar reads align both offsets independently. Merge the possible read spans; + // the GPU retains the actual byte selection and the host only discovers V#s. + std::ranges::sort(selector.byte_ranges); + for (size_t i = 1; i < selector.byte_ranges.size();) { + auto& previous = selector.byte_ranges[i - 1u]; + const auto current = selector.byte_ranges[i]; + if (current.first > previous.first + previous.second) { ++i; continue; } + previous.second = std::max(previous.second, current.first + current.second - previous.first); + selector.byte_ranges.erase(selector.byte_ranges.begin() + i); + } + indirect.selector = std::move(selector); + return true; + } + bool TryMakeIndirectBuffer(Inst& handle, const DescriptorSource& descriptor, IndirectDescriptorPlan& plan) { if (handle.GetOpcode() != ValueOpcode::GetBufferResource || handle.NumArgs() != 4u || handle.Uses().empty()) return false; + if (handle.Uses().front().user->GetOpcode() != ValueOpcode::StoreBufferU32) return false; + const auto indexed = m_program.memory_info[handle.Uses().front().user->Flags().index].idxen; for (const auto& use: handle.Uses()) { const auto& inst = *use.user; if (inst.GetOpcode() != ValueOpcode::StoreBufferU32) return false; const auto& memory = m_program.memory_info[inst.Flags().index]; uint32_t scalar = 1u; if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted || - memory.idxen || !memory.offen || !ImmediateU32(inst.Arg(3), scalar) || scalar != 0u) + memory.idxen != indexed || memory.offen == indexed || + (indexed && (memory.offset != 0u || inst.Arg(2).Resolve() != Value(0u))) || + !ImmediateU32(inst.Arg(3), scalar) || scalar != 0u) return false; } DescriptorSource table_source; @@ -1840,8 +1894,26 @@ private: uint32_t table_stride = 0; uint32_t table_immediate = 0; if (!MatchDescriptorTable(handle, descriptor, plan, table_source, key, table_offset, - table_stride, table_immediate) || table_stride != 16u || - table_source.dword_count != 4u || table_offset != 0u || table_immediate != 0u) return false; + table_stride, table_immediate) || table_stride != 16u) return false; + DescriptorSource::IndirectDescriptor indirect; + indirect.table_stride = table_stride; + indirect.table_offset = table_offset; + indirect.table_immediate = table_immediate; + indirect.indexed_store = indexed; + const auto finish = [&] { + indirect.table_source = InternSource(table_source); + DescriptorSource source; + source.dword_count = 4u; + source.dwords.fill(Value(0u)); + source.indirect_descriptor = std::move(indirect); + plan.handle = &handle; + plan.source = InternSource(source); + plan.key = key; + return true; + }; + if (table_source.dword_count == 2u && MatchPackedScalarKey(key, indirect)) return finish(); + if (table_source.dword_count != 4u || table_offset != 0u || table_immediate != 0u || indexed) + return false; const auto* lane_read = key.Resolve().TryInstruction(); if (lane_read == nullptr || lane_read->GetOpcode() != ValueOpcode::ReadFirstLane) return false; @@ -1869,21 +1941,11 @@ private: !ImmediateU32(read->Arg(3), scalar) || scalar != 0u || !EquivalentValue(m_program, selected->Arg(0), read->Arg(4))) return false; DescriptorSource material_source; - DescriptorSource::IndirectDescriptor indirect; - indirect.table_stride = table_stride; if (!MakeRuntimeTableSource(*read, material_source) || !BoundedSelectedIndex(read->Arg(1), read->Arg(4), indirect)) return false; indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead{.source = InternSource(material_source), .offset = memory.offset + lane * 4u}); - indirect.table_source = InternSource(table_source); - DescriptorSource source; - source.dword_count = 4u; - source.dwords.fill(Value(0u)); - source.indirect_descriptor = indirect; - plan.handle = &handle; - plan.source = InternSource(source); - plan.key = key; - return true; + return finish(); } using SelectorBounds = std::span>; diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index bb66e88f7..215294a4a 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -2427,6 +2427,115 @@ void TestWritableDescriptorPhi() { "control-dependent writable descriptor phi was not rejected transactionally"); } +void TestPackedScalarBufferSelection() { + const auto make = [](uint32_t shift_mask = 24u, bool cycle = false, uint32_t store_offset = 0u, + bool byte_store = false) { + auto fixture = std::make_unique(); + auto& f = *fixture; + auto* entry = f.block; + auto* left = f.AddBlock(); + auto* right = f.AddBlock(); + auto* merge = f.AddBlock(); + entry->AddBranch(left); + entry->AddBranch(right); + left->AddBranch(merge); + right->AddBranch(merge); + const auto base = f.Address(f.UserData(0u), f.UserData(1u)); + const auto lane = f.Emit(ValueOpcode::LaneId); + const auto offset = f.Emit(ValueOpcode::BitwiseAnd32, {lane, Value(511u)}); + const auto shift = f.Emit(ValueOpcode::BitwiseAnd32, {lane, Value(shift_mask)}); + auto& phi = merge->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); + for (const auto [block, immediate] : {std::pair{left, 0x1c1u}, std::pair{right, 0x3c2u}}) { + const auto read = f.Emit(ValueOpcode::LoadAddressU32, + {base, offset, Value(0u), Value(true)}, + f.AddMemory({.kind = ResourceKind::ScalarAddress, .offset = immediate}, immediate), block); + phi.AddPhiOperand(block, cycle ? Value(&phi) : read); + } + f.block = merge; + const auto word = f.Emit(ValueOpcode::ShiftRightLogical32, {Value(&phi), shift}); + const auto key = f.Emit(ValueOpcode::BitwiseAnd32, {word, Value(255u)}); + const auto scaled = f.Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(4u)}); + const auto table_offset = f.Emit(ValueOpcode::IAdd32, {scaled, Value(64u)}); + std::array descriptor; + for (uint32_t component = 0; component < descriptor.size(); ++component) { + descriptor[component] = f.Emit(ValueOpcode::LoadAddressU32, + {base, table_offset, Value(0u), Value(true)}, + f.AddMemory({.kind = ResourceKind::ScalarAddress, .offset = component * 4u, + .component_index = component, .component_count = 4u}, 0x2b4u)); + } + f.Emit(ValueOpcode::StoreBufferU32, + {f.Buffer(descriptor), lane, Value(0u), Value(0u), Value(42u), Value(true)}, + f.AddMemory({.kind = ResourceKind::Buffer, .offset = store_offset, .idxen = true}, 0x2c8u)); + if (byte_store) + f.Emit(ValueOpcode::StoreBufferU32, + {f.Buffer(descriptor), Value(0u), lane, Value(0u), Value(43u), Value(true)}, + f.AddMemory({.kind = ResourceKind::Buffer, .offen = true}, 0x2d0u)); + return fixture; + }; + auto fixture = make(); + fixture->PlanAndTrack(); + auto plan = ExtractResourcePlan(fixture->program); + const auto& source = plan.descriptor_sources[plan.info.buffers[0].source]; + Check(source.indirect_descriptor && source.indirect_descriptor->indexed_store && + source.indirect_descriptor->selector->byte_ranges == + std::vector>{{0x1c0u, 1024u}}, + "packed scalar lookup lost a Phi arm or independent DWORD alignment"); + LinearTestMemory memory; + memory.watched_address = memory.base + 0x1c0u; + for (uint32_t key = 0; key < 19u; ++key) { + const auto descriptor = 16u + key * 4u; + memory.words[descriptor] = 0x100000u + key * 0x10000u; + memory.words[descriptor + 1u] = 4u << 16u; + memory.words[descriptor + 2u] = 14400u; + memory.words[descriptor + 3u] = 0x5204u; + } + for (uint32_t i = 0; i < 256u; ++i) { + for (uint32_t byte = 0; byte < 4u; ++byte) + memory.words[0x1c0u / 4u + i] |= ((i * 4u + byte) % 19u) << (byte * 8u); + } + // No descriptor exists after key 18; broad enumeration would read this address. + memory.fail_address = memory.base + 0x170u; + const std::array user_data{static_cast(memory.base), 0u}; + const SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, + .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + const auto materialize = [&] { return MaterializeResources(plan, runtime, snapshot, specialization); }; + Check(materialize() && snapshot.buffers.size() == 19u && memory.reads == 20u && + memory.watched_reads == 1u && memory.watched_dwords == 256u, + "packed scalar lookup did not batch its read and deduplicate writable candidates"); + Check(std::ranges::find(snapshot.specialization_reads, + std::pair{memory.base + 0x1c0u, uint64_t{1024u}}) != snapshot.specialization_reads.end() && + std::ranges::all_of(snapshot.buffers, [&](const DescriptorValue& value) { + const auto key = (value.dwords[0] - 0x100000u) / 0x10000u; + return std::ranges::find(snapshot.specialization_reads, + std::pair{memory.base + 64u + key * 16u, uint64_t{16u}}) != snapshot.specialization_reads.end(); + }), + "packed lookup or writable descriptors escaped specialization overlap tracking"); + const auto initial = specialization; + memory.words[16] += 4096u; + memory.reads = memory.watched_reads = 0u; + Check(materialize() && specialization == initial && memory.reads == 20u, + "packed table refresh changed shader specialization or repeated scalar reads"); + memory.words[17] = 8u << 16u; + Check(!materialize(), "indexed DWORD write accepted an incompatible descriptor stride"); + memory.words[17] = 4u << 16u; + memory.words[19] |= 3u << 28u; + Check(!materialize(), "indexed DWORD write accepted raw byte bounds"); + for (auto bad : {0u, 1u, 2u}) { + auto invalid = make(bad == 0u ? 28u : 24u, bad == 1u, bad == 2u ? 4u : 0u); + CheckFatal([&] { invalid->PlanAndTrack(); }, "not a valid runtime value", + "packed selector accepted a cross-byte shift, leafless cycle, or indexed byte offset"); + } + auto mixed = make(24u, false, 0u, true); + mixed->PlanAndTrack(); + const auto& buffers = mixed->program.info.buffers; + Check(buffers.size() == 2u && buffers[0].source != buffers[1].source && + mixed->program.descriptor_sources[buffers[0].source].indirect_descriptor->indexed_store && + !mixed->program.descriptor_sources[buffers[1].source].indirect_descriptor->indexed_store, + "indexed and byte-addressed stores incorrectly shared their descriptor bounds contract"); +} + ResourcePlan ConditionalSamplerPlan(bool diamond, bool reverse, bool reverse_phi, bool nonuniform = false, bool writable = false) { @@ -3732,6 +3841,7 @@ int main() { Run("SRT runtime", TestSrtFlatteningAndRuntimeMemoization); Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit); Run("writable descriptor phi", TestWritableDescriptorPhi); + Run("packed scalar buffer selection", TestPackedScalarBufferSelection); Run("conditional sampler phi", TestConditionalSamplerPhi); Run("finite image phi cycle", TestFiniteImagePhiCycle); Run("finite inline sampler", TestFiniteInlineSampler); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index a42d7abc8..4adf61a39 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -18812,6 +18812,7 @@ private: features12.shaderOutputViewportIndex != true || features12.shaderBufferInt64Atomics != true || features12.shaderSampledImageArrayNonUniformIndexing != true || + features12.shaderStorageBufferArrayNonUniformIndexing != true || features12.shaderSharedInt64Atomics != true || image_atomic64.shaderImageInt64Atomics != true || workgroup_layout.workgroupMemoryExplicitLayout != true || @@ -18893,6 +18894,9 @@ private: Require("VulkanHarness", "dispatch", available_features12.shaderSampledImageArrayNonUniformIndexing == true, "nonuniform sampled image indexing is not supported"); + Require("VulkanHarness", "dispatch", + available_features12.shaderStorageBufferArrayNonUniformIndexing == true, + "nonuniform storage buffer indexing is not supported"); Require("VulkanHarness", "dispatch", available_min_lod.minLod == true, "image view minimum LOD is not supported"); Require("VulkanHarness", "graphics", available_features12.shaderOutputLayer == true, @@ -35580,8 +35584,6 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { test.initial.assign(64, 0xdeadbeefu); test.storage_buffer_range_bytes = 64; test.storage_buffer_offsets = {12, 108}; - test.required_spirv = {"OpSwitch"}; - test.forbidden_spirv = {"PhysicalStorageBuffer"}; CompiledShader compiled; auto &program = compiled.program; @@ -35603,13 +35605,19 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { ValueOpcode::GetBufferResource, {Value(&key), Value(0u), Value(0u), Value(0u)}); handle.SetFlags(0u); - auto &offset = block->AppendNewInst(ValueOpcode::IMul32, - {Value(&lane), Value(4u)}); + auto &lane_offset = block->AppendNewInst(ValueOpcode::IMul32, + {Value(&lane), Value(4u)}); + auto &overflow = block->AppendNewInst(ValueOpcode::IEqual32, + {Value(&lane), Value(6u)}); + auto &offset = block->AppendNewInst(ValueOpcode::SelectU32, + {Value(&overflow), Value(UINT32_MAX), Value(&lane_offset)}); + auto &index = block->AppendNewInst(ValueOpcode::SelectU32, + {Value(&overflow), Value(0x40000000u), Value(&lane)}); auto &data = block->AppendNewInst(ValueOpcode::IAdd32, {Value(&lane), Value(0x1000u)}); auto &exec = block->AppendNewInst(ValueOpcode::INotEqual32, {Value(&lane), Value(4u)}); - block->AppendNewInst(ValueOpcode::StoreBufferU32, + auto &store = block->AppendNewInst(ValueOpcode::StoreBufferU32, {Value(&handle), Value(0u), Value(&offset), Value(0u), Value(&data), Value(&exec)}); program.memory_info.push_back({.kind = ResourceKind::Buffer, .offen = true}); @@ -35632,36 +35640,80 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { compute.host_subgroup_size = vulkan.SubgroupSize(); compute.threads_num[0] = 32; compute.threads_num[1] = compute.threads_num[2] = 1; - for (const u32 null_buffers : {0u, 1u, 2u}) { + enum class Layout { Shared, NullChild, AllNull, DifferentStride, ShiftedPushData, StorageData, + Indexed, IndexedNullChild, IndexedSingle }; + for (const auto layout : {Layout::Shared, Layout::NullChild, Layout::AllNull, + Layout::DifferentStride, Layout::ShiftedPushData, Layout::StorageData, + Layout::Indexed, Layout::IndexedNullChild, Layout::IndexedSingle}) { + const bool indexed = layout == Layout::Indexed || layout == Layout::IndexedNullChild || + layout == Layout::IndexedSingle; + const bool null_child = layout == Layout::NullChild || layout == Layout::IndexedNullChild; + const bool single = layout == Layout::AllNull || layout == Layout::IndexedSingle; program.info.buffers = {root, candidate}; - if (null_buffers != 0u) program.info.buffers[1].packed_stride = 0; - if (null_buffers == 2u) { + if (indexed) { + for (auto &buffer : program.info.buffers) buffer.packed_stride = 4; + } + program.descriptor_sources[0].indirect_descriptor->indexed_store = indexed; + program.memory_info[0].idxen = indexed; + program.memory_info[0].offen = !indexed; + store.SetArg(1, indexed ? Value(&index) : Value(0u)); + store.SetArg(2, indexed ? Value(0u) : Value(&offset)); + if (null_child) program.info.buffers[1].packed_stride = 0; + if (layout == Layout::DifferentStride) program.info.buffers[1].packed_stride = 2; + if (single) { // Materialization deduplicates all null destinations into one unmapped root. program.info.buffers.resize(1); - program.info.buffers[0].packed_stride = 0; + if (layout == Layout::AllNull) program.info.buffers[0].packed_stride = 0; program.info.buffers[0].indirect_root = BufferResource::NoIndirectBuffer; program.info.buffers[0].indirect_search_iterations = 0; program.info.buffers[0].indirect_resources.clear(); } program.shader_info_complete = false; CollectShaderInfo(program, {.compute = &compute}); - compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); + const auto push_start = layout == Layout::ShiftedPushData ? 3u : + layout == Layout::StorageData ? PushData::NoStart : 0u; + compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}, push_start); Require(test.name, "candidate bindings", - program.bindings.descriptor_counts[0] == (null_buffers == 2u ? 1u : 2u) && + program.bindings.descriptor_counts[0] == (single ? 1u : 2u) && program.info.buffers[0].descriptor_index == 0u && - (null_buffers == 2u || program.info.buffers[1].descriptor_index == 1u), + (single || program.info.buffers[1].descriptor_index == 1u), "indirect store lost a buffer binding"); compiled.packed_user_data.resize(program.bindings.ShaderDataDwords()); compiled.packed_user_data[program.bindings.memory_offset_dword] = 12u | (108u << 8u); ValidateSpirv(test.name, compiled.spirv); - if (null_buffers == 2u) { - test.required_spirv.clear(); + test.required_spirv.clear(); + test.forbidden_spirv = {"PhysicalStorageBuffer"}; + if (!single && !null_child && layout != Layout::DifferentStride) { + test.required_spirv = {"StorageBufferArrayNonUniformIndexing"}; test.forbidden_spirv.push_back("OpSwitch"); + std::set nonuniform; + std::vector accessed_pointers; + u32 prefix_extracts = 0; + for (size_t offset = 5; offset < compiled.spirv.size();) { + const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u); + const auto opcode = static_cast(words[0] & 0xffffu); + if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform) + nonuniform.insert(words[1]); + if (opcode == spv::OpArrayLength) accessed_pointers.push_back(words[3]); + if (opcode == spv::OpStore) accessed_pointers.push_back(words[1]); + if (opcode == spv::OpBitFieldUExtract) ++prefix_extracts; + offset += words.size(); + } + Require(test.name, "shared store and nonuniform pointers", + prefix_extracts == 1u && accessed_pointers.size() == 2u && + std::ranges::all_of(accessed_pointers, [&](u32 pointer) { + return nonuniform.contains(pointer); + }), + "indirect store retained static prefixes, was duplicated or lost a NonUniform pointer decoration"); + } else if (single) { + test.forbidden_spirv.push_back("OpSwitch"); + } else { + test.required_spirv = {"OpSwitch"}; } CheckSpirvText(test, compiled.spirv); for (const bool swapped : {false, true}) { - if (null_buffers != 0u && swapped) continue; + if (layout != Layout::Shared && layout != Layout::IndexedNullChild && swapped) continue; // Repeated keys select two distinct logical buffers within one native allocation. // Keys below, between and above the live keys must not select either buffer. compiled.resources.flattened_srt = {0u, 0u, 0u, 0u, 3u, @@ -35669,9 +35721,15 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { 12u, swapped ? 1u : 0u, 14u, swapped ? 0u : 1u}; test.expected = test.initial; - for (const u32 lane : {2u, 6u, 10u, 12u, 14u}) { + for (u32 lane = 0; lane < 16u; ++lane) { + if (lane == 4u || lane == 6u) continue; + if (layout == Layout::IndexedSingle) { + test.expected[test.storage_buffer_offsets[0] / 4u + lane] = 0x1000u + lane; + continue; + } + if (lane != 2u && lane != 10u && lane != 12u && lane != 14u) continue; const u32 resource = (lane & 7u) == 4u ? swapped : !swapped; - if (null_buffers == 2u || (null_buffers == 1u && resource == 1u)) continue; + if (layout == Layout::AllNull || (null_child && resource == 1u)) continue; test.expected[test.storage_buffer_offsets[resource] / 4u + lane] = 0x1000u + lane; } auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());