From 26d326d1b0e9bdc1b424e9b18d38a70032c86aec Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 14 Oct 2026 05:07:24 +0200 Subject: [PATCH] shader: share bounded record descriptors for packed loads and byte stores Co-authored-by: fxpw --- .../recompiler/backend/spirv/SpirvEmitter.cpp | 5 + .../backend/spirv/spirvEmitterInstructions.h | 1 + .../backend/spirv/spirvEmitterInternal.h | 5 +- .../backend/spirv/spirvEmitterMemory.cpp | 207 +++++++++++------- .../spirv/spirvEmitterMemoryHelpers.cpp | 24 +- src/graphics/shader/recompiler/ir/ShaderIR.h | 2 +- .../recompiler/ir/opcodes/ValueOpcodes.inc | 1 + .../ir/passes/ResourceMaterialization.cpp | 44 +++- .../recompiler/ir/passes/ResourceTracking.cpp | 106 ++++++--- .../ir/passes/ShaderInfoCollection.cpp | 13 +- tests/ResourceTrackingTests.cpp | 178 ++++++++++++++- tests/ShaderRecompilerComputeTests.cpp | 137 +++++++++++- 12 files changed, 581 insertions(+), 142 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 9ac26195d..973edce2d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -105,6 +105,11 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) { Fail(program, "typed buffer handle has an invalid dense resource"); } break; + case IR::ValueOpcode::GetBufferResourceIndex: + if (dense >= program.info.buffers.size()) { + Fail(program, "typed buffer selector has an invalid dense resource"); + } + break; case IR::ValueOpcode::GetAddressResource: if (planning_only_handle(inst)) { break; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index 985ab8a61..f06c8b2c5 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -246,6 +246,7 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& in uint32_t EmitGetShaderBase(ValueEmitContext& ctx); inline constexpr auto EmitGetSrtResource = EmitVoid; inline constexpr auto EmitGetBufferResource = EmitGetSrtResource; +uint32_t EmitGetBufferResourceIndex(ValueEmitContext& ctx, const IR::Inst& inst); inline constexpr auto EmitGetAddressResource = EmitGetSrtResource; inline constexpr auto EmitGetScratchResource = EmitGetSrtResource; inline constexpr auto EmitGetImageResource = EmitGetSrtResource; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 3b77345d7..6f4952c28 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -399,7 +399,7 @@ uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryIn Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem); -bool UsesSharedIndirectBufferStore(const EmitterState& state, uint32_t resource); +bool UsesSharedIndirectBufferAccess(const EmitterState& state, uint32_t resource); void EmitMemoryOffsets(EmitterState& state); @@ -418,7 +418,8 @@ struct MemoryResourceAccess { uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset, uint32_t search_iterations, uint32_t default_resource); -MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem); +MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem, + uint32_t array_index = 0u); MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, const IR::MemoryInfo& mem, diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index d818b2e76..e119ca463 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -286,6 +286,85 @@ uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& res uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t address, uint32_t index, uint32_t bits, bool sign_extend); +template +auto EmitBufferAccess(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, + uint32_t components, Fn&& emit) { + auto& state = ctx.state; + constexpr bool has_result = !std::is_void_v< + std::invoke_result_t>; + const auto access = [&](uint32_t resource) { + mem.resource = resource; + if (mem.kind == IR::ResourceKind::Buffer) { + const auto& buffer = state.program.info.buffers[resource]; + if (buffer.packed_stride == 0u && + buffer.source < state.program.descriptor_sources.size() && + state.program.descriptor_sources[buffer.source].indirect_descriptor) { + // Empty candidates still carry the format/swizzle needed for OOB loads. + return emit(mem, MemoryResourceAccess{}); + } + } + return emit(mem, PrepareMemoryResourceAccess(state, mem)); + }; + if (mem.kind != IR::ResourceKind::Buffer || + state.program.info.buffers[mem.resource].indirect_root != mem.resource) { + return access(mem.resource); + } + const auto& buffer = state.program.info.buffers[mem.resource]; + const auto& handle = *inst.Arg(0).ResolveInstruction(); + const auto selected = ctx.Arg(handle, 0); + const auto shared = UsesSharedIndirectBufferAccess(state, mem.resource); + const auto selected_access = [&]() { + if (shared) { + const auto root_slot = buffer.descriptor_index; + const auto child_slot = state.program.info.buffers[buffer.indirect_resources[1]].descriptor_index; + auto slot = EmitAddU32(state, selected, ConstantU32(state, child_slot - 1u)); + if (child_slot != root_slot + 1u) { + slot = Select(state, TypeU32(state), + Binary(state, spv::OpIEqual, TypeBool(state), selected, ConstantU32(state, 0u)), + ConstantU32(state, root_slot), slot); + } + return emit(mem, PrepareMemoryResourceAccess(state, mem, slot)); + } + return EmitIndexSwitch(state, selected, buffer.indirect_resources.size(), + has_result ? TypeU32Composite(state, components) : 0u, + [&](uint32_t item) { return access(buffer.indirect_resources[item]); }); + }; + if constexpr (!has_result) { + if (!shared) return selected_access(); + } + const auto valid = Binary(state, spv::OpULessThan, TypeBool(state), selected, + ConstantU32(state, buffer.indirect_resources.size())); + if constexpr (has_result) { + return EmitValueOrDefaultIfCondition(state, valid, TypeU32Composite(state, components), + ConstantU32CompositeZero(state, components), selected_access); + } else { + EmitIfCondition(state, valid, selected_access); + } +} + +uint32_t BufferIndexInBounds(ValueEmitContext& ctx, const IR::Inst& inst, + const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, + uint32_t payload_in_bounds) { + if (mem.kind != IR::ResourceKind::Buffer) return payload_in_bounds; + auto& state = ctx.state; + const auto& buffer = state.program.info.buffers[mem.resource]; + const auto& sources = state.program.descriptor_sources; + if (buffer.source >= sources.size() || !sources[buffer.source].indirect_descriptor || + !sources[buffer.source].indirect_descriptor->indexed) return payload_in_bounds; + const auto bytes = Binary(state, spv::OpIMul, TypeU32(state), resource.length, + ConstantU32(state, resource.element_bits / 8u)); + const auto available = Binary(state, spv::OpISub, TypeU32(state), bytes, resource.byte_offset); + // Test the original index before multiplication can wrap. A truncated binding + // may omit final-record padding; the payload check below still checks every byte. + const auto last_record = Binary(state, spv::OpUDiv, TypeU32(state), + Binary(state, spv::OpISub, TypeU32(state), available, ConstantU32(state, 1u)), + ConstantU32(state, buffer.packed_stride & 0x3fffu)); + return AndCondition(state, payload_in_bounds, + AndCondition(state, + Binary(state, spv::OpUGreaterThan, TypeBool(state), bytes, resource.byte_offset), + Binary(state, spv::OpULessThanEqual, TypeBool(state), ctx.Arg(inst, 1), last_record))); +} + uint32_t LoadWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource) { const auto index = DwordIndex(ctx, inst, mem, resource.byte_offset); @@ -466,20 +545,23 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, void StoreSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t bits, uint32_t data) { + if (resource.object_pointer == 0u) return; const auto address = ByteAddress(ctx, inst, mem, resource.byte_offset); const auto index = Binary( ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address, ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u))); EmitIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { + ctx.state, BufferIndexInBounds(ctx, inst, mem, resource, + EmitMemoryElementInBounds(ctx.state, resource, index)), [&]() { StoreSubwordInBounds(ctx, mem, resource, address, index, bits, data); }); } void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits) { EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - StoreSubwordPrepared(ctx, inst, mem, resource, bits, ctx.Arg(inst, inst.NumArgs() - 2)); + EmitBufferAccess(ctx, inst, mem, 0u, [&](const auto& selected, const auto& resource) { + StoreSubwordPrepared(ctx, inst, selected, resource, bits, ctx.Arg(inst, inst.NumArgs() - 2)); + }); }); } @@ -536,75 +618,20 @@ void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) { void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t data) { - auto& state = ctx.state; - const auto source = mem.kind == IR::ResourceKind::Buffer - ? state.program.info.buffers[mem.resource].source : UINT32_MAX; - const auto& sources = state.program.descriptor_sources; - const auto* indirect = source < sources.size() && sources[source].indirect_descriptor - ? &*sources[source].indirect_descriptor : nullptr; - uint32_t index = 0; - uint32_t in_bounds = 0; - if (indirect != nullptr && indirect->indexed_store) { - // The resource plan proves stride-four, index-only stores. Check the original - // index before any byte multiplication can wrap it into the buffer. - const auto prefix = Binary(state, spv::OpShiftRightLogical, TypeU32(state), - resource.byte_offset, ConstantU32(state, 2)); - const auto guest_index = ctx.Arg(inst, 1); - const auto records = Binary(state, spv::OpISub, TypeU32(state), resource.length, prefix); - in_bounds = AndCondition(state, - Binary(state, spv::OpULessThanEqual, TypeBool(state), prefix, resource.length), - Binary(state, spv::OpULessThan, TypeBool(state), guest_index, records)); - index = EmitAddU32(state, guest_index, prefix); - } else { - index = DwordIndex(ctx, inst, mem, resource.byte_offset); - in_bounds = EmitMemoryElementInBounds(state, resource, index); - } - EmitIfCondition(state, in_bounds, [&]() { + if (resource.object_pointer == 0u) return; + const auto index = DwordIndex(ctx, inst, mem, resource.byte_offset); + const auto in_bounds = BufferIndexInBounds(ctx, inst, mem, resource, + EmitMemoryElementInBounds(ctx.state, resource, index)); + EmitIfCondition(ctx.state, in_bounds, [&]() { StoreWordInBounds(ctx, resource, index, data); }); } void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) { - auto& state = ctx.state; - EmitIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto data = ctx.Arg(inst, inst.NumArgs() - 2); - const auto store = [&](uint32_t resource) { - if (mem.kind == IR::ResourceKind::Buffer) { - const auto& buffer = state.program.info.buffers[resource]; - if (buffer.packed_stride == 0u && - buffer.source < state.program.descriptor_sources.size() && - state.program.descriptor_sources[buffer.source].indirect_descriptor) return; - } - mem.resource = resource; - const auto access = PrepareMemoryResourceAccess(state, mem); - StoreWordPrepared(ctx, inst, mem, access, data); - }; - if (mem.kind != IR::ResourceKind::Buffer || - state.program.info.buffers[mem.resource].indirect_root != mem.resource) { - store(mem.resource); - return; - } - const auto& buffer = state.program.info.buffers[mem.resource]; - const auto* handle = inst.Arg(0).ResolveInstruction(); - if (handle == nullptr || buffer.indirect_resources.empty() || - state.flattened_srt_variable == 0 || buffer.indirect_search_iterations == 0u) { - ctx.Fail(inst, "has no indirect buffer runtime mapping"); - return; - } - const auto selected = EmitIndirectResourceIndex( - state, ctx.Arg(*handle, 0), buffer.indirect_mapping_offset, - buffer.indirect_search_iterations, UINT32_MAX); - if (UsesSharedIndirectBufferStore(state, mem.resource)) { - EmitIfCondition(state, Binary(state, spv::OpULessThan, TypeBool(state), selected, - ConstantU32(state, buffer.indirect_resources.size())), [&]() { - const auto slot = EmitAddU32(state, selected, ConstantU32(state, buffer.descriptor_index)); - const auto access = PrepareStorageBufferResourceAccess(state, mem, state.storage_buffers[2], slot); - StoreWordPrepared(ctx, inst, mem, access, data); - }); - return; - } - EmitIndexSwitch(state, selected, buffer.indirect_resources.size(), 0u, - [&](uint32_t item) { store(buffer.indirect_resources[item]); }); + EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + EmitBufferAccess(ctx, inst, mem, 0u, [&](const auto& selected, const auto& resource) { + StoreWordPrepared(ctx, inst, selected, resource, ctx.Arg(inst, inst.NumArgs() - 2)); + }); }); } @@ -673,9 +700,9 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR:: ConstantU32(ctx.state, info.byte_size - 1u)); const auto last_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), end, ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u))); - plan.in_bounds = AndCondition( + plan.in_bounds = BufferIndexInBounds(ctx, inst, mem, resource, AndCondition( ctx.state, Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), end, start), - EmitMemoryElementInBounds(ctx.state, plan.resource, last_index)); + EmitMemoryElementInBounds(ctx.state, plan.resource, last_index))); plan.base = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), start, ConstantU32(ctx.state, ~(std::min(4u, info.byte_size) - 1u))); return plan; @@ -732,10 +759,11 @@ uint32_t LoadFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const Format::BufferFormatInfo& info, uint32_t components) { auto& state = ctx.state; const auto count = mem.data_bits == 16u ? mem.component_count : components; - const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info); - const auto value = EmitValueOrDefaultIfCondition( - state, plan.in_bounds, TypeU32Composite(state, count), - FormattedOutOfBoundsValue(ctx, mem, plan, count), [&]() { + const auto plan = resource.object_pointer == 0u ? PreparedFormattedMemory{.info = info} + : PrepareFormattedMemory(ctx, inst, mem, resource, info); + const auto out_of_bounds = FormattedOutOfBoundsValue(ctx, mem, plan, count); + const auto value = resource.object_pointer == 0u ? out_of_bounds : EmitValueOrDefaultIfCondition( + state, plan.in_bounds, TypeU32Composite(state, count), out_of_bounds, [&]() { std::array values {}; for (uint32_t component = 0; component < count; component++) { values[component] = LoadFormattedInBounds(ctx, mem, plan, component); @@ -1078,18 +1106,20 @@ uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compon if (mem.kind == IR::ResourceKind::IndirectBuffer) { return LoadIndirectBuffer(ctx, inst, components); } - const auto resource = PrepareMemoryResourceAccess(state, mem); - const auto info = Format::GetFormatInfo( - mem.formatted ? BufferFormat(ctx, mem) : Prospero::BufferFormat::kInvalid); - if (info.type != Format::ComponentType::Unknown) { - return LoadFormattedPrepared(ctx, inst, mem, resource, info, components); - } - std::array values {}; - for (uint32_t component = 0; component < components; component++) { - values[component] = - LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource); - } - return ConstructU32Composite(state, components, values); + return EmitBufferAccess(ctx, inst, mem, components, [&](const auto& selected, const auto& resource) { + const auto info = Format::GetFormatInfo( + selected.formatted ? BufferFormat(ctx, selected) : Prospero::BufferFormat::kInvalid); + if (info.type != Format::ComponentType::Unknown) { + return LoadFormattedPrepared(ctx, inst, selected, resource, info, components); + } + if (resource.object_pointer == 0u) return ConstantU32CompositeZero(state, components); + std::array values {}; + for (uint32_t component = 0; component < components; component++) { + values[component] = + LoadWordPrepared(ctx, inst, RebaseRawComponent(selected, component), resource); + } + return ConstructU32Composite(state, components, values); + }); }); } @@ -1431,6 +1461,13 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) { })); } +uint32_t EmitGetBufferResourceIndex(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto& buffer = ctx.state.program.info.buffers.at(inst.Flags()); + if (buffer.indirect_resources.empty()) return ConstantU32(ctx.state, 0u); + return EmitIndirectResourceIndex(ctx.state, ctx.Arg(inst, 0), buffer.indirect_mapping_offset, + buffer.indirect_search_iterations, UINT32_MAX); +} + void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) { const auto op = inst.GetOpcode(); const auto& mem = ctx.Memory(inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index d7465e87a..cae819067 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -108,17 +108,22 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR:: return state.program.info.buffers[mem.resource].descriptor_format; } -bool UsesSharedIndirectBufferStore(const EmitterState& state, uint32_t resource) { +bool UsesSharedIndirectBufferAccess(const EmitterState& state, uint32_t resource) { const auto& buffers = state.program.info.buffers; const auto& root = buffers[resource]; if (root.indirect_root != resource || root.descriptor_index == UINT32_MAX || - root.packed_stride == 0u || root.indirect_resources.empty()) { + root.packed_stride == 0u || root.indirect_resources.size() < 2u) { return false; } - uint32_t slot = root.descriptor_index; - for (const auto candidate: root.indirect_resources) { - if (buffers[candidate].packed_stride != root.packed_stride || - buffers[candidate].descriptor_index != slot++) return false; + uint32_t next_child_slot = UINT32_MAX; + for (uint32_t ordinal = 0; ordinal < root.indirect_resources.size(); ++ordinal) { + const auto& candidate = buffers[root.indirect_resources[ordinal]]; + if (candidate.packed_stride != root.packed_stride || + candidate.descriptor_format != root.descriptor_format || + candidate.descriptor_swizzle != root.descriptor_swizzle || + candidate.descriptor_index == UINT32_MAX || + (ordinal > 1u && candidate.descriptor_index != next_child_slot)) return false; + if (ordinal != 0u) next_child_slot = candidate.descriptor_index + 1u; } return true; } @@ -126,7 +131,7 @@ bool UsesSharedIndirectBufferStore(const EmitterState& state, uint32_t resource) void EmitMemoryOffsets(EmitterState& state) { std::array dynamic_offsets {}; for (uint32_t resource = 0; resource < state.program.info.buffers.size(); ++resource) { - if (!UsesSharedIndirectBufferStore(state, resource)) continue; + if (!UsesSharedIndirectBufferAccess(state, resource)) continue; for (const auto candidate: state.program.info.buffers[resource].indirect_resources) { dynamic_offsets[state.program.info.buffers[candidate].descriptor_index] = true; } @@ -227,7 +232,8 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, return access; } -MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem) { +MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem, + uint32_t array_index) { MemoryResourceAccess access {.kind = mem.kind}; switch (mem.kind) { case IR::ResourceKind::Lds: @@ -267,7 +273,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR:: } else if (bits == 16u) { buffer = &state.storage_buffers[1]; } - access = PrepareStorageBufferResourceAccess(state, mem, *buffer); + access = PrepareStorageBufferResourceAccess(state, mem, *buffer, array_index); access.element_bits = bits; return access; } diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 20653f9b2..9feb74586 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -537,7 +537,7 @@ struct DescriptorSource { uint32_t table_stride = 0; uint32_t table_record_bytes = 0; bool table_scalar = false; - bool indexed_store = false; + bool indexed = false; bool signed_count = false; uint32_t workgroup_axis = UINT32_MAX; Value key_count; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 69c52a89d..1f5196495 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -235,6 +235,7 @@ VALUE_OPCODE(BpermuteU32, U32, U32, U32, U1) // dense bindings are assigned by resource tracking. VALUE_OPCODE(GetSrtResource, SrtResource) VALUE_OPCODE(GetBufferResource, BufferResource, U32, U32, U32, U32) +VALUE_OPCODE(GetBufferResourceIndex, U32, U32) VALUE_OPCODE(GetAddressResource, AddressResource, U32, U32) VALUE_OPCODE(GetScratchResource, AddressResource) VALUE_OPCODE(GetImageResource, ImageResource, U32, U32, U32, U32, U32, U32, U32, U32) diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 2271586d1..4d3d15295 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -227,16 +227,38 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t offset, return ReadTable(base & AddressMask & ~uint64_t {3}, size, offset & ~uint64_t {3}, runtime, words); } -bool NormalizeIndirectStoreBuffer(DescriptorValue& value, bool indexed, bool formatted) { +bool NormalizeIndirectBuffer(DescriptorValue& value, bool indexed, const BufferResource& resource) { ShaderBufferResource descriptor; if (!DecodeBufferDescriptor(value, descriptor)) return false; + const bool formatted_read = resource.formatted && resource.read && !resource.written; if (descriptor.Type() != 0u || descriptor.Format() == Prospero::BufferFormat::kInvalid || - descriptor.Base48() == 0u || descriptor.GetSize() == 0u) { + (!formatted_read && (descriptor.Base48() == 0u || descriptor.GetSize() == 0u))) { value.dwords.fill(0u); - } else if (descriptor.Stride() != (indexed ? 4u : 1u) || descriptor.SwizzleEnabled() || - descriptor.AddTid() || descriptor.OutOfBounds() != (indexed ? 0u : 3u)) { + return true; + } + if (descriptor.SwizzleEnabled() || descriptor.AddTid() || + descriptor.OutOfBounds() != (indexed ? 0u : 3u)) { + return SpecializationFail("indirect buffers require exact linear index or byte bounds"); + } + if (formatted_read) { + const auto format = Format::GetFormatInfo(descriptor.Format()); + if (!indexed || format.type == Format::ComponentType::Unknown) + return SpecializationFail("indirect formatted loads require a supported indexed format"); + if (descriptor.Base48() == 0u || descriptor.Stride() == 0u || descriptor.NumRecords() == 0u) { + // Every index is out of bounds. Preserve the format and swizzle constants. + value.dwords[1] &= ~(0x3fffu << 16u); + value.dwords[2] = 0u; + return true; + } + const auto alignment = std::min(4u, format.byte_size); + if (descriptor.Stride() < format.byte_size || + (descriptor.Stride() & (alignment - 1u)) != 0u || + (descriptor.Base48() & (alignment - 1u)) != 0u) + return SpecializationFail("indirect formatted loads require aligned full-format records"); + } else if ((!indexed && descriptor.Stride() != 1u) || + (indexed && resource.max_byte_extent != 1u && descriptor.Stride() != 4u)) { return SpecializationFail("indirect buffer stores require exact linear DWORD or byte bounds"); - } else if (formatted && (descriptor.Format() != Prospero::BufferFormat::k32UInt || + } else if (resource.formatted && (descriptor.Format() != Prospero::BufferFormat::k32UInt || descriptor.DstSelXYZW() != DstSel(4, 0, 0, 0) || (descriptor.Base48() & 3u) != 0u)) { return SpecializationFail("indirect formatted X stores require an aligned 32-bit UInt identity mapping"); @@ -248,7 +270,8 @@ bool IsBoundedDescriptorTable(const ResourcePlan& program, const DescriptorSource::IndirectDescriptor& indirect) { const auto* table = Source(program, indirect.table_source); return indirect.sources.empty() && !indirect.selector && - indirect.workgroup_axis == UINT32_MAX && table != nullptr && table->dword_count == 4u; + indirect.key_count.IsEmpty() && indirect.workgroup_axis == UINT32_MAX && + table != nullptr && table->dword_count == 4u; } template @@ -372,7 +395,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p key_count = runtime.workgroup_counts[indirect.workgroup_axis]; if (key_count == 0u) return false; } else { - if (table_value.dword_count != 2u || !evaluate_count(key_count)) + if (!evaluate_count(key_count)) return false; } if (!indirect.selector_first.IsEmpty() && !clean.Evaluate(indirect.selector_first, first_key)) @@ -1308,8 +1331,8 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime program, *source->indirect_descriptor, i, 4u, observed, clean, snapshot, snapshot.buffers, specialization.buffers, ShaderInfo::MaxBuffers, [&](DescriptorValue& value) { - return NormalizeIndirectStoreBuffer( - value, source->indirect_descriptor->indexed_store, base.formatted); + return NormalizeIndirectBuffer( + value, source->indirect_descriptor->indexed, base); })) return false; } @@ -1526,7 +1549,8 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& if (BufferAccessOf(inst.GetOpcode()) == BufferAccess::Read) { const auto& memory = memory_info[inst.Flags().index]; if (memory.kind == ResourceKind::Buffer && - specialization.buffers[memory.resource].zero_stride_oob) { + specialization.buffers[memory.resource].zero_stride_oob && + buffers[memory.resource].indirect_root == BufferResource::NoIndirectBuffer) { // Bounds mode 0 checks offset >= stride, so zero stride // makes every vector read out of bounds regardless of its address. const auto count = BufferComponentCount(inst.GetOpcode()); diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index fc6f7378f..3ccaf8166 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -394,6 +394,7 @@ public: memory.sampler = patch.sampler; } } + std::vector> buffer_selections; for (const auto& plan: m_indirect_descriptors) { auto key = plan.key; if (plan.table_read != nullptr) { @@ -452,8 +453,32 @@ public: plan.handle->SetArg(word, Value(0u)); continue; } - if (plan.handle->NumArgs() == 4u) { - plan.handle->SetArg(0, plan.key); + if (plan.handle->GetOpcode() == ValueOpcode::GetBufferResource) { + Value selected; + for (const auto& [previous, value]: buffer_selections) { + if (plan.selection_anchor != nullptr && previous->reads == plan.reads && + previous->source == plan.source && + previous->handle->Flags() == plan.handle->Flags() && + EquivalentValue(m_program, previous->key, plan.key) && + AvailableBefore(*value.Instruction(), *plan.handle)) { + selected = value; + break; + } + } + if (selected.IsEmpty()) { + const auto* anchor = plan.selection_anchor != nullptr ? plan.selection_anchor : plan.handle; + auto* block = anchor->Parent(); + const auto where = std::ranges::find_if(block->Instructions(), + [&](const Inst& inst) { return &inst == anchor; }); + auto& selection = *block->PrependNewInst(where, ValueOpcode::GetBufferResourceIndex, {plan.key}); + selection.SetFlags(plan.handle->Flags()); + selected = Value(&selection); + buffer_selections.emplace_back(&plan, selected); + } + plan.handle->SetArg(0, selected); + for (uint32_t word = 1; word < 4u; ++word) plan.handle->SetArg(word, Value(0u)); + } else if (plan.handle->NumArgs() == 4u) { + plan.handle->SetArg(0, key); for (uint32_t word = 1; word < 4u; ++word) plan.handle->SetArg(word, Value(0u)); } else { if (plan.reads[0] == nullptr) plan.handle->SetArg(0, key); @@ -519,6 +544,7 @@ private: bool retain_reads = false; bool table_indexed = false; const Inst* table_read = nullptr; + const Inst* selection_anchor = nullptr; }; [[noreturn]] void Fail(uint32_t pc, const std::string& reason) const { @@ -931,7 +957,7 @@ private: const auto& b = *descriptor.indirect_descriptor; if (a.selector != b.selector || a.table_source != b.table_source || a.table_offset != b.table_offset || a.table_immediate != b.table_immediate || - a.table_stride != b.table_stride || a.indexed_store != b.indexed_store || + a.table_stride != b.table_stride || a.indexed != b.indexed || a.workgroup_axis != b.workgroup_axis || a.sources != b.sources || a.signed_count != b.signed_count || !EquivalentValue(m_program, a.key_count, b.key_count) || @@ -2522,6 +2548,25 @@ private: return true; } + bool IndirectBufferAccess(const Inst& inst) const { + const auto opcode = inst.GetOpcode(); + if (opcode != ValueOpcode::StoreBufferU32 && opcode != ValueOpcode::StoreBufferU8 && + opcode != ValueOpcode::LoadBufferU32x2) return false; + const auto index = inst.Flags().index; + if (index >= m_program.memory_info.size()) return false; + const auto& memory = m_program.memory_info[index]; + if (memory.kind != ResourceKind::Buffer || memory.typed || memory.idxen == memory.offen || + (memory.idxen && (memory.offset != 0u || inst.Arg(2).Resolve() != Value(0u))) || + inst.Arg(3).Resolve() != Value(0u)) return false; + if (opcode == ValueOpcode::StoreBufferU32) + return !memory.formatted || (memory.idxen && memory.data_bits == 32u && memory.data_dwords == 1u); + if (!memory.idxen) return false; + if (opcode == ValueOpcode::StoreBufferU8) + return !memory.formatted && memory.data_bits == 8u; + return memory.formatted && memory.data_bits == 16u && memory.data_dwords == 2u && + memory.component_count == 4u; + } + bool MatchDescriptorTable(Inst& handle, const DescriptorSource& descriptor, IndirectDescriptorPlan& plan, DescriptorSource& table_source, Value& key, uint32_t& table_offset, @@ -2599,10 +2644,10 @@ private: if (handle.NumArgs() == 8u) { if (memory->kind == ResourceKind::ScalarAddress) plan.retain_reads |= !UsesOnlyImageDescriptors(*read); - } else if (std::ranges::any_of(read->Uses(), [](const Use& use) { + } else if (std::ranges::any_of(read->Uses(), [&](const Use& use) { return use.user->GetOpcode() != ValueOpcode::GetBufferResource || - std::ranges::any_of(use.user->Uses(), [](const Use& consumer) { - return consumer.user->GetOpcode() != ValueOpcode::StoreBufferU32; + std::ranges::any_of(use.user->Uses(), [&](const Use& consumer) { + return !IndirectBufferAccess(*consumer.user); }); })) { return false; @@ -2780,19 +2825,14 @@ private: IndirectDescriptorPlan& plan) { if (handle.GetOpcode() != ValueOpcode::GetBufferResource || handle.NumArgs() != 4u || handle.Uses().empty()) return false; - if (handle.Uses().front().user->GetOpcode() != ValueOpcode::StoreBufferU32) return false; + if (!IndirectBufferAccess(*handle.Uses().front().user)) return false; + const bool legacy_store = handle.Uses().front().user->GetOpcode() == ValueOpcode::StoreBufferU32; const auto indexed = m_program.memory_info[handle.Uses().front().user->Flags().index].idxen; for (const auto& use: handle.Uses()) { const auto& inst = *use.user; - if (inst.GetOpcode() != ValueOpcode::StoreBufferU32) return false; + if (!IndirectBufferAccess(inst)) return false; const auto& memory = m_program.memory_info[inst.Flags().index]; - uint32_t scalar = 1u; - if (memory.kind != ResourceKind::Buffer || memory.typed || - (memory.formatted && (!indexed || memory.data_bits != 32u || memory.data_dwords != 1u)) || - memory.idxen != indexed || memory.offen == indexed || - (indexed && (memory.offset != 0u || inst.Arg(2).Resolve() != Value(0u))) || - !ImmediateU32(inst.Arg(3), scalar) || scalar != 0u) - return false; + if (memory.idxen != indexed) return false; } DescriptorSource table_source; Value key; @@ -2800,12 +2840,25 @@ private: uint32_t table_stride = 0; uint32_t table_immediate = 0; if (!MatchDescriptorTable(handle, descriptor, plan, table_source, key, table_offset, - table_stride, table_immediate) || table_stride != 16u) return false; + table_stride, table_immediate) || + table_stride < 16u || (table_stride & 3u) != 0u) return false; DescriptorSource::IndirectDescriptor indirect; indirect.table_stride = table_stride; indirect.table_offset = table_offset; indirect.table_immediate = table_immediate; - indirect.indexed_store = indexed; + indirect.indexed = indexed; + for (uint32_t word = 0; word < 4u; ++word) { + const auto* anchor = plan.reads[word]; + const auto* key_inst = key.Resolve().TryInstruction(); + if (AvailableBefore(*anchor, handle) && + (key_inst == nullptr || (key_inst != anchor && AvailableBefore(*key_inst, *anchor))) && + std::all_of(plan.reads.begin(), plan.reads.begin() + 4u, + [&](const Inst* read) { return AvailableBefore(*read, *anchor); })) { + plan.selection_anchor = anchor; + break; + } + } + if (!legacy_store && plan.selection_anchor == nullptr) return false; const auto finish = [&] { indirect.table_source = InternSource(table_source); DescriptorSource source; @@ -2817,19 +2870,20 @@ private: plan.key = key; return true; }; - if (table_source.dword_count == 2u && MatchPackedScalarKey(key, indirect)) return finish(); + if (const auto* bound = BoundedLoop(key, handle.Parent()); bound != nullptr) { + indirect.key_count = bound->Arg(1); + indirect.signed_count = bound->GetOpcode() == ValueOpcode::SLessThan32; + return finish(); + } + if (!legacy_store || table_stride != 16u) return false; if (table_source.dword_count == 2u) { - const auto* bound = BoundedLoop(key, handle.Parent()); - if (bound != nullptr) { - indirect.key_count = bound->Arg(1); - indirect.signed_count = bound->GetOpcode() == ValueOpcode::SLessThan32; - } else { + if (!MatchPackedScalarKey(key, indirect)) { // EXEC implications must describe the same iteration as the selected value. if (m_native_cfg.irreducible || !m_native_cfg.back_edges.empty()) return false; const auto* lane = key.Resolve().TryInstruction(); if (lane == nullptr || lane->GetOpcode() != ValueOpcode::ReadFirstLane || !std::ranges::all_of(handle.Uses(), [&](const Use& use) { - return Implies(use.user->Arg(5), lane->Arg(1)); + return Implies(use.user->Arg(use.user->NumArgs() - 1u), lane->Arg(1)); })) return false; const auto maximum = SelectorMaximum(lane->Arg(0), {}, 0u, lane->Arg(1)); if (maximum >= MaxIndirectDescriptorProbes) return false; @@ -3112,7 +3166,7 @@ private: for (auto* block: m_program.blocks) { for (auto& inst: *block) { if ((ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None && - inst.GetOpcode() != ValueOpcode::StoreBufferU32) || + !IndirectBufferAccess(inst)) || inst.NumArgs() == 0u) { continue; } @@ -3138,7 +3192,7 @@ private: m_indirect_descriptors.push_back(std::move(plan)); } }; - plan_handle(inst.Arg(0), inst.GetOpcode() == ValueOpcode::StoreBufferU32 ? 4u : 8u, + plan_handle(inst.Arg(0), IndirectBufferAccess(inst) ? 4u : 8u, memory.resource * 4u, false); if (inst.GetOpcode() == ValueOpcode::ImageSampleRaw && (memory.image_sample_flags & Decoder::ImageSampleFlagAdjust) == 0u) diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index 29aecffe1..b23097c51 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -343,9 +343,16 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs if (memory.resource >= info.buffers.size()) { return Fail("buffer operation has invalid resource metadata"); } - const auto bits = StorageBufferElementBits(program, memory); - info.buffer_u8 |= bits == 8u; - info.buffer_u16 |= bits == 16u; + const auto collect_element = [&](uint32_t resource) { + auto selected = memory; + selected.resource = resource; + const auto bits = StorageBufferElementBits(program, selected); + info.buffer_u8 |= bits == 8u; + info.buffer_u16 |= bits == 16u; + }; + collect_element(memory.resource); + for (const auto child: info.buffers[memory.resource].indirect_resources) + collect_element(child); if ((info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) { if (program.stage != ShaderType::Compute) { return Fail("buffer ADD_TID is only valid for compute shaders"); diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 846a324d6..996036b49 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -4293,7 +4293,7 @@ void TestBoundedFormattedBufferStore() { auto fixture = make(); auto plan = ExtractResourcePlan(fixture->program); const auto& source = plan.descriptor_sources[plan.info.buffers[0].source]; - Check(source.indirect_descriptor && source.indirect_descriptor->indexed_store && + Check(source.indirect_descriptor && source.indirect_descriptor->indexed && !source.indirect_descriptor->signed_count && !source.indirect_descriptor->selector && !source.indirect_descriptor->key_count.IsEmpty() && @@ -4393,6 +4393,175 @@ void TestBoundedFormattedBufferStore() { "finite store accepted an unproved induction or indexed byte offset"); } +void TestBoundedBufferRecords() { + namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; + namespace Prospero = Libs::Graphics::Prospero; + using Libs::Graphics::DstSel; + const auto make = [](uint32_t step = 1u, bool bypass = false) { + auto fixture = std::make_unique(); + auto& f = *fixture; + auto* entry = f.block; + auto* header = f.AddBlock(); + auto* body = f.AddBlock(); + auto* latch = f.AddBlock(); + auto* exit = f.AddBlock(); + const auto branch = [](Block* from, Block* to) { + from->AddBranch(to); + from->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = to}; + }; + branch(entry, header); + branch(body, latch); + branch(latch, header); + header->AddBranch(body); + header->AddBranch(bypass ? latch : exit); + header->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = body, .false_block = bypass ? latch : exit}; + exit->terminator.kind = CFG::TerminatorKind::Return; + std::array table_words; + for (uint32_t word = 0; word < 4u; ++word) table_words[word] = f.UserData(word); + const auto table = f.Buffer(table_words); + const auto count = f.Emit(ValueOpcode::LoadAddressU32, + {f.Address(f.UserData(4u), f.UserData(5u)), Value(0u), Value(0u), Value(true)}, + f.AddMemory({.kind = ResourceKind::ScalarAddress}, 0x424u)); + const auto lane = f.Emit(ValueOpcode::LaneId); + auto& phi = header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); + phi.AddPhiOperand(entry, Value(0u)); + phi.AddPhiOperand(latch, f.Emit(ValueOpcode::IAdd32, {Value(&phi), Value(step)}, 0, latch)); + header->condition = f.Emit(ValueOpcode::SLessThan32, {Value(&phi), count}, 0, header); + f.block = body; + const auto offset = f.Emit(ValueOpcode::IMul32, {Value(&phi), Value(192u)}); + for (const bool formatted : {true, false}) { + std::array words; + for (uint32_t word = 0; word < 4u; ++word) + words[word] = f.Emit(ValueOpcode::ReadConstBuffer, {table, offset}, + f.AddMemory({.kind = ResourceKind::ScalarBuffer, + .offset = (formatted ? 160u : 176u) + word * 4u, + .component_index = word, .component_count = 4u}, 0x1808u)); + for (uint32_t use = 0; use < (formatted ? 112u : 16u); ++use) { + const auto handle = f.Buffer(words); + if (formatted) { + const auto loaded = f.Emit(ValueOpcode::LoadBufferU32x2, + {handle, lane, Value(0u), Value(0u), Value(true)}, + f.AddMemory({.kind = ResourceKind::Buffer, .data_dwords = 2u, .data_bits = 16u, + .component_count = 4u, .formatted = true, .idxen = true}, 0x1914u)); + f.Emit(ValueOpcode::ReferenceU32, + {f.Emit(ValueOpcode::CompositeExtractU32x2, {loaded, Value(0u)})}); + } else { + f.Emit(ValueOpcode::StoreBufferU8, + {handle, lane, Value(0u), Value(0u), Value(uint8_t{42u}), Value(true)}, + f.AddMemory({.kind = ResourceKind::Buffer, .data_bits = 8u, .idxen = true}, 0x190cu)); + } + } + } + f.PlanAndTrack(); + return fixture; + }; + auto fixture = make(); + const auto plan = ExtractResourcePlan(fixture->program); + Check(plan.info.buffers.size() == 2u, "record descriptors did not share their resource roots"); + uint32_t selections = 0u; + std::array selected{}; + for (const auto* block : fixture->program.blocks) { + for (const auto& inst : *block) { + if (inst.GetOpcode() == ValueOpcode::GetBufferResourceIndex) { + ++selections; + Check(block->id == 2u && inst.Flags() < 2u, + "record selection moved outside its loop or lost its buffer root"); + selected[inst.Flags()] = &inst; + } + if (inst.GetOpcode() == ValueOpcode::LoadBufferU32x2 || inst.GetOpcode() == ValueOpcode::StoreBufferU8) { + const auto root = fixture->program.memory_info[inst.Flags().index].resource; + Check(inst.Arg(0).Instruction()->Arg(0).Instruction() == selected[root], + "record operation did not use its dominating shared selection"); + } + } + } + Check(selections == 2u, "repeated record operations duplicated candidate mapping searches"); + for (uint32_t root = 0; root < 2u; ++root) { + const auto& source = plan.descriptor_sources[plan.info.buffers[root].source]; + Check(source.indirect_descriptor && source.indirect_descriptor->indexed && + source.indirect_descriptor->signed_count && + source.indirect_descriptor->table_stride == 192u && + source.indirect_descriptor->table_immediate == (root == 0u ? 160u : 176u) && + !source.indirect_descriptor->key_count.IsEmpty(), + "record descriptor lost its bounded loop or member offset"); + } + LinearTestMemory memory; + constexpr uint32_t table_offset = 256u; + const auto format = static_cast(Prospero::BufferFormat::k16_16_16_16Float); + for (uint32_t row = 0; row < 4u; ++row) { + const auto first = (table_offset + row * 192u) / 4u; + std::fill_n(memory.words.begin() + first, 48u, 0xdeadbeefu); + for (const bool formatted : {true, false}) { + const auto word = first + (formatted ? 40u : 44u); + memory.words[word] = 0x100000u + row * 0x1000u + (formatted ? 0u : 0x8000u); + memory.words[word + 1u] = (formatted ? 8u : 3u) << 16u; + memory.words[word + 2u] = 16u; + memory.words[word + 3u] = formatted ? (format << 12u) | DstSel(4, 5, 6, 1) : 0x5204u; + } + } + std::array user_data{uint32_t(memory.base) + table_offset, 8u << 16u, + 4u * 192u / 8u, 0x5204u, uint32_t(memory.base), 0u}; + SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, + .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + const auto materialize = [&] { return MaterializeResources(plan, runtime, snapshot, specialization); }; + for (const uint32_t count : {0u, 1u, 4u, 2u, UINT32_MAX, 0u, 4u}) { + memory.words[0] = count; + const uint32_t rows = count == UINT32_MAX ? 0u : count; + user_data[2] = rows == 0u ? 0u : 4u * 192u / 8u; + memory.reads = 0u; + Check(materialize() && snapshot.buffers.size() == 2u * std::max(rows, 1u) && + memory.reads == 1u + rows * 2u, + "bounded record enumeration read unrelated fields or retained stale candidates"); + for (uint32_t row = 0; row < rows; ++row) + for (const auto member : {160u, 176u}) + Check(std::ranges::find(snapshot.specialization_reads, + std::pair{memory.base + table_offset + row * 192u + member, uint64_t{16u}}) != + snapshot.specialization_reads.end(), + "record descriptor escaped same-dispatch overlap capture"); + } + const auto original = specialization; + memory.words[(table_offset + 160u) / 4u] += 0x100u; + Check(materialize() && specialization == original, + "record address refresh changed the shader specialization"); + user_data[2] = 4u; + memory.reads = 0u; + Check(materialize() && snapshot.buffers.size() == 2u && memory.reads == 1u, + "record enumeration read beyond the scalar buffer table bounds"); + user_data[2] = 4u * 192u / 8u; + const auto first = (table_offset + 160u) / 4u; + const auto first_address = memory.words[first]; + memory.words[first] = 0u; + memory.words[first + 2u] = 0u; + Check(materialize() && specialization.buffers[0].zero_stride_oob && + specialization.buffers[0].descriptor_swizzle == DstSel(4, 5, 6, 1) && + specialization.buffers[0].indirect_root == 0u, + "empty formatted candidate lost its OOB constants or other candidates"); + ApplyResourceSpecialization(fixture->program, specialization); + uint32_t loads = 0u; + for (const auto* block : fixture->program.blocks) + for (const auto& inst : *block) loads += inst.GetOpcode() == ValueOpcode::LoadBufferU32x2; + Check(loads == 112u, "empty root incorrectly folded nonempty formatted candidates"); + memory.words[first] = first_address; + memory.words[first + 2u] = 16u; + for (const auto [word, replacement] : std::array, 5>{ + {{first + 1u, 4u << 16u}, {first + 1u, 9u << 16u}, + {first, memory.words[first] + 1u}, {first + 3u, memory.words[first + 3u] | (1u << 28u)}, + {first + 1u, (8u << 16u) | (1u << 31u)}}}) { + const auto saved = memory.words[word]; + memory.words[word] = replacement; + Check(!materialize(), "formatted record accepted unproved alignment or addressing"); + memory.words[word] = saved; + } + memory.words[0] = MaxIndirectDescriptorProbes + 1u; + Check(!materialize(), "record table accepted an unbounded candidate count"); + for (const auto [step, bypass] : {std::pair{2u, false}, std::pair{1u, true}}) + CheckFatal([&] { make(step, bypass); }, "not a valid runtime value", + "record table accepted an unproved loop bound"); +} + void TestFiniteBufferSelection() { enum class Variant { Valid, WiderExec, Unbounded, Cycle, TooMany }; const auto make = [](Variant variant) { @@ -4516,7 +4685,7 @@ void TestPackedScalarBufferSelection() { fixture->PlanAndTrack(); auto plan = ExtractResourcePlan(fixture->program); const auto& source = plan.descriptor_sources[plan.info.buffers[0].source]; - Check(source.indirect_descriptor && source.indirect_descriptor->indexed_store && + Check(source.indirect_descriptor && source.indirect_descriptor->indexed && source.indirect_descriptor->selector->byte_ranges == std::vector>{{0x1c0u, 1024u}}, "packed scalar lookup lost a Phi arm or independent DWORD alignment"); @@ -4571,8 +4740,8 @@ void TestPackedScalarBufferSelection() { mixed->PlanAndTrack(); const auto& buffers = mixed->program.info.buffers; Check(buffers.size() == 2u && buffers[0].source != buffers[1].source && - mixed->program.descriptor_sources[buffers[0].source].indirect_descriptor->indexed_store && - !mixed->program.descriptor_sources[buffers[1].source].indirect_descriptor->indexed_store, + mixed->program.descriptor_sources[buffers[0].source].indirect_descriptor->indexed && + !mixed->program.descriptor_sources[buffers[1].source].indirect_descriptor->indexed, "indexed and byte-addressed stores incorrectly shared their descriptor bounds contract"); } @@ -5888,6 +6057,7 @@ int main() { Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit); Run("writable descriptor phi", TestWritableDescriptorPhi); Run("bounded formatted buffer store", TestBoundedFormattedBufferStore); + Run("bounded buffer records", TestBoundedBufferRecords); Run("finite buffer selection", TestFiniteBufferSelection); Run("packed scalar buffer selection", TestPackedScalarBufferSelection); Run("conditional sampler phi", TestConditionalSamplerPhi); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 76b14271c..81defcd48 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -36575,6 +36575,135 @@ void CheckBoundedFormattedBufferCopy(VulkanHarness &vulkan) { std::printf("[compute] %-32s ok\n", "BoundedFormattedBufferCopy"); } +void CheckBoundedRecordBufferAccess(VulkanHarness &vulkan) { + using namespace ShaderRecompiler::IR; + using F = Prospero::BufferFormat; + const u32 alignment = vulkan.RuntimeContext().GetPhysicalDeviceProperties() + .limits.minStorageBufferOffsetAlignment; + const u32 slot = (4096u + alignment - 1u) / alignment * alignment; + for (const u32 wave : {32u, 64u}) { + for (const u32 count : {0u, 2u, 6u}) { + TestCase test; + test.name = wave == 32 ? "BoundedRecordBufferAccess32" : "BoundedRecordBufferAccess64"; + constexpr u32 table = 256u; + test.initial.assign((slot * 13u + 6u * wave * 16u) / 4u, 0xdeadbeefu); + test.has_user_data = test.has_compute_info = true; + test.user_data[0] = table; + test.user_data[1] = 8u << 16u; + test.user_data[2] = 6u * 192u / 8u; + test.user_data[3] = 0x5204u; + test.user_data[4] = count; + const std::array output{slot * 13u, 16u << 16u, 6u * wave, 0x5204u}; + std::copy(output.begin(), output.end(), test.user_data.begin() + 16u); + test.compute_info.wave_size = wave; + test.compute_info.threads_num[0] = wave; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u; + test.compute_info.thread_ids_num = 1; + test.compute_info.lds_size_dwords = 0; + auto &code = test.code; + code.insert(code.end(), {EncodeSMovB32(32, InlineU32(0)), EncodeSop1(0x04, 36, 126)}); + AppendVMovLiteral(&code, 13, 0x80000000u); + code.insert(code.end(), {EncodeVopc(0xc2, InlineU32(6), 0), + EncodeVop2(0x01, 12, Vgpr(0), 13)}); + AppendVMovLiteral(&code, 14, 0x55555556u); + code.insert(code.end(), {EncodeVopc(0xc2, InlineU32(6), 0), + EncodeVop2(0x01, 15, Vgpr(0), 14)}); + AppendVMovLiteral(&code, 10, 0xa5c39b72u); + const auto header = code.size(); + code.push_back(EncodeSopc(0x04, 32, 4)); + const auto exit_branch = code.size(); + code.push_back(0); + code.insert(code.end(), {EncodeSop2(0x26, 33, 32, 255), 192u, + EncodeSmem0(0x0a, 8, 0), EncodeSmem1(160u, 33), + EncodeSmem0(0x0a, 12, 0), EncodeSmem1(176u, 33), EncodeSopp(0x0c, 0), + EncodeMubuf0(0x83, 0, true, false), EncodeMubuf1(4, 2, 12), + EncodeMubuf0(0x83, 0, true, false, true), EncodeMubuf1(6, 2, 12), + EncodeSopp(0x0c, 0), EncodeSop2(0x1e, 34, 32, InlineU32(std::countr_zero(wave))), + EncodeVop2(0x25, 20, 34, 0), + EncodeMubuf0(0x1e, 0, true, false), EncodeMubuf1(4, 4, 20)}); + AppendSMovLiteral(&code, 126, 0x55555555u); + AppendSMovLiteral(&code, 127, wave == 64 ? 0x55555555u : 0u); + code.insert(code.end(), {EncodeMubuf0(0x19, 0, true, false, true), EncodeMubuf1(10, 3, 15), + EncodeSop1(0x04, 126, 36), EncodeSop2(0x02, 32, 32, InlineU32(1))}); + code.push_back(EncodeSopp(0x02, header - code.size() - 1u)); + code[exit_branch] = EncodeSopp(0x04, code.size() - exit_branch - 1u); + AppendEnd(&code); + for (u32 row = 0; row < 6u; ++row) { + const u32 src = slot * (row + 1u) + (row + 1u) * 4u; + const u32 dst = slot * (row + 7u) + row + 1u; + const auto format = count == 6u && row == 0u ? F::k32_32_32_32Float : + count == 6u && row == 1u ? F::k8_8_8_8UNorm : F::k16_16_16_16Float; + const u32 swizzle = row == 2u ? DstSel(1, 4, 1, 4) : DstSel(4, 5, 6, 7); + std::array input{row == 2u ? 0u : src, 16u << 16u, + row == 2u ? 0u : 2u, static_cast(format) << 12u | swizzle}; + if (row == 3u) input.fill(0u); + const std::array destination{dst, 3u << 16u, 16u, 0x5204u}; + std::copy(input.begin(), input.end(), test.initial.begin() + table / 4u + row * 48u + 40u); + std::copy(destination.begin(), destination.end(), test.initial.begin() + table / 4u + row * 48u + 44u); + for (u32 record = 0; record < 2u; ++record) { + const auto word = src / 4u + record * 4u; + if (format == F::k32_32_32_32Float) { + const std::array data{0x3f800000u, 0x3f000000u, 0u, 0xc0000000u}; + std::copy(data.begin(), data.end(), test.initial.begin() + word); + } else if (format == F::k8_8_8_8UNorm) test.initial[word] = 0xffffffffu; + else { + test.initial[word] = 0x38003c00u; + test.initial[word + 1u] = 0xc0000000u; + } + } + } + test.expected = test.initial; + auto expected_bytes = std::as_writable_bytes(std::span(test.expected)); + for (u32 row = 0; row < count; ++row) { + for (u32 lane = 0; lane < wave; ++lane) { + std::array value{}; + if (row == 2u) value = {0x3c00u, 0x3c00u}; + else if (row != 3u && lane < 2u && !(row == 5u && lane == 1u)) + value = count == 6u && row == 1u ? std::array{0x3c003c00u, 0x3c003c00u} + : std::array{0x38003c00u, 0xc0000000u}; + const u32 out = slot * 13u / 4u + (row * wave + lane) * 4u; + for (u32 word = 0; word < 4u; ++word) test.expected[out + word] = value[word % 2u]; + if (lane < 16u && lane != 6u && (lane & 1u) == 0u) + expected_bytes[slot * (row + 7u) + row + 1u + lane * 3u] = std::byte{0xc3}; + } + } + if (count == 2u) { + test.required_spirv = {"StorageBufferArrayNonUniformIndexing"}; + test.forbidden_spirv = {"OpSwitch", "PhysicalStorageBuffer"}; + } else if (count == 6u) test.required_spirv = {"OpSwitch"}; + auto compiled = CompileCase(test, vulkan.SubgroupSize()); + Require(test.name, "mixed native aliases", count != 6u || + (compiled.program.info.buffer_u8 && compiled.program.info.buffer_u16), + "mixed candidates lost an 8-bit or 16-bit native binding"); + u32 selections = 0; + for (auto *block : compiled.program.blocks) + for (const auto &inst : *block) selections += inst.GetOpcode() == ValueOpcode::GetBufferResourceIndex; + Require(test.name, "shared loop selection", count == 0u ? selections <= 2u : selections == 2u, + "repeated formatted loads repeated the descriptor lookup"); + for (u32 resource = 0; resource < compiled.resources.buffers.size(); ++resource) { + ShaderBufferResource descriptor{}; + std::copy_n(compiled.resources.buffers[resource].dwords.begin(), 4u, descriptor.fields); + const u32 address = descriptor.Base48(); + const u32 prefix = address % alignment; + u32 bytes = std::max(4u, static_cast(descriptor.GetSize())); + if (address == slot * 5u + 20u) bytes = 24u; + if (address == slot * 6u + 24u) bytes = 23u; + test.storage_buffer_ranges.emplace_back(address - prefix, bytes + prefix); + const u32 binding = compiled.program.info.buffers[resource].descriptor_index; + if (binding != UINT32_MAX) + compiled.packed_user_data[compiled.program.bindings.memory_offset_dword + binding / 4u] |= + prefix << ((binding % 4u) * 8u); + } + auto output_buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size()); + vulkan.Dispatch(test, compiled, output_buffer); + CompareWords(test, "row keys, mixed formats, empty-One, partial payloads, prefixes and byte canaries", + test.expected, vulkan.ReadBuffer(test.name, output_buffer, test.expected.size())); + vulkan.DestroyBuffer(&output_buffer); + } + } + std::printf("[compute] %-32s ok\n", "BoundedRecordBufferAccess"); +} + void CheckFiniteBufferSelection(VulkanHarness &vulkan) { for (const u32 wave : {32u, 64u}) { TestCase test; @@ -36663,9 +36792,11 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { {Value(&lane), Value(7u)}); auto &key = block->AppendNewInst(ValueOpcode::IAdd32, {Value(&selector), Value(8u)}); + auto &selection = block->AppendNewInst(ValueOpcode::GetBufferResourceIndex, {Value(&key)}); + selection.SetFlags(0u); auto &handle = block->AppendNewInst( ValueOpcode::GetBufferResource, - {Value(&key), Value(0u), Value(0u), Value(0u)}); + {Value(&selection), Value(0u), Value(0u), Value(0u)}); handle.SetFlags(0u); auto &lane_offset = block->AppendNewInst(ValueOpcode::IMul32, {Value(&lane), Value(4u)}); @@ -36715,7 +36846,7 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { if (indexed) { for (auto &buffer : program.info.buffers) buffer.packed_stride = 4; } - program.descriptor_sources[0].indirect_descriptor->indexed_store = indexed; + program.descriptor_sources[0].indirect_descriptor->indexed = indexed; program.memory_info[0].idxen = indexed; program.memory_info[0].offen = !indexed; store.SetArg(1, indexed ? Value(&index) : Value(0u)); @@ -45378,6 +45509,7 @@ int main(int argc, char **argv) { CheckBoundedFormattedBufferCopy(vulkan); CheckFiniteBufferSelection(vulkan); CheckIndirectBufferStore(vulkan); + CheckBoundedRecordBufferAccess(vulkan); CheckRuntimeBufferRecords(vulkan); RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadDwordGpuSelectedDescriptors()); @@ -46079,6 +46211,7 @@ int main(int argc, char **argv) { CheckRectListShaders(); CheckBoundedFormattedBufferCopy(vulkan); CheckIndirectBufferStore(vulkan); + CheckBoundedRecordBufferAccess(vulkan); CheckFiniteBufferSelection(vulkan); CheckRuntimeBufferRecords(vulkan); CheckComputeThreadDimensions(vulkan);