diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index 7c974bcd5..e645ed2e9 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -213,7 +213,7 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_ EmitWrite(0); return; } - EmitImageMipSwitch(state, mip_lod, image.mip_count, 0, EmitWrite); + EmitIndexSwitch(state, mip_lod, image.mip_count, 0, EmitWrite); } spv::ExecutionModel ExecutionModelForStage(ShaderType stage) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index 17c34a0b6..c51b23618 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -728,7 +728,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { return sample; }; const auto sample = image.mip_mode == IR::ImageMipMode::Dynamic && image.mip_count > 1u - ? EmitImageMipSwitch( + ? EmitIndexSwitch( state, GatherMip(ctx, inst, mem, *address, layout, image.mip_count), image.mip_count, result_type, EmitGather) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 0ea16b947..74fc754ea 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -520,17 +520,17 @@ uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high // These templates accept local lambdas from several emitter translation units. template -auto EmitImageMipSwitch(EmitterState& state, uint32_t mip_lod, uint32_t mip_count, +auto EmitIndexSwitch(EmitterState& state, uint32_t index, uint32_t count, uint32_t result_type, Fn&& emit) { constexpr bool has_result = !std::is_void_v>; - EXIT_IF(mip_count == 0u); + EXIT_IF(count == 0u); const auto merge_label = state.builder.AllocateId(); - std::vector labels(mip_count); - std::vector words {spv::OpSwitch, mip_lod, merge_label}; - for (uint32_t mip = 0; mip < mip_count; mip++) { - labels[mip] = state.builder.AllocateId(); - words.push_back(mip); - words.push_back(labels[mip]); + std::vector labels(count); + std::vector words {spv::OpSwitch, index, merge_label}; + for (uint32_t item = 0; item < count; item++) { + labels[item] = state.builder.AllocateId(); + words.push_back(item); + words.push_back(labels[item]); } if constexpr (has_result) { words[2] = labels.front(); @@ -541,13 +541,13 @@ auto EmitImageMipSwitch(EmitterState& state, uint32_t mip_lod, uint32_t mip_coun if constexpr (has_result) { phi = {spv::OpPhi, result_type, state.builder.AllocateId()}; } - for (uint32_t mip = 0; mip < mip_count; mip++) { - EmitLabel(state, labels[mip]); + for (uint32_t item = 0; item < count; item++) { + EmitLabel(state, labels[item]); if constexpr (has_result) { - phi.push_back(emit(mip)); + phi.push_back(emit(item)); phi.push_back(state.current_label); } else { - emit(mip); + emit(item); } state.builder.AddFunction(spv::OpBranch, merge_label); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index ad10b2bb4..b0b7bd8de 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -4,6 +4,7 @@ #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include +#include namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { namespace { @@ -339,14 +340,6 @@ Prospero::BufferFormat BufferFormat(const ValueEmitContext& ctx, const IR::Memor : StorageBufferFormat(ctx.state, mem); } -IR::MemoryInfo RebaseFormattedComponent(IR::MemoryInfo mem, const Format::BufferFormatInfo& info, - uint32_t component) { - mem.offset += Format::GetFormatComponentByteOffset(info, component); - mem.data_dwords = 1u; - mem.component_index = component; - return mem; -} - IR::MemoryInfo RebaseRawComponent(IR::MemoryInfo mem, uint32_t component) { mem.offset += component * 4u; mem.data_dwords = 1u; @@ -381,19 +374,17 @@ uint32_t FormattedConstant(ValueEmitContext& ctx, const Format::BufferFormatInfo } template -uint32_t LoadFormattedComponent(ValueEmitContext& ctx, const IR::MemoryInfo& mem, - const Format::BufferFormatInfo& info, - uint32_t output_component, LoadWordFn&& load_word, +uint32_t LoadFormattedComponent(ValueEmitContext& ctx, const Format::BufferFormatInfo& info, + FormattedSource source, LoadWordFn&& load_word, LoadSubwordFn&& load_subword) { - const auto source = ResolveFormattedSource(ctx, mem, info, output_component); if (source.kind != FormattedSourceKind::Memory) { return FormattedConstant(ctx, info, source.kind); } const auto component = source.component; const auto bits = info.component_bits[component]; - uint32_t raw = 0; - if (info.packed_bitfield) { - raw = load_word(component); + auto raw = info.packed_bitfield || bits == 32u ? load_word(component) + : load_subword(component, bits); + if (info.packed_bitfield || (bits < 32u && IsSignedFormatComponent(info.type))) { const auto type = IsSignedFormatComponent(info.type) ? TypeI32(ctx.state) : TypeU32(ctx.state); const auto source_value = @@ -402,45 +393,17 @@ uint32_t LoadFormattedComponent(ValueEmitContext& ctx, const IR::MemoryInfo& mem ctx.state.builder.AddFunction(IsSignedFormatComponent(info.type) ? spv::OpBitFieldSExtract : spv::OpBitFieldUExtract, type, extracted, source_value, - ConstantU32(ctx.state, info.component_bit_offset[component]), + ConstantU32(ctx.state, info.packed_bitfield + ? info.component_bit_offset[component] + : 0u), ConstantU32(ctx.state, bits)); raw = type == TypeI32(ctx.state) ? Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), extracted) : extracted; - } else if (bits == 32u) { - raw = load_word(component); - } else { - raw = load_subword(component, bits, IsSignedFormatComponent(info.type)); } return NormalizeFormatComponent(ctx.state, info, component, raw); } -uint32_t FormattedLoadPrepared(ValueEmitContext& ctx, const IR::Inst& inst, - const IR::MemoryInfo& mem, uint32_t output_component, - const MemoryResourceAccess& resource) { - const auto info = Format::GetFormatInfo(BufferFormat(ctx, mem)); - if (info.type == Format::ComponentType::Unknown) { - return LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, output_component), resource); - } - return LoadFormattedComponent( - ctx, mem, info, output_component, - [&](uint32_t component) { - return LoadWordPrepared(ctx, inst, RebaseFormattedComponent(mem, info, component), - resource); - }, - [&](uint32_t component, uint32_t bits, bool sign_extend) { - return LoadSubwordPrepared(ctx, inst, RebaseFormattedComponent(mem, info, component), - resource, bits, sign_extend); - }); -} - -uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { - return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - return FormattedLoadPrepared(ctx, inst, mem, 0u, resource); - }); -} - void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t address, uint32_t index, uint32_t bits, uint32_t data) { @@ -639,78 +602,63 @@ uint32_t AtomicDecrement(EmitterState& state, uint32_t old, uint32_t limit) { struct PreparedFormattedMemory { Format::BufferFormatInfo info; MemoryResourceAccess resource; - std::array addresses {}; - std::array indices {}; + uint32_t base = 0; uint32_t in_bounds = 0; }; -enum class FormattedAccess { Load, Store }; - PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::Inst& inst, - const IR::MemoryInfo& mem, + const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, - const Format::BufferFormatInfo& info, - uint32_t components, FormattedAccess access) { + const Format::BufferFormatInfo& info) { PreparedFormattedMemory plan; - plan.info = info; - plan.resource = resource; - std::array required_components {}; - if (access == FormattedAccess::Load) { - for (uint32_t output = 0; output < components; output++) { - const auto source = ResolveFormattedSource(ctx, mem, plan.info, output); - if (source.kind == FormattedSourceKind::Memory) { - required_components[source.component] = true; - } - } - } else { - for (uint32_t component = 0; component < std::min(components, plan.info.component_count); - component++) { - required_components[component] = true; - } + plan.info = info; + plan.resource = resource; + auto start = ByteAddress(ctx, inst, mem); + plan.in_bounds = ConstantBool(ctx.state, true); + if (resource.add_index_offset) { + const auto prefixed = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), start, + resource.byte_offset); + plan.in_bounds = Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), + prefixed, start); + start = prefixed; } - bool first_bound = true; - for (uint32_t component = 0; component < plan.info.component_count; component++) { - if (!required_components[component]) continue; - const auto byte_offset = Format::GetFormatComponentByteOffset(plan.info, component); - bool reused = false; - for (uint32_t previous = 0; previous < component; previous++) { - if (required_components[previous] && - Format::GetFormatComponentByteOffset(plan.info, previous) == byte_offset) { - plan.addresses[component] = plan.addresses[previous]; - plan.indices[component] = plan.indices[previous]; - reused = true; - break; - } - } - if (reused) continue; - const auto component_mem = RebaseFormattedComponent(mem, plan.info, component); - plan.addresses[component] = ByteAddress(ctx, inst, component_mem); - const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), - plan.addresses[component], ConstantU32(ctx.state, 2)); - plan.indices[component] = EmitMemoryElementIndex(ctx.state, resource, raw_index); - const auto component_bound = - EmitMemoryElementInBounds(ctx.state, resource, plan.indices[component]); - if (first_bound) { - plan.in_bounds = component_bound; - first_bound = false; - } else { - plan.in_bounds = AndCondition(ctx.state, plan.in_bounds, component_bound); - } - } - if (first_bound) plan.in_bounds = ConstantBool(ctx.state, true); + // Check the whole, unaligned format span before aligning the actual memory address. + const auto end = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), start, + ConstantU32(ctx.state, info.byte_size - 1u)); + const auto last_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), end, + ConstantU32(ctx.state, 2)); + plan.in_bounds = AndCondition( + ctx.state, plan.in_bounds, + AndCondition(ctx.state, + Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), end, start), + EmitMemoryElementInBounds(ctx.state, plan.resource, last_index))); + plan.base = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), start, + ConstantU32(ctx.state, ~(std::min(4u, info.byte_size) - 1u))); return plan; } +std::pair FormattedComponentAddress(EmitterState& state, + const PreparedFormattedMemory& plan, + uint32_t component) { + // Components advance sequentially from one address; do not reapply buffer swizzling. + const auto offset = Format::GetFormatComponentByteOffset(plan.info, component); + const auto address = offset == 0u ? plan.base : Binary( + state, spv::OpIAdd, TypeU32(state), plan.base, ConstantU32(state, offset)); + return {address, Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, + ConstantU32(state, 2))}; +} + uint32_t LoadFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const PreparedFormattedMemory& plan, uint32_t output_component) { return LoadFormattedComponent( - ctx, mem, plan.info, output_component, + ctx, plan.info, ResolveFormattedSource(ctx, mem, plan.info, output_component), [&](uint32_t component) { - return LoadWordInBounds(ctx, plan.resource, plan.indices[component]); + return LoadWordInBounds(ctx, plan.resource, + FormattedComponentAddress(ctx.state, plan, component).second); }, - [&](uint32_t component, uint32_t bits, bool sign_extend) { - return LoadSubwordInBounds(ctx, plan.resource, plan.addresses[component], - plan.indices[component], bits, sign_extend); + [&](uint32_t component, uint32_t bits) { + const auto [address, index] = FormattedComponentAddress(ctx.state, plan, component); + return LoadSubwordInBounds(ctx, plan.resource, address, index, bits, false); }); } @@ -731,7 +679,22 @@ uint32_t FormattedOutOfBoundsValue(ValueEmitContext& ctx, const IR::MemoryInfo& const auto source = ResolveFormattedSource(ctx, mem, plan.info, component); values[component] = FormattedConstant(ctx, plan.info, source.kind); } - return ConstructU32Composite(ctx.state, components, values); + return components == 1u ? values[0] : ConstructU32Composite(ctx.state, components, values); +} + +uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { + return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); + const auto info = Format::GetFormatInfo(BufferFormat(ctx, mem)); + if (info.type == Format::ComponentType::Unknown) { + return LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource); + } + const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info); + return EmitValueOrDefaultIfCondition( + ctx.state, plan.in_bounds, TypeU32(ctx.state), + FormattedOutOfBoundsValue(ctx, mem, plan, 1u), + [&]() { return LoadFormattedInBounds(ctx, mem, plan, 0u); }); + }); } uint32_t EncodeFormattedStoreComponent(ValueEmitContext& ctx, @@ -764,8 +727,7 @@ void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const Format::BufferFormatInfo& info, uint32_t data, uint32_t components) { // RDNA2 formatted stores transfer the entire format, zero-filling missing VGPRs. - const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, info.component_count, - FormattedAccess::Store); + const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info); EmitIfCondition(ctx.state, plan.in_bounds, [&]() { auto packed = ConstantU32(ctx.state, 0); for (uint32_t component = 0; component < info.component_count; component++) { @@ -785,14 +747,19 @@ void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst, ctx.state, packed, value, ConstantU32(ctx.state, info.component_bit_offset[component]), ConstantU32(ctx.state, bits)); - } else if (bits == 8u || bits == 16u) { - StoreSubwordInBounds(ctx, mem, resource, plan.addresses[component], - plan.indices[component], bits, value); } else { - StoreWordInBounds(ctx, resource, plan.indices[component], value); + const auto [address, index] = FormattedComponentAddress(ctx.state, plan, component); + if (bits == 8u || bits == 16u) { + StoreSubwordInBounds(ctx, mem, plan.resource, address, index, bits, value); + } else { + StoreWordInBounds(ctx, plan.resource, index, value); + } } } - if (info.packed_bitfield) StoreWordInBounds(ctx, resource, plan.indices[0], packed); + if (info.packed_bitfield) { + StoreWordInBounds(ctx, plan.resource, + FormattedComponentAddress(ctx.state, plan, 0u).second, packed); + } }); } @@ -838,77 +805,161 @@ uint32_t LoadIndirectScalarBuffer(ValueEmitContext& ctx, const IR::Inst& inst) { }); } -uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) { - auto& state = ctx.state; - const auto& handle = *inst.Arg(0).ResolveInstruction(); - const auto word1 = ctx.Arg(handle, 1); - const auto records = ctx.Arg(handle, 2); - const auto word3 = ctx.Arg(handle, 3); - const auto field = [&](uint32_t word, uint32_t first, uint32_t count) { - return EmitBitFieldUExtract(state, word, ConstantU32(state, first), - ConstantU32(state, count)); +struct IndirectBufferAccess { + uint32_t address; + uint32_t offset; + uint32_t stride; + uint32_t swizzle; + uint32_t format; + uint32_t selector; + uint32_t mode; + uint32_t records; + uint32_t index_in_bounds; + uint32_t scalar_in_bounds; + uint32_t raw_records; + uint32_t raw_index_in_bounds; +}; + +IndirectBufferAccess PrepareIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + const auto& handle = *inst.Arg(0).ResolveInstruction(); + const auto word1 = ctx.Arg(handle, 1); + const auto records = ctx.Arg(handle, 2); + const auto word3 = ctx.Arg(handle, 3); + const auto field = [&](uint32_t word, uint32_t first, uint32_t count) { + return EmitBitFieldUExtract(state, word, ConstantU32(state, first), ConstantU32(state, count)); }; const auto nonzero = [&](uint32_t value) { return Binary(state, spv::OpINotEqual, TypeBool(state), value, ConstantU32(state, 0)); }; - const auto has_dword = [&](uint32_t offset, uint32_t size) { - return AndCondition( - state, - Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), size, ConstantU32(state, 4)), - Binary(state, spv::OpULessThanEqual, TypeBool(state), offset, - Binary(state, spv::OpISub, TypeU32(state), size, ConstantU32(state, 4)))); - }; - const auto stride = field(word1, 16, 14); - const auto swizzle = AndCondition(state, nonzero(stride), nonzero(field(word1, 31, 1))); - const auto index_stride = field(word3, 21, 2); - const auto add_tid = nonzero(field(word3, 23, 1)); - const auto index = - Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1), - Select(state, TypeU32(state), add_tid, BufferLane(state), ConstantU32(state, 0))); + const auto stride = field(word1, 16, 14); + const auto swizzle = AndCondition(state, nonzero(stride), nonzero(field(word1, 31, 1))); + const auto index = Binary( + state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1), + Select(state, TypeU32(state), nonzero(field(word3, 23, 1)), BufferLane(state), + ConstantU32(state, 0))); const auto soffset = ctx.Arg(inst, 3); - const auto base = DeviceAddressFromWords(state, ctx.Arg(handle, 0), field(word1, 0, 16)); - const auto valid_format = nonzero(field(word3, 12, 7)); - const auto mode = field(word3, 28, 2); - const auto index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records); - const auto scalar_in_bounds = - Binary(state, spv::OpULessThanEqual, TypeBool(state), soffset, records); + const auto address = CalculateBufferAddress(state, index, ctx.Arg(inst, 2), soffset, + ctx.Memory(inst).offset, stride, swizzle, + field(word3, 21, 2)); + const auto base = DeviceAddressFromWords(state, ctx.Arg(handle, 0), field(word1, 0, 16)); const auto raw_records = Binary(state, spv::OpISub, TypeU32(state), records, soffset); - const auto raw_index_in_bounds = - Binary(state, spv::OpULessThan, TypeBool(state), index, raw_records); + return { + .address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base, + Unary(state, spv::OpUConvert, TypeScalarU64(state), address.byte)), + .offset = address.offset, + .stride = stride, + .swizzle = swizzle, + .format = field(word3, 12, 7), + .selector = ctx.Memory(inst).formatted ? field(word3, 0, 3) : 0u, + .mode = field(word3, 28, 2), + .records = records, + .index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records), + .scalar_in_bounds = Binary(state, spv::OpULessThanEqual, TypeBool(state), soffset, records), + .raw_records = raw_records, + .raw_index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, raw_records), + }; +} + +uint32_t IndirectBufferInBounds(EmitterState& state, const IndirectBufferAccess& buffer, + uint32_t displacement, uint32_t bytes, bool formatted) { + const auto offset = displacement == 0u ? buffer.offset : Binary( + state, spv::OpIAdd, TypeU32(state), buffer.offset, ConstantU32(state, displacement)); + const auto has_payload = [&](uint32_t size) { + return AndCondition( + state, Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), size, + ConstantU32(state, bytes)), + Binary(state, spv::OpULessThanEqual, TypeBool(state), offset, + Binary(state, spv::OpISub, TypeU32(state), size, ConstantU32(state, bytes)))); + }; + const auto structured = AndCondition( + state, buffer.index_in_bounds, + formatted ? has_payload(buffer.stride) + : Binary(state, spv::OpULessThan, TypeBool(state), offset, buffer.stride)); + // OOB_SELECT=3 reduces NUM_RECORDS by SOFFSET before its offset/index checks. + const auto raw = AndCondition( + state, buffer.scalar_in_bounds, + Select(state, TypeBool(state), buffer.swizzle, + AndCondition(state, buffer.raw_index_in_bounds, has_payload(buffer.stride)), + has_payload(buffer.raw_records))); + auto in_bounds = Select( + state, TypeBool(state), + Binary(state, spv::OpIEqual, TypeBool(state), buffer.mode, ConstantU32(state, 0)), + structured, buffer.index_in_bounds); + in_bounds = Select( + state, TypeBool(state), + Binary(state, spv::OpULessThan, TypeBool(state), buffer.mode, ConstantU32(state, 2)), + in_bounds, + Select(state, TypeBool(state), + Binary(state, spv::OpIEqual, TypeBool(state), buffer.mode, ConstantU32(state, 2)), + Binary(state, spv::OpINotEqual, TypeBool(state), buffer.records, ConstantU32(state, 0)), + raw)); + return in_bounds; +} + +uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) { + auto& state = ctx.state; + const auto buffer = PrepareIndirectBuffer(ctx, inst); + const auto valid_format = Binary(state, spv::OpINotEqual, TypeBool(state), buffer.format, + ConstantU32(state, 0)); std::array values {}; - for (uint32_t component = 0; component < components; component++) { - const auto address = CalculateBufferAddress(state, index, ctx.Arg(inst, 2), soffset, - ctx.Memory(inst).offset + component * 4u, - stride, swizzle, index_stride); - // RDNA2 raw DWORD vectors check each component, unlike formatted accesses. - const auto structured_bounds = - AndCondition(state, index_in_bounds, - Binary(state, spv::OpULessThan, TypeBool(state), address.offset, stride)); - // OOB_SELECT=3 reduces NUM_RECORDS by SOFFSET before the offset/index check. - const auto raw_bounds = AndCondition( - state, scalar_in_bounds, - Select(state, TypeBool(state), swizzle, - AndCondition(state, raw_index_in_bounds, has_dword(address.offset, stride)), - has_dword(address.offset, raw_records))); - auto in_bounds = - Select(state, TypeBool(state), - Binary(state, spv::OpIEqual, TypeBool(state), mode, ConstantU32(state, 0)), - structured_bounds, index_in_bounds); - in_bounds = Select( - state, TypeBool(state), - Binary(state, spv::OpULessThan, TypeBool(state), mode, ConstantU32(state, 2)), - in_bounds, - Select(state, TypeBool(state), - Binary(state, spv::OpIEqual, TypeBool(state), mode, ConstantU32(state, 2)), - nonzero(records), raw_bounds)); - const auto guest = - Binary(state, spv::OpIAdd, TypeScalarU64(state), base, - Unary(state, spv::OpUConvert, TypeScalarU64(state), address.byte)); - values[component] = LoadBda(ctx, guest, AndCondition(state, valid_format, in_bounds), 32u); + for (uint32_t component = 0; component < components; ++component) { + const auto address = component == 0u ? buffer.address : Binary( + state, spv::OpIAdd, TypeScalarU64(state), buffer.address, + ConstantDeviceAddress(state, component * 4u)); + values[component] = LoadBda( + ctx, address, AndCondition(state, valid_format, + IndirectBufferInBounds(state, buffer, component * 4u, 4u, false)), + 32u); } return ConstructU32Composite(state, components, values); } +uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + return EmitValueOrZeroIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto buffer = PrepareIndirectBuffer(ctx, inst); + return EmitIndexSwitch( + state, buffer.format, static_cast(Prospero::BufferFormat::k32_32_32_32Float) + 1u, + TypeU32(state), [&](uint32_t format) { + const auto info = Format::GetFormatInfo(static_cast(format)); + if (info.type == Format::ComponentType::Unknown) return ConstantU32(state, 0); + const auto constant = Select( + state, TypeU32(state), + Binary(state, spv::OpIEqual, TypeBool(state), buffer.selector, ConstantU32(state, 1)), + FormattedConstant(ctx, info, FormattedSourceKind::One), ConstantU32(state, 0)); + return EmitValueOrDefaultIfCondition( + state, Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), buffer.selector, + ConstantU32(state, 4)), + TypeU32(state), constant, [&]() { + const auto in_bounds = IndirectBufferInBounds(state, buffer, 0u, info.byte_size, true); + const auto base = Binary( + state, spv::OpBitwiseAnd, TypeScalarU64(state), buffer.address, + ConstantDeviceAddress(state, ~(uint64_t(std::min(4u, info.byte_size)) - 1u))); + const auto load = [&](uint32_t component, uint32_t bits) { + const auto offset = Format::GetFormatComponentByteOffset(info, component); + const auto address = offset == 0u ? base : Binary( + state, spv::OpIAdd, TypeScalarU64(state), base, + ConstantDeviceAddress(state, offset)); + return LoadBda(ctx, address, in_bounds, bits); + }; + const auto emit_component = [&](uint32_t component) { + return LoadFormattedComponent( + ctx, info, {FormattedSourceKind::Memory, component}, + [&](uint32_t source) { return load(source, 32u); }, load); + }; + if (info.component_count == 1u) return emit_component(0u); + const auto component = Binary( + state, spv::OpUMod, TypeU32(state), + Binary(state, spv::OpISub, TypeU32(state), buffer.selector, ConstantU32(state, 4)), + ConstantU32(state, info.component_count)); + return EmitIndexSwitch(state, component, info.component_count, TypeU32(state), + emit_component); + }); + }); + }); +} + uint32_t LoadWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) { auto& state = ctx.state; return EmitValueOrDefaultIfCondition( @@ -922,8 +973,7 @@ uint32_t LoadWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t co const auto info = Format::GetFormatInfo( mem.formatted ? BufferFormat(ctx, mem) : Prospero::BufferFormat::kInvalid); if (info.type != Format::ComponentType::Unknown) { - const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, - FormattedAccess::Load); + const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info); return EmitValueOrDefaultIfCondition( state, plan.in_bounds, TypeU32Composite(state, components), FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() { @@ -1319,7 +1369,8 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) { value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1), address_info.data_bits); else if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted) - value = FormattedLoad(ctx, inst, mem); + value = mem.kind == IR::ResourceKind::IndirectBuffer ? LoadIndirectFormattedX(ctx, inst) + : FormattedLoad(ctx, inst, mem); else if (inst.GetType() == IR::Type::U8) value = LoadSubword(ctx, inst, mem, 8, false); else if (inst.GetType() == IR::Type::U16) diff --git a/src/graphics/shader/recompiler/ir/Program.cpp b/src/graphics/shader/recompiler/ir/Program.cpp index ffaa2355e..fe76ed8e0 100644 --- a/src/graphics/shader/recompiler/ir/Program.cpp +++ b/src/graphics/shader/recompiler/ir/Program.cpp @@ -458,7 +458,7 @@ void ValidateProgram(const Program& program, bool require_ssa) { } if (memory.kind == ResourceKind::IndirectBuffer && !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) { - return Fail("indirect buffer requires a scalar or raw DWORD x2/x3/x4 load"); + return Fail("indirect buffer requires a scalar, raw DWORD x2/x3/x4, or formatted X load"); } if (buffer_components > 1u && (!vector_buffer || memory.data_bits != 32u || diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 82ac5153c..8c8d817ec 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -73,9 +73,12 @@ struct MemoryInfo { bool planning_only = false; [[nodiscard]] bool SupportsIndirectBufferLoad(ValueOpcode opcode) const { - return !formatted && !typed && data_bits == 32u && - (opcode == ValueOpcode::ReadConstBuffer || opcode == ValueOpcode::LoadBufferU32x2 || - opcode == ValueOpcode::LoadBufferU32x3 || opcode == ValueOpcode::LoadBufferU32x4); + return !typed && data_bits == 32u && + (formatted ? opcode == ValueOpcode::LoadBufferU32 + : opcode == ValueOpcode::ReadConstBuffer || + opcode == ValueOpcode::LoadBufferU32x2 || + opcode == ValueOpcode::LoadBufferU32x3 || + opcode == ValueOpcode::LoadBufferU32x4); } bool operator==(const MemoryInfo& other) const = default; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index d200e5121..cd4210854 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -2078,7 +2078,7 @@ private: !memory.SupportsIndirectBufferLoad(op)) { Fail(flags.pc, "buffer descriptor is not a valid runtime value; GPU-selected access " - "requires a scalar or raw DWORD x2/x3/x4 load"); + "requires a scalar, raw DWORD x2/x3/x4, or formatted X load"); } m_program.memory_info[flags.index].kind = ResourceKind::IndirectBuffer; m_info.uses_dma = true; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index fb42553b1..f985c3237 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -26245,8 +26245,8 @@ TestCase BufferLoadsGpuSelectedDescriptors(bool xyz) { {12, 20, 3, 4, false, true, {5, 6, 7, 8, 0, 0}}, {12, 2, 0, 0, false, false, {}}, // RDNA2 swizzling uses four-byte elements; soffset is added afterward. - {12, 2, 0, 4, true, true, {3, 11, 19, 0, 0, 0}}, - {12, 8, 3, 4, true, true, {3, 11, 19, 0, 0, 0}}, + {12, 2, 0, 4, true, true, {3, 4, 5, 0, 0, 0}}, + {12, 8, 3, 4, true, true, {3, 4, 5, 0, 0, 0}}, {12, 4, 3, 4, true, true, {}}, }; TestCase test; @@ -26309,6 +26309,95 @@ TestCase BufferLoadDwordx3GpuSelectedDescriptors() { return BufferLoadsGpuSelectedDescriptors(true); } +TestCase BufferLoadFormatXGpuSelectedDescriptors() { + using O = ShaderOpcode; + using F = Prospero::BufferFormat; + constexpr uint64_t GuestBase = 0x0000000110000000ull; + struct DescriptorCase { + F format; + u32 selector, stride, records, mode, index, offset, soffset; + bool swizzle; + u32 expected; + }; + constexpr DescriptorCase cases[] = { + {F::k8UInt, 4, 1, 4, 3, 1, 0, 0, false, 0x7f}, + {F::k8UInt, 4, 0, 4, 3, 0, 3, 0, false, 0x80}, + {F::k16UInt, 4, 2, 4, 0, 1, 0, 0, false, 0x80ff}, + {F::k32UInt, 4, 4, 2, 0, 1, 0, 0, false, 0x8001ff00}, + {F::k32_32_32_32Float, 7, 16, 1, 0, 0, 0, 0, false, 0x3fc00000}, + {F::k8SInt, 4, 0, 16, 3, 0, 3, 0, false, 0xffffff80}, + {F::k16SInt, 4, 0, 16, 3, 0, 2, 0, false, 0xffff80ff}, + {F::k8UNorm, 4, 0, 16, 3, 0, 2, 0, false, 0x3f800000}, + {F::k8SNorm, 4, 0, 16, 3, 0, 3, 0, false, 0xbf800000}, + {F::k8UScaled, 4, 0, 16, 3, 0, 1, 0, false, 0x42fe0000}, + {F::k8SScaled, 4, 0, 16, 3, 0, 3, 0, false, 0xc3000000}, + {F::k16Float, 4, 0, 16, 3, 0, 8, 0, false, 0x3f800000}, + {F::k11_11_10UInt, 5, 0, 16, 3, 0, 0, 0, false, 0x7ef}, + {F::k10_10_10_2UInt, 7, 0, 16, 3, 0, 0, 0, false, 2}, + {F::k8_8_8_8UInt, 6, 0, 16, 3, 0, 0, 0, false, 0xff}, + // Align the whole format first, then advance components from one swizzled base. + {F::k16UInt, 4, 0, 16, 3, 0, 1, 0, false, 0x7f01}, + {F::k16UInt, 4, 0, 4, 3, 0, 3, 0, false, 0}, + {F::k32_32Float, 5, 8, 2, 0, 1, 0, 4, true, 0x3fc00000}, + // FORMAT_X still transfers a complete RGBA32 record for range checking. + {F::k32_32_32_32Float, 4, 0, 16, 3, 0, 4, 0, false, 0}, + {F::k32_32_32_32Float, 4, 8, 1, 0, 0, 0, 0, false, 0}, + {F::k32_32_32_32Float, 1, 0, 16, 3, 0, 4, 0, false, 0x3f800000}, + {F::k32UInt, 1, 4, 0, 0, 0, 0, 0, false, 1}, + {F::k32UInt, 0, 4, 1, 0, 0, 0, 0, false, 0}, + {F::kInvalid, 1, 4, 1, 0, 0, 0, 0, false, 0}, + {F::k32UInt, 4, 4, 0, 2, 0, 0, 0, false, 0}, + {F::k32UInt, 4, 0, 4, 3, 0, 0, 8, false, 0}, + }; + TestCase test; + test.name = "BufferLoadFormatXGpuSelectedDescriptors"; + test.initial.resize(2048); + constexpr std::array payload{0x80ff7f01u, 0x8001ff00u, 0x40003c00u, 0x3fc00000u}; + for (u32 i = 0; i < std::size(cases); ++i) { + const auto &input = cases[i]; + const u32 data_offset = 4096 + i * 64; + const std::array row{ + static_cast(GuestBase + data_offset), + 1u | (input.stride << 16u) | (input.swizzle ? 1u << 31u : 0u), + input.records, input.selector | (static_cast(input.format) << 12u) | + (input.mode << 28u), + input.index, input.offset, input.soffset, 0}; + std::copy(row.begin(), row.end(), test.initial.begin() + 128 + i * 8); + std::copy(payload.begin(), payload.end(), test.initial.begin() + data_offset / 4); + // Read the selection on the GPU, as GTA's descriptor waterfall does. + test.initial[64 + i] = std::size(cases) - 1 - i; + test.expected.push_back(cases[std::size(cases) - 1 - i].expected); + } + auto &code = test.code; + AppendSMovLiteral(&code, 28, 0); + const auto loop = static_cast(code.size()); + code.push_back(EncodeVop1(0x01, 30, 28)); + code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 30)); + code.push_back(EncodeMubuf0(0x0c, 256)); + code.push_back(EncodeMubuf1(2, 0, 30)); + code.push_back(EncodeVop1(0x02, 20, Vgpr(2))); + code.push_back(EncodeSop2(0x1e, 20, 20, InlineU32(5))); + code.push_back(EncodeSmem0(0x0b, 8, 0)); + code.push_back(EncodeSmem1(512, 20)); + code.push_back(EncodeVop1(0x01, 21, 12)); + code.push_back(EncodeVop1(0x01, 22, 13)); + code.push_back(EncodeMubuf0(0x00, 0, true, true)); + code.push_back(EncodeMubuf1(4, 2, 21, 14)); + AppendBufferStoreDword(&code, 4, 30); + code.push_back(EncodeSop2(0x00, 28, 28, InlineU32(1))); + code.push_back(EncodeSopc(0x0a, 28, InlineU32(std::size(cases)))); + code.push_back(EncodeSopp(0x05, loop - static_cast(code.size()) - 1u)); + AppendEnd(&code); + test.bda_mappings = {{GuestBase, 0}}; + test.required_spirv = {"PhysicalStorageBuffer", "OpSwitch", "OpLoopMerge"}; + test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHLREV_B32, + O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_LSHL_B32, + O::S_BUFFER_LOAD_DWORDX8, O::BUFFER_LOAD_FORMAT_X, + O::BUFFER_STORE_DWORD, O::S_ADD_U32, O::S_CMP_LT_U32, + O::S_CBRANCH_SCC1, O::S_ENDPGM}; + return test; +} + TestCase BufferStoreDwordx4DropsOnlyOutOfBoundsTail() { using O = ShaderOpcode; @@ -26383,23 +26472,27 @@ TestCase BufferStoreFormatXyzwDropsPartialRecord() { return test; } -TestCase BufferLoadFormatXChecksOnlyTransferredComponent() { +TestCase BufferLoadFormatXRejectsPartialRecord() { using O = ShaderOpcode; std::vector code; AppendVMovU32(&code, 20, 12); AppendBufferLoadOpcode(&code, 0x00, 0, 20); + // Bounds precede alignment: rounding offset 1 down must not make its full format fit. + AppendVMovU32(&code, 20, 1); + AppendBufferLoadOpcode(&code, 0x00, 1, 20); AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); AppendEnd(&code); TestCase test; - test.name = "BufferLoadFormatXChecksOnlyTransferredComponent"; + test.name = "BufferLoadFormatXRejectsPartialRecord"; test.code = std::move(code); test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; - test.expected = {0x44444444u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.expected = {0u, 0u, 0x33333333u, 0x44444444u}; test.storage_buffer_range_dwords = 4; test.user_data = MakeStructuredStorageBufferData( - 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + 0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -26429,7 +26522,7 @@ TestCase BufferStoreFormatXRejectsPartialRecord() { return test; } -TestCase BufferLoadFormatXyChecksOnlyTransferredComponents() { +TestCase BufferLoadFormatXyRejectsPartialRecord() { using O = ShaderOpcode; std::vector code; @@ -26440,13 +26533,13 @@ TestCase BufferLoadFormatXyChecksOnlyTransferredComponents() { AppendEnd(&code); TestCase test; - test.name = "BufferLoadFormatXyChecksOnlyTransferredComponents"; + test.name = "BufferLoadFormatXyRejectsPartialRecord"; test.code = std::move(code); test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; - test.expected = {0x33333333u, 0x44444444u, 0x33333333u, 0x44444444u}; + test.expected = {0u, 0u, 0x33333333u, 0x44444444u}; test.storage_buffer_range_dwords = 4; test.user_data = MakeStructuredStorageBufferData( - 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + 0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -32824,12 +32917,13 @@ std::vector MakeCases() { AddCase(BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail); AddCase(BufferLoadsGpuSelectedDescriptors); AddCase(BufferLoadDwordx3GpuSelectedDescriptors); + AddCase(BufferLoadFormatXGpuSelectedDescriptors); AddCase(BufferStoreDwordx4DropsOnlyOutOfBoundsTail); AddCase(BufferLoadFormatXyzwRejectsPartialRecord); AddCase(BufferStoreFormatXyzwDropsPartialRecord); - AddCase(BufferLoadFormatXChecksOnlyTransferredComponent); + AddCase(BufferLoadFormatXRejectsPartialRecord); AddCase(BufferStoreFormatXRejectsPartialRecord); - AddCase(BufferLoadFormatXyChecksOnlyTransferredComponents); + AddCase(BufferLoadFormatXyRejectsPartialRecord); AddCase(BufferStoreFormatXyRejectsPartialRecord); AddCase(BufferStoreVariants); AddCase(BufferFormatVariants); @@ -37960,6 +38054,9 @@ int main(int argc, char **argv) { CheckIndirectBufferStore(vulkan); RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadDwordx3GpuSelectedDescriptors()); + RunCase(&vulkan, BufferLoadFormatXGpuSelectedDescriptors()); + RunCase(&vulkan, BufferLoadFormatXRejectsPartialRecord()); + RunCase(&vulkan, BufferLoadFormatXyRejectsPartialRecord()); RunCase(&vulkan, BufferLoadDwordx4SnapshotsOverlappingAddress()); RunCase(&vulkan, BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail()); RunCase(&vulkan, BufferLoadDwordIdxenUsesDescriptorStride());