diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 8cd053e93..b81d886f3 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -632,7 +632,9 @@ uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputIn continue; } const auto* load = FindEmbeddedFetchLoad(loads, inst.pc); - if (load == nullptr || inst.memory.component_index >= load->components) { + if (load == nullptr || inst.memory.data_dwords != load->components || + inst.dst.kind != IR::OperandKind::Register || + inst.dst.reg.file != IR::RegisterFile::Vector) { continue; } @@ -644,16 +646,16 @@ uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputIn continue; } - inst.op = IR::Opcode::LoadInputF32; - inst.input_info.attr = static_cast(resource_id); - inst.input_info.chan = inst.memory.component_index; - inst.memory = {}; - inst.src_count = 0; - + inst.op = IR::Opcode::LoadInputF32; + inst.input_info.attr = static_cast(resource_id); + inst.input_info.chan = 0; + inst.input_info.component_count = load->components; + inst.memory = {}; + inst.src_count = 0; mutable_input_info->resource_fetch_components[resource_id] = std::max(mutable_input_info->resource_fetch_components[resource_id], - static_cast(inst.input_info.chan + 1u)); - rewritten++; + static_cast(load->components)); + rewritten += load->components; } } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp index 774986b62..bdf1d23c1 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp @@ -323,6 +323,10 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); return true; + case IR::ValueOpcode::CompositeConstructU32x3: + ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x3, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); + return true; case IR::ValueOpcode::CompositeConstructF32x2: ctx.Emit(inst, OpCompositeConstruct, IR::Type::F32x2, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); @@ -332,6 +336,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) { {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)}); return true; case IR::ValueOpcode::CompositeExtractU32x2: + case IR::ValueOpcode::CompositeExtractU32x3: case IR::ValueOpcode::CompositeExtractU32x4: ctx.Emit(inst, OpCompositeExtract, IR::Type::U32, {ctx.Arg(inst, 0), inst.Arg(1).U32()}); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index 30ba1b7a3..4ce3f14ed 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -392,7 +392,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Define( inst, EmitValueOrDefaultIfCondition( - state, condition, TypeU32Vector(state, 4), ConstantU32x4Zero(state), [&]() { + state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4), [&]() { const auto image = LoadSampledImageDescriptor(state, mem, pc, view); const auto color = state.builder.AllocateId(); const auto coord = CoordU32(ctx, mem, *address, view); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 1f442f41e..75318c646 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -342,6 +342,8 @@ uint32_t TypeI32(EmitterState& state); uint32_t TypeI32Pair(EmitterState& state); uint32_t TypeF32(EmitterState& state); uint32_t TypeU32Vector(EmitterState& state, uint32_t components); + +uint32_t TypeU32Composite(EmitterState& state, uint32_t components); uint32_t TypeI32Vector(EmitterState& state, uint32_t components); uint32_t TypeF32Vector(EmitterState& state, uint32_t components); uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t pointee); @@ -615,7 +617,7 @@ uint32_t ConstantBool(EmitterState& state, bool value); uint32_t ConstantU64(EmitterState& state, uint64_t value); -uint32_t ConstantU32x4Zero(EmitterState& state); +uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components); uint32_t GlslStd450(EmitterState& state); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 4aa43caa1..31034c617 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -163,47 +163,78 @@ PreparedMemoryElement PrepareMemoryElement(ValueEmitContext& ctx, const IR::Memo return {.resource = resource, .index = index}; } +PreparedMemoryElement PrepareMemoryElement(ValueEmitContext& ctx, const IR::MemoryInfo& mem, + const MemoryResourceAccess& resource, + uint32_t raw_index) { + return {.resource = resource, .index = EmitMemoryElementIndex(ctx.state, resource, raw_index)}; +} + +uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, + uint32_t index); + +uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, + uint32_t address, uint32_t index, uint32_t bits, bool sign_extend); + +uint32_t LoadWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + const MemoryResourceAccess& resource) { + const auto access = PrepareMemoryElement(ctx, mem, resource, DwordIndex(ctx, inst, mem)); + return EmitValueOrZeroIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), + [&]() { return LoadWordInBounds(ctx, access.resource, access.index); }); +} + uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) { return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); - return EmitValueOrZeroIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { - const auto value = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction( - {OpLoad, TypeU32(ctx.state), value, - EmitMemoryElementPointer(ctx.state, access.resource, access.index)}); - return value; - }); + const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); + return LoadWordPrepared(ctx, inst, mem, resource); }); } +uint32_t LoadSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + const MemoryResourceAccess& resource, uint32_t bits, + bool sign_extend) { + const auto address = ByteAddress(ctx, inst, mem); + const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address, + ConstantU32(ctx.state, 2)); + const auto access = PrepareMemoryElement(ctx, mem, resource, raw_index); + return EmitValueOrZeroIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + return LoadSubwordInBounds(ctx, access.resource, address, access.index, bits, + sign_extend); + }); +} + +uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, + uint32_t index) { + const auto value = ctx.state.builder.AllocateId(); + const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index); + ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), value, pointer}); + return value; +} + +uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, + uint32_t address, uint32_t index, uint32_t bits, bool sign_extend) { + const auto word = LoadWordInBounds(ctx, resource, index); + const auto byte = + Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address, ConstantU32(ctx.state, 3)); + const auto shift = + Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), byte, ConstantU32(ctx.state, 3)); + const auto value = + Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), + Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), word, shift), + ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)); + if (!sign_extend) return value; + const auto left = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), value, + ConstantU32(ctx.state, 32u - bits)); + return Binary(ctx.state, OpShiftRightArithmetic, TypeU32(ctx.state), left, + ConstantU32(ctx.state, 32u - bits)); +} + uint32_t LoadSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits, bool sign_extend) { return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto address = ByteAddress(ctx, inst, mem); - const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address, - ConstantU32(ctx.state, 2)); - const auto access = PrepareMemoryElement(ctx, mem, raw_index); - return EmitValueOrZeroIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { - const auto word = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction( - {OpLoad, TypeU32(ctx.state), word, - EmitMemoryElementPointer(ctx.state, access.resource, access.index)}); - const auto byte = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address, - ConstantU32(ctx.state, 3)); - const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), byte, - ConstantU32(ctx.state, 3)); - const auto value = - Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), - Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), word, shift), - ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)); - if (!sign_extend) return value; - const auto left = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), value, - ConstantU32(ctx.state, 32u - bits)); - return Binary(ctx.state, OpShiftRightArithmetic, TypeU32(ctx.state), left, - ConstantU32(ctx.state, 32u - bits)); - }); + const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); + return LoadSubwordPrepared(ctx, inst, mem, resource, bits, sign_extend); }); } @@ -215,60 +246,117 @@ Prospero::BufferFormat BufferFormat(const ValueEmitContext& ctx, const IR::Inst& IR::MemoryInfo RebaseFormattedComponent(IR::MemoryInfo mem, Prospero::BufferFormat format, uint32_t component) { - if (!mem.typed) { - const auto component_slot = mem.component_index * 4u; - mem.offset = mem.offset >= component_slot ? mem.offset - component_slot : 0u; - mem.offset += Format::GetFormatComponentByteOffset(format, component); - } + mem.offset += Format::GetFormatComponentByteOffset(format, component); + mem.data_dwords = 1u; + mem.component_index = component; return mem; } -uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { - const auto format = BufferFormat(ctx, inst, mem); - if (!Format::IsKnownFormat(format)) return LoadWord(ctx, inst, mem); - const auto info = Format::GetFormatInfo(format); - auto component = mem.component_index; - if (!mem.typed) { - const auto selector = - GetDstSel(ctx.state.program.info.buffers[mem.resource].descriptor_swizzle, component); - if (selector == 0u) return ConstantU32(ctx.state, 0); - if (selector == 1u) { - const auto integer = info.type == Format::ComponentType::Uint || - info.type == Format::ComponentType::Sint; - return ConstantU32(ctx.state, integer ? 1u : 0x3f800000u); - } - if (selector < 4u) { - ExitDescriptorBindingFailure(ctx.state, IR::DescriptorBindingKind::Buffers, - mem.resource, "buffer descriptor has reserved dst_sel"); - } - component = selector - 4u; +IR::MemoryInfo RebaseRawComponent(IR::MemoryInfo mem, uint32_t component) { + mem.offset += component * 4u; + mem.data_dwords = 1u; + mem.component_index = component; + return mem; +} + +enum class FormattedSourceKind { Memory, Zero, One }; + +struct FormattedSource { + FormattedSourceKind kind = FormattedSourceKind::Zero; + uint32_t component = 0; +}; + +FormattedSource ResolveFormattedSource(ValueEmitContext& ctx, const IR::MemoryInfo& mem, + const Format::BufferFormatInfo& info, + uint32_t output_component) { + if (mem.typed) { + return output_component < info.component_count + ? FormattedSource {FormattedSourceKind::Memory, output_component} + : FormattedSource {}; } - if (component >= info.component_count) return ConstantU32(ctx.state, 0); - uint32_t raw = 0; - const auto bits = info.component_bits[component]; + const auto selector = GetDstSel(ctx.state.program.info.buffers[mem.resource].descriptor_swizzle, + output_component); + if (selector == 0u) return {}; + if (selector == 1u) return {FormattedSourceKind::One, 0}; + if (selector < 4u) { + ExitDescriptorBindingFailure(ctx.state, IR::DescriptorBindingKind::Buffers, mem.resource, + "buffer descriptor has reserved dst_sel"); + } + const auto component = selector - 4u; + return component < info.component_count + ? FormattedSource {FormattedSourceKind::Memory, component} + : FormattedSource {}; +} + +uint32_t FormattedConstant(ValueEmitContext& ctx, const Format::BufferFormatInfo& info, + FormattedSourceKind kind) { + if (kind != FormattedSourceKind::One) return ConstantU32(ctx.state, 0); + const auto integer = + info.type == Format::ComponentType::Uint || info.type == Format::ComponentType::Sint; + return ConstantU32(ctx.state, integer ? 1u : 0x3f800000u); +} + +template +uint32_t LoadFormattedComponent(ValueEmitContext& ctx, const IR::MemoryInfo& mem, + Prospero::BufferFormat format, const Format::BufferFormatInfo& info, + uint32_t output_component, LoadWordFn&& load_word, + LoadSubwordFn&& load_subword) { + const auto source = ResolveFormattedSource(ctx, mem, info, output_component); + if (source.kind != FormattedSourceKind::Memory) { + return FormattedConstant(ctx, info, source.kind); + } + const auto component = source.component; + const auto bits = info.component_bits[component]; + uint32_t raw = 0; if (info.packed_bitfield) { - raw = LoadWord(ctx, inst, RebaseFormattedComponent(mem, format, component)); + raw = load_word(component); const auto type = IsSignedFormatComponent(info.type) ? TypeI32(ctx.state) : TypeU32(ctx.state); - const auto source = + const auto source_value = type == TypeI32(ctx.state) ? Unary(ctx.state, OpBitcast, type, raw) : raw; const auto extracted = ctx.state.builder.AllocateId(); ctx.state.builder.AddFunction( {IsSignedFormatComponent(info.type) ? OpBitFieldSExtract : OpBitFieldUExtract, type, - extracted, source, ConstantU32(ctx.state, info.component_bit_offset[component]), + extracted, source_value, ConstantU32(ctx.state, info.component_bit_offset[component]), ConstantU32(ctx.state, bits)}); raw = type == TypeI32(ctx.state) ? Unary(ctx.state, OpBitcast, TypeU32(ctx.state), extracted) : extracted; } else if (bits == 32u) { - raw = LoadWord(ctx, inst, RebaseFormattedComponent(mem, format, component)); + raw = load_word(component); } else { - raw = LoadSubword(ctx, inst, RebaseFormattedComponent(mem, format, component), bits, - IsSignedFormatComponent(info.type)); + raw = load_subword(component, bits, IsSignedFormatComponent(info.type)); } return NormalizeFormatComponent(ctx.state, info, component, raw); } +uint32_t FormattedLoadPrepared(ValueEmitContext& ctx, const IR::Inst& inst, + const IR::MemoryInfo& mem, uint32_t output_component, + const MemoryResourceAccess& resource) { + const auto format = BufferFormat(ctx, inst, mem); + if (!Format::IsKnownFormat(format)) { + return LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, output_component), resource); + } + const auto info = Format::GetFormatInfo(format); + return LoadFormattedComponent( + ctx, mem, format, info, output_component, + [&](uint32_t component) { + return LoadWordPrepared(ctx, inst, RebaseFormattedComponent(mem, format, component), + resource); + }, + [&](uint32_t component, uint32_t bits, bool sign_extend) { + return LoadSubwordPrepared(ctx, inst, RebaseFormattedComponent(mem, format, component), + resource, bits, sign_extend); + }); +} + +uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { + return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); + return FormattedLoadPrepared(ctx, inst, mem, 0u, resource); + }); +} + template uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) { const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; @@ -304,75 +392,104 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki return observed; } +void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, + const MemoryResourceAccess& resource, uint32_t address, uint32_t index, + uint32_t bits, uint32_t data) { + const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index); + const auto shift = Binary( + ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), + Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address, ConstantU32(ctx.state, 3)), + ConstantU32(ctx.state, 3)); + const auto mask = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), + ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift); + const auto value = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), + Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), data, + ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)), + shift); + const auto merge = [&](uint32_t old) { + return Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), + Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), old, + Unary(ctx.state, OpNot, TypeU32(ctx.state), mask)), + value); + }; + if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) { + AtomicUpdate(ctx.state, pointer, mem.kind, merge); + } else { + const auto old = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), old, pointer}); + ctx.state.builder.AddFunction({OpStore, pointer, merge(old)}); + } +} + +void StoreSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + const MemoryResourceAccess& resource, uint32_t bits, uint32_t data) { + const auto address = ByteAddress(ctx, inst, mem); + const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address, + ConstantU32(ctx.state, 2)); + const auto access = PrepareMemoryElement(ctx, mem, resource, raw_index); + EmitIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + StoreSubwordInBounds(ctx, mem, access.resource, address, access.index, bits, data); + }); +} + void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits) { EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto address = ByteAddress(ctx, inst, mem); - const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address, - ConstantU32(ctx.state, 2)); - const auto access = PrepareMemoryElement(ctx, mem, raw_index); - EmitIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { - const auto pointer = - EmitMemoryElementPointer(ctx.state, access.resource, access.index); - const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), - Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), - address, ConstantU32(ctx.state, 3)), - ConstantU32(ctx.state, 3)); - const auto mask = - Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), - ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift); - const auto value = - Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), - Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), - ctx.Arg(inst, inst.NumArgs() - 2), - ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)), - shift); - const auto merge = [&](uint32_t old) { - return Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), - Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), old, - Unary(ctx.state, OpNot, TypeU32(ctx.state), mask)), - value); - }; - if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) { - AtomicUpdate(ctx.state, pointer, mem.kind, merge); - } else { - const auto old = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), old, pointer}); - ctx.state.builder.AddFunction({OpStore, pointer, merge(old)}); - } - }); + const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); + StoreSubwordPrepared(ctx, inst, mem, resource, bits, ctx.Arg(inst, inst.NumArgs() - 2)); }); } +void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + const MemoryResourceAccess& resource, uint32_t data) { + const auto access = PrepareMemoryElement(ctx, mem, resource, DwordIndex(ctx, inst, mem)); + EmitIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + ctx.state.builder.AddFunction( + {OpStore, EmitMemoryElementPointer(ctx.state, access.resource, access.index), + data}); + }); +} + +void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index, + uint32_t data) { + ctx.state.builder.AddFunction( + {OpStore, EmitMemoryElementPointer(ctx.state, resource, index), data}); +} + void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) { EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); - EmitIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { - ctx.state.builder.AddFunction( - {OpStore, EmitMemoryElementPointer(ctx.state, access.resource, access.index), - ctx.Arg(inst, inst.NumArgs() - 2)}); - }); + const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); + StoreWordPrepared(ctx, inst, mem, resource, ctx.Arg(inst, inst.NumArgs() - 2)); }); } -void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { +void FormattedStorePrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + uint32_t component, const MemoryResourceAccess& resource, + uint32_t data) { const auto format = BufferFormat(ctx, inst, mem); if (!Format::IsKnownFormat(format)) { - StoreWord(ctx, inst, mem); + StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data); return; } const auto info = Format::GetFormatInfo(format); - if (mem.component_index >= info.component_count) return; - const auto bits = info.component_bits[mem.component_index]; - const auto component_mem = RebaseFormattedComponent(mem, format, mem.component_index); + if (component >= info.component_count) return; + const auto bits = info.component_bits[component]; + const auto component_mem = RebaseFormattedComponent(mem, format, component); if (bits == 8u || bits == 16u) { - StoreSubword(ctx, inst, component_mem, bits); + StoreSubwordPrepared(ctx, inst, component_mem, resource, bits, data); } else { - StoreWord(ctx, inst, component_mem); + StoreWordPrepared(ctx, inst, component_mem, resource, data); } } +void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { + EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); + FormattedStorePrepared(ctx, inst, mem, 0u, resource, ctx.Arg(inst, inst.NumArgs() - 2)); + }); +} + uint32_t AtomicOpcode(IR::ValueOpcode opcode) { switch (opcode) { case IR::ValueOpcode::BufferAtomicSwap32: @@ -573,11 +690,192 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo return result; } +struct PreparedFormattedMemory { + Prospero::BufferFormat format = Prospero::BufferFormat::kInvalid; + Format::BufferFormatInfo info; + MemoryResourceAccess resource; + std::array addresses {}; + std::array indices {}; + uint32_t in_bounds = 0; +}; + +enum class FormattedAccess { Load, Store }; + +PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::Inst& inst, + const IR::MemoryInfo& mem, + const MemoryResourceAccess& resource, + uint32_t components, FormattedAccess access) { + PreparedFormattedMemory plan; + plan.format = BufferFormat(ctx, inst, mem); + plan.info = Format::GetFormatInfo(plan.format); + plan.resource = resource; + std::array required_components {}; + if (access == FormattedAccess::Load) { + for (uint32_t output = 0; output < components; output++) { + const auto source = ResolveFormattedSource(ctx, mem, plan.info, output); + if (source.kind == FormattedSourceKind::Memory) { + required_components[source.component] = true; + } + } + } else { + for (uint32_t component = 0; + component < std::min(components, plan.info.component_count); component++) { + required_components[component] = true; + } + } + bool first_bound = true; + for (uint32_t component = 0; component < plan.info.component_count; component++) { + if (!required_components[component]) continue; + const auto byte_offset = Format::GetFormatComponentByteOffset(plan.format, component); + bool reused = false; + for (uint32_t previous = 0; previous < component; previous++) { + if (required_components[previous] && + Format::GetFormatComponentByteOffset(plan.format, previous) == byte_offset) { + plan.addresses[component] = plan.addresses[previous]; + plan.indices[component] = plan.indices[previous]; + reused = true; + break; + } + } + if (reused) continue; + const auto component_mem = RebaseFormattedComponent(mem, plan.format, component); + plan.addresses[component] = ByteAddress(ctx, inst, component_mem); + const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), + plan.addresses[component], ConstantU32(ctx.state, 2)); + plan.indices[component] = EmitMemoryElementIndex(ctx.state, resource, raw_index); + const auto component_bound = + EmitMemoryElementInBounds(ctx.state, resource, plan.indices[component]); + if (first_bound) { + plan.in_bounds = component_bound; + first_bound = false; + } else { + plan.in_bounds = AndCondition(ctx.state, plan.in_bounds, component_bound); + } + } + if (first_bound) plan.in_bounds = ConstantBool(ctx.state, true); + return plan; +} + +uint32_t LoadFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, + const PreparedFormattedMemory& plan, uint32_t output_component) { + return LoadFormattedComponent( + ctx, mem, plan.format, plan.info, output_component, + [&](uint32_t component) { + return LoadWordInBounds(ctx, plan.resource, plan.indices[component]); + }, + [&](uint32_t component, uint32_t bits, bool sign_extend) { + return LoadSubwordInBounds(ctx, plan.resource, plan.addresses[component], + plan.indices[component], bits, sign_extend); + }); +} + +uint32_t ConstructU32Composite(EmitterState& state, uint32_t components, + const std::array& values) { + const auto result = state.builder.AllocateId(); + std::vector words {OpCompositeConstruct, TypeU32Composite(state, components), result}; + words.insert(words.end(), values.begin(), values.begin() + components); + state.builder.AddFunction(words); + return result; +} + +uint32_t FormattedOutOfBoundsValue(ValueEmitContext& ctx, const IR::MemoryInfo& mem, + const PreparedFormattedMemory& plan, uint32_t components) { + std::array values {}; + for (uint32_t component = 0; component < components; component++) { + const auto source = ResolveFormattedSource(ctx, mem, plan.info, component); + values[component] = FormattedConstant(ctx, plan.info, source.kind); + } + return ConstructU32Composite(ctx.state, components, values); +} + +void StoreFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, + const PreparedFormattedMemory& plan, uint32_t component, + uint32_t data) { + if (component >= plan.info.component_count) return; + const auto bits = plan.info.component_bits[component]; + if (bits == 8u || bits == 16u) { + StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses[component], + plan.indices[component], bits, data); + } else { + StoreWordInBounds(ctx, plan.resource, plan.indices[component], data); + } +} + +uint32_t LoadWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) { + auto& state = ctx.state; + return EmitValueOrDefaultIfCondition( + state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU32Composite(state, components), + ConstantU32CompositeZero(state, components), [&]() { + const auto mem = ctx.Memory(inst); + const auto resource = PrepareMemoryResourceAccess(state, mem); + const auto format = + mem.formatted ? BufferFormat(ctx, inst, mem) : Prospero::BufferFormat::kInvalid; + if (Format::IsKnownFormat(format)) { + const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, components, + FormattedAccess::Load); + return EmitValueOrDefaultIfCondition( + state, plan.in_bounds, TypeU32Composite(state, components), + FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() { + std::array values {}; + for (uint32_t component = 0; component < components; component++) { + values[component] = LoadFormattedInBounds(ctx, mem, plan, component); + } + return ConstructU32Composite(state, components, values); + }); + } + std::array values {}; + for (uint32_t component = 0; component < components; component++) { + values[component] = + LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource); + } + return ConstructU32Composite(state, components, values); + }); +} + +void StoreWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) { + auto& state = ctx.state; + EmitIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto mem = ctx.Memory(inst); + const auto resource = PrepareMemoryResourceAccess(state, mem); + const auto composite = ctx.Arg(inst, inst.NumArgs() - 2); + const auto format = + mem.formatted ? BufferFormat(ctx, inst, mem) : Prospero::BufferFormat::kInvalid; + if (Format::IsKnownFormat(format)) { + const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, components, + FormattedAccess::Store); + EmitIfCondition(state, plan.in_bounds, [&]() { + for (uint32_t component = 0; component < components; component++) { + const auto data = state.builder.AllocateId(); + state.builder.AddFunction( + {OpCompositeExtract, TypeU32(state), data, composite, component}); + StoreFormattedInBounds(ctx, mem, plan, component, data); + } + }); + return; + } + for (uint32_t component = 0; component < components; component++) { + const auto data = state.builder.AllocateId(); + state.builder.AddFunction( + {OpCompositeExtract, TypeU32(state), data, composite, component}); + StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data); + } + }); +} + } // namespace bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { - auto& state = ctx.state; - const auto op = inst.GetOpcode(); + auto& state = ctx.state; + const auto op = inst.GetOpcode(); + const auto buffer_components = IR::BufferComponentCount(op); + if (buffer_components > 1u) { + if (IR::BufferAccessOf(op) == IR::BufferAccess::Read) { + ctx.Define(inst, LoadWideBuffer(ctx, inst, buffer_components)); + } else { + StoreWideBuffer(ctx, inst, buffer_components); + } + return true; + } if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) && ctx.Memory(inst).planning_only) { return true; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index f7ed75f69..3f5b0dc89 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -42,6 +42,11 @@ uint32_t TypeU32Vector(EmitterState& state, uint32_t components) { return state.builder.Type(OpTypeVector, {TypeU32(state), components}); } +uint32_t TypeU32Composite(EmitterState& state, uint32_t components) { + EXIT_IF(components < 2u || components > 4u); + return components == 2u ? TypeU32Pair(state) : TypeU32Vector(state, components); +} + uint32_t TypeI32Vector(EmitterState& state, uint32_t components) { return state.builder.Type(OpTypeVector, {TypeI32(state), components}); } @@ -241,10 +246,11 @@ uint32_t ConstantU64(EmitterState& state, uint64_t value) { ConstantU32(state, static_cast(value >> 32u))}); } -uint32_t ConstantU32x4Zero(EmitterState& state) { - const auto zero = ConstantU32(state, 0); - return state.builder.Constant(OpConstantComposite, TypeU32Vector(state, 4), - {zero, zero, zero, zero}); +uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components) { + EXIT_IF(components < 2u || components > 4u); + const auto zero = ConstantU32(state, 0); + std::vector values(components, zero); + return state.builder.Constant(OpConstantComposite, TypeU32Composite(state, components), values); } uint32_t GlslStd450(EmitterState& state) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index a345ea67d..fbb0aedd2 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -46,6 +46,8 @@ uint32_t SpillPointerType(EmitterState& state, IR::Type type) { case IR::Type::F32: return TypePointer(state, StorageClassFunction, TypeF32(state)); case IR::Type::U64: case IR::Type::U32x2: return TypePointer(state, StorageClassFunction, TypeU32Pair(state)); + case IR::Type::U32x3: + return TypePointer(state, StorageClassFunction, TypeU32Vector(state, 3)); case IR::Type::U32x4: return TypePointer(state, StorageClassFunction, TypeU32Vector(state, 4)); case IR::Type::F32x2: @@ -405,6 +407,7 @@ uint32_t ValueEmitContext::TypeId(IR::Type type) const { case IR::Type::U64: case IR::Type::U32x2: return TypeU32Pair(state); case IR::Type::F32: return TypeF32(state); + case IR::Type::U32x3: return TypeU32Vector(state, 3); case IR::Type::U32x4: return TypeU32Vector(state, 4); case IR::Type::F32x2: return TypeF32Vector(state, 2); default: return 0; diff --git a/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp b/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp index 19779d564..e3a2e96ac 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp @@ -13,9 +13,12 @@ IR::ExportFlags Translator::AddExportInfo(const IR::Instruction& inst) { bool Translator::TranslateAttributeOperation(const IR::Instruction& inst) { if (inst.op == IR::Opcode::LoadInputF32) { - WriteOperand(inst.dst, - ir.Emit(IR::ValueOpcode::GetAttribute, - {IR::Value(inst.input_info.attr), IR::Value(inst.input_info.chan)})); + for (uint32_t component = 0; component < inst.input_info.component_count; component++) { + WriteOperand(OffsetOperand(inst.dst, component), + ir.Emit(IR::ValueOpcode::GetAttribute, + {IR::Value(inst.input_info.attr), + IR::Value(inst.input_info.chan + component)})); + } return true; } if (inst.op == IR::Opcode::Export) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp index e47961890..2b476bca9 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp @@ -12,7 +12,6 @@ bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) { } bool Translator::TranslateInstruction(const IR::Instruction& inst, - const BufferAddressValues* address_snapshot, const ScalarMemorySourceValues* scalar_source_snapshot) { switch (IR::GetOpcodeInfo(inst.op).lowering_class) { case IR::LoweringClass::Control: return TranslateControlOperation(inst); @@ -20,7 +19,7 @@ bool Translator::TranslateInstruction(const IR::Instruction& inst, case IR::LoweringClass::Lane: return TranslateLaneOperation(inst); case IR::LoweringClass::State: return TranslateStateOperation(inst); case IR::LoweringClass::Memory: - return TranslateMemoryOperation(inst, address_snapshot, scalar_source_snapshot); + return TranslateMemoryOperation(inst, scalar_source_snapshot); case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst); case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst); case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst); diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index de266410a..b782525b0 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -125,8 +125,8 @@ IR::ValueOpcode Translator::ImageAtomicOpcode(IR::Opcode opcode) { } } -BufferAddressValues Translator::ReadBufferAddress(const IR::Instruction& inst, - uint32_t first_source) { +Translator::BufferAddress Translator::ReadBufferAddress(const IR::Instruction& inst, + uint32_t first_source) { uint32_t cursor = first_source; const auto next = [&]() { return cursor < inst.src_count ? ReadU32(inst.src[cursor++]) : IR::U32(IR::Value(0u)); @@ -201,12 +201,6 @@ IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) { } } -bool Translator::IsBufferLoadOperation(IR::Opcode opcode) { - return opcode == IR::Opcode::BufferLoadUbyte || opcode == IR::Opcode::BufferLoadSbyte || - opcode == IR::Opcode::BufferLoadUshort || opcode == IR::Opcode::BufferLoadSshort || - opcode == IR::Opcode::BufferLoadDword; -} - bool Translator::IsScalarMemoryLoadOperation(IR::Opcode opcode) { return opcode == IR::Opcode::SLoadDword || opcode == IR::Opcode::SBufferLoadDword; } @@ -247,8 +241,7 @@ bool Translator::TranslateScalarMemory(const IR::Instruction& inst, } } -bool Translator::TranslateBufferLoad(const IR::Instruction& inst, - const BufferAddressValues* address_snapshot) { +bool Translator::TranslateBufferLoad(const IR::Instruction& inst) { IR::ValueOpcode opcode; uint32_t bits; bool sign; @@ -266,19 +259,33 @@ bool Translator::TranslateBufferLoad(const IR::Instruction& inst, sign = inst.op == IR::Opcode::BufferLoadSshort; break; case IR::Opcode::BufferLoadDword: - opcode = IR::ValueOpcode::LoadBufferU32; - bits = 32u; - sign = false; + switch (inst.memory.data_dwords) { + case 1u: opcode = IR::ValueOpcode::LoadBufferU32; break; + case 2u: opcode = IR::ValueOpcode::LoadBufferU32x2; break; + case 3u: opcode = IR::ValueOpcode::LoadBufferU32x3; break; + case 4u: opcode = IR::ValueOpcode::LoadBufferU32x4; break; + default: return false; + } + bits = 32u; + sign = false; break; default: return false; } const auto resource = GetBufferResource(inst.memory); - const auto address = - address_snapshot != nullptr ? *address_snapshot : ReadBufferAddress(inst, 0); + const auto address = ReadBufferAddress(inst, 0); const auto loaded = ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, ir.GetExec()}, AddMemoryInfo(inst)); - WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign)); + if (bits != 32u) { + WriteOperand(inst.dst, WidenSubdword(loaded, bits, sign)); + } else if (inst.memory.data_dwords == 1u) { + WriteOperand(inst.dst, loaded); + } else { + for (uint32_t component = 0; component < inst.memory.data_dwords; component++) { + WriteOperand(OffsetOperand(inst.dst, component), + ir.CompositeExtract(loaded, component)); + } + } return true; } @@ -298,8 +305,31 @@ bool Translator::TranslateBufferStore(const IR::Instruction& inst) { value = NarrowSubdword(data, 16u); break; case IR::Opcode::BufferStoreDword: - opcode = IR::ValueOpcode::StoreBufferU32; - value = data; + switch (inst.memory.data_dwords) { + case 1u: + opcode = IR::ValueOpcode::StoreBufferU32; + value = data; + break; + case 2u: + opcode = IR::ValueOpcode::StoreBufferU32x2; + value = ir.Emit(IR::ValueOpcode::CompositeConstructU32x2, + {data, ReadU32(OffsetOperand(inst.src[0], 1u))}); + break; + case 3u: + opcode = IR::ValueOpcode::StoreBufferU32x3; + value = ir.Emit(IR::ValueOpcode::CompositeConstructU32x3, + {data, ReadU32(OffsetOperand(inst.src[0], 1u)), + ReadU32(OffsetOperand(inst.src[0], 2u))}); + break; + case 4u: + opcode = IR::ValueOpcode::StoreBufferU32x4; + value = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4, + {data, ReadU32(OffsetOperand(inst.src[0], 1u)), + ReadU32(OffsetOperand(inst.src[0], 2u)), + ReadU32(OffsetOperand(inst.src[0], 3u))}); + break; + default: return false; + } break; default: return false; } @@ -594,7 +624,6 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) { } bool Translator::TranslateMemoryOperation(const IR::Instruction& inst, - const BufferAddressValues* address_snapshot, const ScalarMemorySourceValues* scalar_source_snapshot) { switch (inst.op) { case IR::Opcode::LoadSrtDword: @@ -606,7 +635,7 @@ bool Translator::TranslateMemoryOperation(const IR::Instruction& inst, case IR::Opcode::BufferLoadSbyte: case IR::Opcode::BufferLoadUshort: case IR::Opcode::BufferLoadSshort: - case IR::Opcode::BufferLoadDword: return TranslateBufferLoad(inst, address_snapshot); + case IR::Opcode::BufferLoadDword: return TranslateBufferLoad(inst); case IR::Opcode::BufferStoreByte: case IR::Opcode::BufferStoreShort: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 89afd4b97..22d9b13fb 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -668,26 +668,12 @@ void Translator::WriteCompareResult(const IR::Operand& operand, IR::U1 value) { } bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error) { - BufferAddressValues buffer_address_snapshot {}; ScalarMemorySourceValues scalar_source_snapshot {}; - uint32_t buffer_address_pc = 0; - uint32_t scalar_source_pc = 0; - bool has_buffer_address_snapshot = false; - bool has_scalar_source_snapshot = false; + uint32_t scalar_source_pc = 0; + bool has_scalar_source_snapshot = false; for (const auto& source_inst: source.instructions) { current_opcode = source_inst.op; current_pc = source_inst.pc; - const bool grouped_buffer_load = - IsBufferLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u; - if (grouped_buffer_load && - (!has_buffer_address_snapshot || buffer_address_pc != source_inst.pc || - source_inst.memory.component_index == 0u)) { - buffer_address_snapshot = ReadBufferAddress(source_inst, 0); - buffer_address_pc = source_inst.pc; - has_buffer_address_snapshot = true; - } else if (!grouped_buffer_load) { - has_buffer_address_snapshot = false; - } const bool grouped_scalar_load = IsScalarMemoryLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u; if (grouped_scalar_load && @@ -703,7 +689,6 @@ bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error return Fail(error, "value IR input opcode is out of range"); } if (TranslateInstruction(source_inst, - grouped_buffer_load ? &buffer_address_snapshot : nullptr, grouped_scalar_load ? &scalar_source_snapshot : nullptr)) { continue; } @@ -780,11 +765,11 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, const uint32_t wave_size = source.wave_size; const bool per_invocation_masks = source.lane_mask_mode == ShaderLaneMaskMode::PerInvocation; uint32_t vector_limit = 1u; - const auto include_vector = [&](const IR::Operand& operand) { + const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) { if (operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Vector) { vector_limit = - std::min(IR::NumVectorRegs, std::max(vector_limit, operand.reg.index + 1u)); + std::min(IR::NumVectorRegs, std::max(vector_limit, operand.reg.index + count)); } }; for (const auto& block: source.blocks) { @@ -794,6 +779,13 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, for (uint32_t index = 0; index < inst.src_count; index++) { include_vector(inst.src[index]); } + if (inst.op == IR::Opcode::BufferLoadDword) { + include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u)); + } else if (inst.op == IR::Opcode::BufferStoreDword) { + include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u)); + } else if (inst.op == IR::Opcode::LoadInputF32) { + include_vector(inst.dst, std::max(inst.input_info.component_count, 1u)); + } } } result.block_storage.reserve(source.blocks.size() + 1u); @@ -846,8 +838,8 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, } entry_ir.SetExec(IR::U1(IR::Value(true))); entry_ir.SetExecLo(IR::U32(IR::Value(per_invocation_masks ? 1u : 0xffffffffu))); - entry_ir.SetExecHi(IR::U32(IR::Value( - !per_invocation_masks && source.wave_size > 32u ? 0xffffffffu : 0u))); + entry_ir.SetExecHi( + IR::U32(IR::Value(!per_invocation_masks && source.wave_size > 32u ? 0xffffffffu : 0u))); if (source.stage == ShaderType::Compute) { const auto* cs = compute_input_info; const auto thread_ids = diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 35c44935f..0c9651d81 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -7,12 +7,6 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { -struct BufferAddressValues { - IR::U32 index; - IR::U32 offset; - IR::U32 soffset; -}; - struct ScalarMemorySourceValues { IR::Value resource; IR::U32 offset; @@ -70,6 +64,11 @@ private: IR::Value low; IR::Value high; }; + struct BufferAddress { + IR::U32 index; + IR::U32 offset; + IR::U32 soffset; + }; AddressOperands ReadAddressOperands(const IR::Instruction& inst, uint32_t first_source); IR::U32 GetResourceDword(uint32_t index, uint32_t dword); IR::Value GetBufferResource(const IR::MemoryInfo& memory); @@ -83,18 +82,16 @@ private: void WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask, uint32_t component_limit); IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode); - BufferAddressValues ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset); - IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign); - IR::Value NarrowSubdword(IR::U32 value, uint32_t bits); - IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode); - IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode, bool gds); - bool IsBufferLoadOperation(IR::Opcode opcode); - bool IsScalarMemoryLoadOperation(IR::Opcode opcode); + BufferAddress ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset); + IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign); + IR::Value NarrowSubdword(IR::U32 value, uint32_t bits); + IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode); + IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode, bool gds); + bool IsScalarMemoryLoadOperation(IR::Opcode opcode); ScalarMemorySourceValues ReadScalarMemorySource(const IR::Instruction& inst); bool TranslateScalarMemory(const IR::Instruction& inst, const ScalarMemorySourceValues* source_snapshot); - bool TranslateBufferLoad(const IR::Instruction& inst, - const BufferAddressValues* address_snapshot); + bool TranslateBufferLoad(const IR::Instruction& inst); bool TranslateBufferStore(const IR::Instruction& inst); bool TranslateAtomicMemory(const IR::Instruction& inst); bool TranslateFlatLoad(const IR::Instruction& inst); @@ -111,7 +108,6 @@ private: IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high); bool TranslateInstruction(const IR::Instruction& inst, - const BufferAddressValues* address_snapshot, const ScalarMemorySourceValues* scalar_source_snapshot); bool TranslateStateOperation(const IR::Instruction& inst); bool TranslateControlOperation(const IR::Instruction& inst); @@ -119,7 +115,6 @@ private: bool TranslateLaneOperation(const IR::Instruction& inst); bool TranslateAttributeOperation(const IR::Instruction& inst); bool TranslateMemoryOperation(const IR::Instruction& inst, - const BufferAddressValues* address_snapshot, const ScalarMemorySourceValues* scalar_source_snapshot); bool TranslateIntegerCompare(const IR::Instruction& inst); bool TranslateInteger16Compare(const IR::Instruction& inst); diff --git a/src/graphics/shader/recompiler/ir/IREmitter.cpp b/src/graphics/shader/recompiler/ir/IREmitter.cpp index 8e53c4bc4..73bc277b2 100644 --- a/src/graphics/shader/recompiler/ir/IREmitter.cpp +++ b/src/graphics/shader/recompiler/ir/IREmitter.cpp @@ -140,7 +140,14 @@ U64 IREmitter::PackUint2x32(U32 low, U32 high) { } U32 IREmitter::CompositeExtract(Value composite, uint32_t index) { - return U32(Emit(ValueOpcode::CompositeExtractU32x2, {composite, Value(index)})); + ValueOpcode opcode; + switch (composite.GetType()) { + case Type::U32x2: opcode = ValueOpcode::CompositeExtractU32x2; break; + case Type::U32x3: opcode = ValueOpcode::CompositeExtractU32x3; break; + case Type::U32x4: opcode = ValueOpcode::CompositeExtractU32x4; break; + default: EXIT("invalid U32 composite type\n"); + } + return U32(Emit(opcode, {composite, Value(index)})); } U32 IREmitter::IAdd(U32 lhs, U32 rhs) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.cpp b/src/graphics/shader/recompiler/ir/ShaderIR.cpp index 789654bb8..6c15fce5c 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIR.cpp @@ -328,54 +328,6 @@ MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind ki return mem; } -uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded, - uint32_t component_index) { - if (!decoded.typed || !decoded.formatted) { - return component_index * 4u; - } - - const auto format = Format::DecodeTBufferFormat(decoded.data_format, decoded.number_format); - if (!Format::IsKnownFormat(format)) { - return component_index * 4u; - } - return Format::GetFormatComponentByteOffset(format, component_index); -} - -uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded) { - if (!decoded.typed || !decoded.formatted) { - return decoded.data_dwords; - } - - const auto format = Format::DecodeTBufferFormat(decoded.data_format, decoded.number_format); - if (!Format::IsKnownFormat(format)) { - return decoded.data_dwords; - } - return Format::GetFormatComponentCount(format); -} - -bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value, - BasicBlock& block, std::string* error) { - Instruction inst; - inst.pc = pc; - inst.op = Opcode::MoveU32; - inst.src[0] = MakeImmediateU32(value); - inst.src_count = 1; - if (!LowerRegisterOperand(dst, inst.dst, error)) { - return false; - } - block.instructions.push_back(inst); - return true; -} - -MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, uint32_t component_index) { - auto mem = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer); - mem.offset += TypedBufferComponentOffsetBytes(decoded, component_index); - mem.data_dwords = 1; - mem.component_index = component_index; - mem.component_count = decoded.data_dwords; - return mem; -} - MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, uint32_t byte_offset) { auto mem = MemoryInfoFromDecoded(decoded, kind); diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 1b34237d1..4c660ea62 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -123,8 +123,9 @@ struct ExportInfo { }; struct InputInfo { - uint32_t attr = 0; - uint32_t chan = 0; + uint32_t attr = 0; + uint32_t chan = 0; + uint32_t component_count = 1; bool operator==(const InputInfo& other) const = default; }; diff --git a/src/graphics/shader/recompiler/ir/ShaderIRInternal.h b/src/graphics/shader/recompiler/ir/ShaderIRInternal.h index 0bb0cb69a..9229ec0d5 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIRInternal.h +++ b/src/graphics/shader/recompiler/ir/ShaderIRInternal.h @@ -36,15 +36,9 @@ bool TryOffsetScalarDestination(const Decoder::Operand& operand, uin Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index); MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, uint32_t dword_index); -uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded, - uint32_t component_index); -uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded); -bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value, - BasicBlock& block, std::string* error); -MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, uint32_t component_index); -MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, - uint32_t byte_offset); -ResourceKind FlatSegmentResourceKind(uint32_t segment); +MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind, + uint32_t byte_offset); +ResourceKind FlatSegmentResourceKind(uint32_t segment); const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index); Decoder::Operand M0Operand(); diff --git a/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp b/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp index 573d6b172..6e29636aa 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp @@ -130,7 +130,8 @@ std::string InputInfoToString(Opcode op, const InputInfo& input) { if (op != Opcode::LoadInputF32) { return ""; } - return fmt::format(" ; input_attr={} input_chan={}", input.attr, input.chan); + return fmt::format(" ; input_attr={} input_chan={} input_components={}", input.attr, input.chan, + input.component_count); } std::string TerminatorToString(const CFG::Terminator& term) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp b/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp index 6fc445f52..0c5997c57 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp @@ -121,52 +121,29 @@ bool LowerBufferAddressSources(const Decoder::Instruction& decoded, Instruction& } bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - const auto ir_op = BufferLoadOpcode(decoded); - const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; - const auto typed_format_components = TypedBufferFormatComponentCount(decoded); - for (uint32_t i = 0; i < count; i++) { - if (i >= typed_format_components) { - if (!LowerMoveImmediateU32(decoded.pc, OffsetDecodedRegister(decoded.dst, i), 0, block, - error)) { - return false; - } - continue; - } - - Instruction inst; - inst.pc = decoded.pc; - inst.op = ir_op; - inst.memory = OffsetBufferMemoryInfo(decoded, i); - if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) || - !LowerBufferAddressSources(decoded, inst, 0, error)) { - return false; - } - block.instructions.push_back(inst); + Instruction inst; + inst.pc = decoded.pc; + inst.op = BufferLoadOpcode(decoded); + inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer); + if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || + !LowerBufferAddressSources(decoded, inst, 0, error)) { + return false; } + block.instructions.push_back(inst); return true; } bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - const auto ir_op = BufferStoreOpcode(decoded.data_bits); - auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; - if (decoded.data_bits == 32u) { - const auto typed_format_components = TypedBufferFormatComponentCount(decoded); - if (typed_format_components < count) { - count = typed_format_components; - } - } - for (uint32_t i = 0; i < count; i++) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = ir_op; - inst.memory = OffsetBufferMemoryInfo(decoded, i); - inst.dst.kind = OperandKind::Null; - if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, i), inst.src[0], error) || - !LowerBufferAddressSources(decoded, inst, 1, error)) { - return false; - } - block.instructions.push_back(inst); + Instruction inst; + inst.pc = decoded.pc; + inst.op = BufferStoreOpcode(decoded.data_bits); + inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer); + inst.dst.kind = OperandKind::Null; + if (!LowerSourceOperand(decoded.dst, inst.src[0], error) || + !LowerBufferAddressSources(decoded, inst, 1, error)) { + return false; } + block.instructions.push_back(inst); return true; } diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.cpp b/src/graphics/shader/recompiler/ir/ValueProgram.cpp index 9f9bea01d..8264f0739 100644 --- a/src/graphics/shader/recompiler/ir/ValueProgram.cpp +++ b/src/graphics/shader/recompiler/ir/ValueProgram.cpp @@ -304,6 +304,46 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st return Fail(error, fmt::format("untyped opcode {} survived translation", ValueOpcodeName(inst.GetOpcode()))); } + const auto buffer_components = BufferComponentCount(inst.GetOpcode()); + if (buffer_components != 0u) { + const auto memory_index = inst.Flags().index; + if (memory_index >= program.memory_info.size()) { + return Fail(error, fmt::format("{} has an invalid memory-info index", + ValueOpcodeName(inst.GetOpcode()))); + } + const auto& memory = program.memory_info[memory_index]; + if (memory.kind != ResourceKind::Buffer && + memory.kind != ResourceKind::ScalarBuffer) { + return Fail(error, fmt::format("{} has a non-buffer resource kind", + ValueOpcodeName(inst.GetOpcode()))); + } + if (buffer_components > 1u && + (memory.kind != ResourceKind::Buffer || memory.data_bits != 32u || + memory.data_dwords != buffer_components || memory.component_index != 0u)) { + return Fail(error, fmt::format("{} has inconsistent native-wide metadata", + ValueOpcodeName(inst.GetOpcode()))); + } + if (buffer_components == 1u && + (inst.GetOpcode() == ValueOpcode::LoadBufferU32 || + inst.GetOpcode() == ValueOpcode::StoreBufferU32) && + memory.data_dwords != 1u) { + return Fail(error, fmt::format("{} retains scalar-sibling width metadata", + ValueOpcodeName(inst.GetOpcode()))); + } + } + uint32_t composite_components = 0u; + switch (inst.GetOpcode()) { + case ValueOpcode::CompositeExtractU32x2: composite_components = 2u; break; + case ValueOpcode::CompositeExtractU32x3: composite_components = 3u; break; + case ValueOpcode::CompositeExtractU32x4: composite_components = 4u; break; + default: break; + } + if (composite_components != 0u && + (!inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32 || + inst.Arg(1).U32() >= composite_components)) { + return Fail(error, fmt::format("{} has an invalid component index", + ValueOpcodeName(inst.GetOpcode()))); + } for (size_t arg_index = 0; arg_index < inst.NumArgs(); arg_index++) { const auto arg = inst.Arg(arg_index); if (arg.IsEmpty()) { diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index ee9656ee8..05ec0e5e7 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -34,6 +34,7 @@ constexpr Type U64 = Type::U64; constexpr Type F16 = Type::F16; constexpr Type F32 = Type::F32; constexpr Type U32x2 = Type::U32x2; +constexpr Type U32x3 = Type::U32x3; constexpr Type U32x4 = Type::U32x4; constexpr Type F32x2 = Type::F32x2; constexpr Type SrtResource = Type::SrtResource; @@ -133,10 +134,16 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) { case ValueOpcode::ReadConstBuffer: case ValueOpcode::LoadBufferU8: case ValueOpcode::LoadBufferU16: - case ValueOpcode::LoadBufferU32: return BufferAccess::Read; + case ValueOpcode::LoadBufferU32: + case ValueOpcode::LoadBufferU32x2: + case ValueOpcode::LoadBufferU32x3: + case ValueOpcode::LoadBufferU32x4: return BufferAccess::Read; case ValueOpcode::StoreBufferU8: case ValueOpcode::StoreBufferU16: - case ValueOpcode::StoreBufferU32: return BufferAccess::Write; + case ValueOpcode::StoreBufferU32: + case ValueOpcode::StoreBufferU32x2: + case ValueOpcode::StoreBufferU32x3: + case ValueOpcode::StoreBufferU32x4: return BufferAccess::Write; case ValueOpcode::BufferAtomicSwap32: case ValueOpcode::BufferAtomicIAdd32: case ValueOpcode::BufferAtomicISub32: @@ -153,4 +160,16 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) { } } +uint32_t BufferComponentCount(ValueOpcode opcode) { + switch (opcode) { + case ValueOpcode::LoadBufferU32x2: + case ValueOpcode::StoreBufferU32x2: return 2u; + case ValueOpcode::LoadBufferU32x3: + case ValueOpcode::StoreBufferU32x3: return 3u; + case ValueOpcode::LoadBufferU32x4: + case ValueOpcode::StoreBufferU32x4: return 4u; + default: return BufferAccessOf(opcode) == BufferAccess::None ? 0u : 1u; + } +} + } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h index c2040fcc4..135afa50d 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h @@ -55,6 +55,7 @@ static_assert(std::is_trivially_copyable_v); [[nodiscard]] size_t NumArgsOf(ValueOpcode opcode); [[nodiscard]] bool HasSideEffects(ValueOpcode opcode); [[nodiscard]] BufferAccess BufferAccessOf(ValueOpcode opcode); +[[nodiscard]] uint32_t BufferComponentCount(ValueOpcode opcode); [[nodiscard]] std::string_view ValueOpcodeName(ValueOpcode opcode); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 73d851bef..4928f9c4c 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -57,9 +57,11 @@ VALUE_OPCODE(ConvertU32F32, U32, F32) VALUE_OPCODE(ConvertF32S32, F32, U32) VALUE_OPCODE(ConvertF32U32, F32, U32) VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32) +VALUE_OPCODE(CompositeConstructU32x3, U32x3, U32, U32, U32) VALUE_OPCODE(CompositeConstructF32x2, F32x2, F32, F32) VALUE_OPCODE(CompositeConstructU32x4, U32x4, U32, U32, U32, U32) VALUE_OPCODE(CompositeExtractU32x2, U32, U32x2, U32) +VALUE_OPCODE(CompositeExtractU32x3, U32, U32x3, U32) VALUE_OPCODE(CompositeExtractU32x4, U32, U32x4, U32) VALUE_OPCODE(PackUint2x32, U64, U32x2) VALUE_OPCODE(UnpackUint2x32, U32x2, U64) @@ -215,9 +217,15 @@ VALUE_OPCODE(StoreAddressU32, Void, AddressResource, U32, U32, U32, U1) VALUE_OPCODE(LoadBufferU8, U8, BufferResource, U32, U32, U32, U1) VALUE_OPCODE(LoadBufferU16, U16, BufferResource, U32, U32, U32, U1) VALUE_OPCODE(LoadBufferU32, U32, BufferResource, U32, U32, U32, U1) +VALUE_OPCODE(LoadBufferU32x2, U32x2, BufferResource, U32, U32, U32, U1) +VALUE_OPCODE(LoadBufferU32x3, U32x3, BufferResource, U32, U32, U32, U1) +VALUE_OPCODE(LoadBufferU32x4, U32x4, BufferResource, U32, U32, U32, U1) VALUE_OPCODE(StoreBufferU8, Void, BufferResource, U32, U32, U32, U8, U1) VALUE_OPCODE(StoreBufferU16, Void, BufferResource, U32, U32, U32, U16, U1) VALUE_OPCODE(StoreBufferU32, Void, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(StoreBufferU32x2, Void, BufferResource, U32, U32, U32, U32x2, U1) +VALUE_OPCODE(StoreBufferU32x3, Void, BufferResource, U32, U32, U32, U32x3, U1) +VALUE_OPCODE(StoreBufferU32x4, Void, BufferResource, U32, U32, U32, U32x4, U1) VALUE_OPCODE(BufferAtomicSwap32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicIAdd32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1) diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index a1cb4c3d0..f9a0b78eb 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -287,6 +287,9 @@ void FoldInstruction(Inst& inst) { case ValueOpcode::CompositeExtractU32x2: FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x2, 2); return; + case ValueOpcode::CompositeExtractU32x3: + FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x3, 3); + return; case ValueOpcode::CompositeExtractU32x4: FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x4, 4); return; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 9f2670924..304849ea7 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -14910,6 +14910,202 @@ TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() { return test; } +TestCase BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 8); + AppendBufferLoadOpcode(&code, 0x0e, 0, 20); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + TestCase test; + test.name = "BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail"; + test.code = std::move(code); + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.expected = {0x33333333u, 0x44444444u, 0u, 0u}; + test.storage_buffer_range_dwords = 4; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + +TestCase BufferStoreDwordx4DropsOnlyOutOfBoundsTail() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0xaaaaaaaau); + AppendVMovLiteral(&code, 1, 0xbbbbbbbbu); + AppendVMovLiteral(&code, 2, 0xccccccccu); + AppendVMovLiteral(&code, 3, 0xddddddddu); + AppendVMovU32(&code, 20, 8); + code.push_back(EncodeMubuf0(0x1eu)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreDwordx4DropsOnlyOutOfBoundsTail"; + test.code = std::move(code); + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.expected = {0x11111111u, 0x22222222u, 0xaaaaaaaau, 0xbbbbbbbbu}; + test.storage_buffer_range_dwords = 4; + test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORDX4, O::S_ENDPGM}; + return test; +} + +TestCase BufferLoadFormatXyzwRejectsPartialRecord() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 8); + AppendBufferLoadOpcode(&code, 0x03, 0, 20); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); + + TestCase test; + test.name = "BufferLoadFormatXyzwRejectsPartialRecord"; + test.code = std::move(code); + test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; + test.expected = {0u, 0u, 0u, 0u}; + test.storage_buffer_range_dwords = 4; + test.user_data = MakeStructuredStorageBufferData( + 0, 4, false, + BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + test.has_user_data = true; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + +TestCase BufferStoreFormatXyzwDropsPartialRecord() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0xaaaaaaaau); + AppendVMovLiteral(&code, 1, 0xbbbbbbbbu); + AppendVMovLiteral(&code, 2, 0xccccccccu); + AppendVMovLiteral(&code, 3, 0xddddddddu); + AppendVMovU32(&code, 20, 8); + code.push_back(EncodeMubuf0(0x07u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXyzwDropsPartialRecord"; + test.code = std::move(code); + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.expected = test.initial; + test.storage_buffer_range_dwords = 4; + test.user_data = MakeStructuredStorageBufferData( + 0, 4, false, + BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + test.has_user_data = true; + test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM}; + return test; +} + +TestCase BufferLoadFormatXChecksOnlyTransferredComponent() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 12); + AppendBufferLoadOpcode(&code, 0x00, 0, 20); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); + + TestCase test; + test.name = "BufferLoadFormatXChecksOnlyTransferredComponent"; + test.code = std::move(code); + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.expected = {0x44444444u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.storage_buffer_range_dwords = 4; + test.user_data = MakeStructuredStorageBufferData( + 0, 4, false, + BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + test.has_user_data = true; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_X, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + +TestCase BufferStoreFormatXChecksOnlyTransferredComponent() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0xaaaaaaaau); + AppendVMovU32(&code, 20, 12); + code.push_back(EncodeMubuf0(0x04u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXChecksOnlyTransferredComponent"; + test.code = std::move(code); + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0xaaaaaaaau}; + test.storage_buffer_range_dwords = 4; + test.user_data = MakeStructuredStorageBufferData( + 0, 4, false, + BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + test.has_user_data = true; + test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM}; + return test; +} + +TestCase BufferLoadFormatXyChecksOnlyTransferredComponents() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 20, 8); + AppendBufferLoadOpcode(&code, 0x01, 0, 20); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); + + TestCase test; + test.name = "BufferLoadFormatXyChecksOnlyTransferredComponents"; + test.code = std::move(code); + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.expected = {0x33333333u, 0x44444444u, 0x33333333u, 0x44444444u}; + test.storage_buffer_range_dwords = 4; + test.user_data = MakeStructuredStorageBufferData( + 0, 4, false, + BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + test.has_user_data = true; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + +TestCase BufferStoreFormatXyChecksOnlyTransferredComponents() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovLiteral(&code, 0, 0xaaaaaaaau); + AppendVMovLiteral(&code, 1, 0xbbbbbbbbu); + AppendVMovU32(&code, 20, 8); + code.push_back(EncodeMubuf0(0x05u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); + + TestCase test; + test.name = "BufferStoreFormatXyChecksOnlyTransferredComponents"; + test.code = std::move(code); + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.expected = {0x11111111u, 0x22222222u, 0xaaaaaaaau, 0xbbbbbbbbu}; + test.storage_buffer_range_dwords = 4; + test.user_data = MakeStructuredStorageBufferData( + 0, 4, false, + BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + test.has_user_data = true; + test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM}; + return test; +} + TestCase BufferStoreVariants() { using O = ShaderOpcode; @@ -18202,6 +18398,14 @@ std::vector MakeCases() { AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress); AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress); AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress); + AddCase(BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail); + AddCase(BufferStoreDwordx4DropsOnlyOutOfBoundsTail); + AddCase(BufferLoadFormatXyzwRejectsPartialRecord); + AddCase(BufferStoreFormatXyzwDropsPartialRecord); + AddCase(BufferLoadFormatXChecksOnlyTransferredComponent); + AddCase(BufferStoreFormatXChecksOnlyTransferredComponent); + AddCase(BufferLoadFormatXyChecksOnlyTransferredComponents); + AddCase(BufferStoreFormatXyChecksOnlyTransferredComponents); AddCase(BufferStoreVariants); AddCase(BufferFormatVariants); AddCase(BufferLoadFormatXyzwSnapshotsOverlappingAddress); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 806934161..492124925 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -5582,14 +5582,20 @@ void TestNewShaderRecompilerWideMemoryLowering() { "s_buffer_load_dwordx2 did not expand to VCC low dword"); Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_hi"), "s_buffer_load_dwordx2 did not expand to VCC high dword"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v20"), - "buffer_load_dwordx2 did not expand to dword IR loads"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v31"), - "buffer_load_dwordx4 did not expand to the last dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v40"), - "buffer_store_dwordx4 did not expand to first dword IR store"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v43"), - "buffer_store_dwordx4 did not expand to last dword IR store"); + Check( + CountSourceOccurrences(result.ir_dump, "BufferLoadDword v20") == 1u && + CountSourceOccurrences(result.ir_dump, "BufferLoadDword v24") == 1u && + CountSourceOccurrences(result.ir_dump, "BufferLoadDword v28") == 1u && + !Common::ContainsStr(result.ir_dump, "BufferLoadDword v31"), + "wide buffer loads were not preserved as one native IR operation"); + Check(CountSourceOccurrences(result.ir_dump, "BufferStoreDword null, v32") == + 1u && + CountSourceOccurrences(result.ir_dump, + "BufferStoreDword null, v36") == 1u && + CountSourceOccurrences(result.ir_dump, + "BufferStoreDword null, v40") == 1u && + !Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v43"), + "wide buffer stores were not preserved as one native IR operation"); Check(Common::ContainsStr(result.ir_dump, "BufferLoadUbyte v44"), "buffer_load_ubyte did not lower to sub-dword IR load"); Check(Common::ContainsStr(result.ir_dump, "BufferLoadUshort v45"), @@ -5615,6 +5621,64 @@ void TestNewShaderRecompilerWideMemoryLowering() { CheckSpirvBinaryValidates(result.spirv); } +void TestNewShaderRecompilerNativeWideBufferIr() { + const uint32_t shader[] = { + EncodeMubuf0(0x0d, 0), + EncodeMubuf1(0, 20, 1), // buffer_load_dwordx2 v[0:1] + EncodeMubuf0(0x1d, 16), + EncodeMubuf1(0, 20, 1), // buffer_store_dwordx2 v[0:1] + EncodeMubuf0(0x0f, 32), + EncodeMubuf1(4, 20, 1), // buffer_load_dwordx3 v[4:6] + EncodeMubuf0(0x1f, 48), + EncodeMubuf1(4, 20, 1), // buffer_store_dwordx3 v[4:6] + EncodeMubuf0(0x0e, 64), + EncodeMubuf1(8, 20, 1), // buffer_load_dwordx4 v[8:11] + EncodeMubuf0(0x1e, 80), + EncodeMubuf1(8, 20, 1), // buffer_store_dwordx4 v[8:11] + EncodeSopp(0x01), + }; + auto options = MakeCompileOptions(ShaderType::Compute); + options.dump_ir = true; + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + std::array counts{}; + for (const auto *block : result.program.values->blocks) { + for (const auto &inst : *block) { + switch (inst.GetOpcode()) { + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32: + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32: + counts[0]++; + break; + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32x2: + counts[1]++; + break; + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32x2: + counts[2]++; + break; + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32x3: + counts[3]++; + break; + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32x3: + counts[4]++; + break; + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32x4: + counts[5]++; + break; + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32x4: + counts[6]++; + break; + default: + break; + } + } + } + Check(counts == std::array{0u, 1u, 1u, 1u, 1u, 1u, 1u}, + "native-wide buffer lowering retained scalar siblings or lost a width"); + CheckSpirvBinaryValidates(result.spirv); +} + void TestNewShaderRecompilerBufferSignedLoadLowering() { const uint32_t shader[] = { EncodeMubuf0(0x09, 3), EncodeMubuf1(46, 0, 1), // buffer_load_sbyte @@ -5724,27 +5788,23 @@ void TestNewShaderRecompilerMubufFormatLowering() { std::string error; Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_format_x"), + Check(Common::ContainsStr(result.decoded_dump, "BUFFER_LOAD_FORMAT_X"), "new decoder did not decode MUBUF format-x load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_format_xyzw"), + Check(Common::ContainsStr(result.decoded_dump, "BUFFER_LOAD_FORMAT_XYZW"), "new decoder did not decode MUBUF format-xyzw load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_x"), + Check(Common::ContainsStr(result.decoded_dump, "BUFFER_STORE_FORMAT_X"), "new decoder did not decode MUBUF format-x store"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_xyzw"), + Check(Common::ContainsStr(result.decoded_dump, "BUFFER_STORE_FORMAT_XYZW"), "new decoder did not decode MUBUF format-xyzw store"); Check(Common::ContainsStr(result.decoded_dump, "typed=0 formatted=1"), "MUBUF format decode did not preserve formatted non-typed metadata"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v84"), - "MUBUF format-x load did not lower through shared buffer load IR"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v99"), - "MUBUF format-xyzw load did not expand to the last shared dword load"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v100"), - "MUBUF format-x store did not lower through shared buffer store IR"); - Check( - Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v115"), - "MUBUF format-xyzw store did not expand to the last shared dword store"); - Check(Common::ContainsStr(result.ir_dump, "typed=0 formatted=1"), - "MUBUF format metadata did not survive into IR"); + Check(!Common::ContainsStr(result.ir_dump, "BufferLoadDword v99"), + "MUBUF formatted load retained a scalar tail sibling"); + Check(CountSourceOccurrences(result.ir_dump, "StoreBufferU32 ") == 1u && + CountSourceOccurrences(result.ir_dump, "StoreBufferU32x2 ") == 1u && + CountSourceOccurrences(result.ir_dump, "StoreBufferU32x3 ") == 1u && + CountSourceOccurrences(result.ir_dump, "StoreBufferU32x4 ") == 1u, + "MUBUF formatted stores were not preserved as native-width operations"); Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); Check(SpirvContainsOpcode(result.spirv, 61), @@ -5817,19 +5877,19 @@ void TestNewShaderRecompilerTypedBufferLowering() { std::string error; Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_x"), + Check(Common::ContainsStr(result.decoded_dump, "TBUFFER_LOAD_FORMAT_X"), "new decoder did not decode typed buffer format-x load"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xy"), + Check(Common::ContainsStr(result.decoded_dump, "TBUFFER_LOAD_FORMAT_XY"), "new decoder did not decode typed buffer format-xy load"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xyz"), + Check(Common::ContainsStr(result.decoded_dump, "TBUFFER_LOAD_FORMAT_XYZ"), "new decoder did not decode typed buffer format-xyz load"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xyzw"), + Check(Common::ContainsStr(result.decoded_dump, "TBUFFER_LOAD_FORMAT_XYZW"), "new decoder did not decode typed buffer format-xyzw load"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_x"), + Check(Common::ContainsStr(result.decoded_dump, "TBUFFER_STORE_FORMAT_X"), "new decoder did not decode typed buffer format-x store"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_xy"), + Check(Common::ContainsStr(result.decoded_dump, "TBUFFER_STORE_FORMAT_XY"), "new decoder did not decode typed buffer format-xy store"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_xyzw"), + Check(Common::ContainsStr(result.decoded_dump, "TBUFFER_STORE_FORMAT_XYZW"), "new decoder did not decode typed buffer format-xyzw store"); Check(Common::ContainsStr(result.decoded_dump, "dfmt=14 nfmt=7"), "MTBUF decode did not expose dfmt/nfmt metadata"); @@ -5837,18 +5897,12 @@ void TestNewShaderRecompilerTypedBufferLowering() { "MTBUF decode did not preserve typed metadata"); Check(Common::ContainsStr(result.decoded_dump, "offen=1"), "MTBUF decode did not preserve offen metadata"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v60"), - "typed buffer x load did not lower through shared buffer load IR"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v71"), - "typed buffer xyzw load did not expand to last shared dword load"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v72"), - "typed buffer x store did not lower through shared buffer store IR"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v83"), - "typed buffer xyzw store did not expand to last shared dword store"); - Check(Common::ContainsStr(result.ir_dump, "typed=1"), - "typed buffer metadata did not survive into IR"); - Check(Common::ContainsStr(result.ir_dump, "dfmt=13 nfmt=7"), - "typed buffer format metadata did not survive into IR"); + Check(!Common::ContainsStr(result.ir_dump, "BufferLoadDword v71"), + "typed buffer load retained a scalar tail sibling"); + Check(CountSourceOccurrences(result.ir_dump, "StoreBufferU32 ") == 1u && + CountSourceOccurrences(result.ir_dump, "StoreBufferU32x2 ") == 1u && + CountSourceOccurrences(result.ir_dump, "StoreBufferU32x4 ") == 1u, + "typed buffer stores were not preserved as native-width operations"); Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); Check(SpirvContainsOpcode(result.spirv, 61), @@ -7764,6 +7818,47 @@ void TestNewShaderRecompilerCfgIrreducibleDispatcher() { CheckSpirvBinaryValidates(result.spirv); } +void TestNewShaderRecompilerDispatcherSpillsU32x3() { + using namespace ShaderRecompiler; + + const uint32_t shader[] = { + EncodeSopp(0x05, 2), // entry -> B, fallthrough A + EncodeSopp(0x02, 0), // A -> C + EncodeSopp(0x05, 0xfffeu), // C -> A, fallthrough B + EncodeSopp(0x02, 0xfffeu), // B -> C + EncodeSopp(0x01), + }; + auto options = MakeCompileOptions(ShaderType::Compute); + CompileResult result; + std::string error; + Check(TryRecompile(shader, options, result, &error), error.c_str()); + Check(result.program.dispatcher_fallback && + result.program.values->blocks.size() >= 2u, + "U32x3 spill fixture did not select dispatcher mode"); + + auto program = result.program; + IR::IREmitter definition(program.values->blocks[0]); + const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU32x3, + {IR::Value(1u), IR::Value(2u), + IR::Value(3u)}); + IR::IREmitter use(program.values->blocks[1]); + use.Emit(IR::ValueOpcode::CompositeExtractU32x3, + {vector, IR::Value(2u)}); + Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str()); + + std::vector spirv; + Check(Spirv::EmitProgram(program, result.resources, options.input_info, + spirv, &error), + error.c_str()); + CheckSpirvBinaryValidates(spirv); + const auto before = MeasureSpirv(result.spirv); + const auto after = MeasureSpirv(spirv); + Check(after.function_variables == before.function_variables + 1u && + after.loads == before.loads + 1u && + after.stores == before.stores + 1u, + "dispatcher did not use one canonical U32x3 spill slot"); +} + void TestComputeShaderInputWaveSize() { const auto decode_wave_size = [](uint32_t rsrc1) { const bool wave32 = (((rsrc1 >> Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT) & @@ -8408,6 +8503,82 @@ void TestValuePhiValidation() { validate_cfg(false); } +void TestNativeWideValueValidation() { + using namespace ShaderRecompiler::IR; + + const auto make_program = [] { + ValueProgram program; + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + program.block_info.emplace_back(); + program.block_info.front().id = 0; + program.block_info.front().terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::Return; + return program; + }; + const auto append_load = [](ValueProgram &program, ValueOpcode opcode, + MemoryFlags flags) { + IREmitter ir(program.blocks.front()); + const auto resource = ir.Emit(ValueOpcode::GetBufferResource, + {Value(0u), Value(0u), Value(0u), Value(0u)}); + ir.Emit(opcode, + {resource, Value(0u), Value(0u), Value(0u), Value(true)}, flags); + }; + const auto check_rejected = [](const ValueProgram &program, + const char *expected) { + std::string error; + Check(!ValidateValueProgram(program, true, &error) && + Common::ContainsStr(error, expected), + "malformed native-wide value operation was not rejected"); + }; + + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::Buffer; + memory.data_dwords = 3; + program.memory_info.push_back(memory); + append_load(program, ValueOpcode::LoadBufferU32x3, + MemoryFlags{.index = 1}); + check_rejected(program, "invalid memory-info index"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::Lds; + memory.data_dwords = 3; + program.memory_info.push_back(memory); + append_load(program, ValueOpcode::LoadBufferU32x3, {}); + check_rejected(program, "non-buffer resource kind"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::Buffer; + memory.data_dwords = 2; + program.memory_info.push_back(memory); + append_load(program, ValueOpcode::LoadBufferU32x3, {}); + check_rejected(program, "inconsistent native-wide metadata"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::Buffer; + memory.data_dwords = 3; + program.memory_info.push_back(memory); + append_load(program, ValueOpcode::LoadBufferU32, {}); + check_rejected(program, "scalar-sibling width metadata"); + } + { + auto program = make_program(); + IREmitter ir(program.blocks.front()); + const auto vector = ir.Emit(ValueOpcode::CompositeConstructU32x3, + {Value(1u), Value(2u), Value(3u)}); + ir.Emit(ValueOpcode::CompositeExtractU32x3, {vector, Value(3u)}); + check_rejected(program, "invalid component index"); + } +} + void TestNewShaderRecompilerZeroInitialRegisterState() { const uint32_t shader[] = { EncodeVop1(0x01, 0, 100), // v_mov_b32 v0, s100 @@ -9658,9 +9829,15 @@ void TestResourceTrackingRealDensePatching() { case ShaderRecompiler::IR::ValueOpcode::LoadBufferU8: case ShaderRecompiler::IR::ValueOpcode::LoadBufferU16: case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32: + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32x2: + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32x3: + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32x4: case ShaderRecompiler::IR::ValueOpcode::StoreBufferU8: case ShaderRecompiler::IR::ValueOpcode::StoreBufferU16: - case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32: { + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32: + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32x2: + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32x3: + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32x4: { const auto &memory = ir.values->memory_info [inst.Flags().index]; @@ -9694,7 +9871,7 @@ void TestResourceTrackingRealDensePatching() { } } } - Check(buffer_use == 4 && image_use == 2 && ir.info.buffers[0].read && + Check(buffer_use == 3 && image_use == 2 && ir.info.buffers[0].read && ir.info.buffers[1].written, "real tracked resource access facts were incomplete"); } @@ -10177,22 +10354,49 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { EncodeSopp(0x01), }; const auto wide_result = compile("wide-buffer", wide_buffer, - {.words = 915, - .instructions = 254, + {.words = 797, + .instructions = 209, .runtime_arrays = 1, .variables = 2, .loads = 9, .stores = 4, - .array_lengths = 8, - .phis = 8, - .labels = 52, - .selection_merges = 16, - .branches = 35, - .conditional_branches = 16}); + .array_lengths = 2, + .phis = 5, + .labels = 34, + .selection_merges = 10, + .branches = 23, + .conditional_branches = 10}); Check(Common::ContainsStr(wide_result.decoded_dump, "BUFFER_LOAD_DWORDX4"), "wide buffer size fixture no longer decodes its x4 load"); Check(Common::ContainsStr(wide_result.decoded_dump, "BUFFER_STORE_DWORDX4"), "wide buffer size fixture no longer decodes its x4 store"); + uint32_t wide_loads = 0; + uint32_t wide_stores = 0; + uint32_t scalar_loads = 0; + uint32_t scalar_stores = 0; + for (const auto *block : wide_result.program.values->blocks) { + for (const auto &inst : *block) { + switch (inst.GetOpcode()) { + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32x4: + wide_loads++; + break; + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32x4: + wide_stores++; + break; + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32: + scalar_loads++; + break; + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32: + scalar_stores++; + break; + default: + break; + } + } + } + Check(wide_loads == 1u && wide_stores == 1u && scalar_loads == 0u && + scalar_stores == 0u, + "wide buffer fixture retained scalar sibling memory operations"); const uint32_t wqm[] = { EncodeVopc(0xc1, 5 + 256, 8), // v_cmp_lt_u32 vcc, v5, v8 @@ -10260,6 +10464,9 @@ int main() { TestDemandDrivenSpirvDeclarations(); TestNativeSubgroupPolicy(); TestNewShaderRecompilerSMovB32(); + TestNewShaderRecompilerNativeWideBufferIr(); + TestNewShaderRecompilerMubufFormatLowering(); + TestNewShaderRecompilerTypedBufferLowering(); TestNewShaderRecompilerCapturedVop1SdwaByteConvert(); TestNewShaderRecompilerScalarMemoryBindingDomains(); // Opcode semantics and optimized direct SPIR-V are exercised by @@ -10302,6 +10509,7 @@ int main() { TestNewShaderRecompilerCfgNestedEarlyExitSharedTerminal(); TestNewShaderRecompilerCfgExternallyEnteredSelectionDispatcher(); TestNewShaderRecompilerCfgIrreducibleDispatcher(); + TestNewShaderRecompilerDispatcherSpillsU32x3(); TestComputeShaderInputWaveSize(); TestNewShaderRecompilerBufferLoadsGuardedByExec(); TestNewShaderRecompilerBufferAtomicsGuardedByBounds(); @@ -10313,6 +10521,7 @@ int main() { TestTypedEntryStateIsMinimal(); TestFinalSsaRejectsRegisterStatePseudos(); TestValuePhiValidation(); + TestNativeWideValueValidation(); TestNewShaderRecompilerZeroInitialRegisterState(); TestNewShaderRecompilerVertexSystemInputsWithoutMirrors(); TestNewShaderRecompilerVertexExportUsesLaneExecMask();