diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index bc4134e26..7ca1947ba 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -95,12 +95,25 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { if (!program.info.samplers.empty()) { Expect(Kind::Samplers, Dense(program.info.samplers.size())); } - const auto uses_gds = program.values != nullptr && - std::ranges::any_of(program.values->blocks, [](const IR::Block* block) { - return std::ranges::any_of(*block, [](const IR::Inst& inst) { - return inst.GetOpcode() == IR::ValueOpcode::GetGdsResource; - }); - }); + bool uses_gds = false; + if (program.values != nullptr) { + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) { + continue; + } + const auto index = inst.Flags().index; + if (index >= program.values->memory_info.size()) { + return Fail(error, "shared operation has invalid memory metadata"); + } + const auto kind = program.values->memory_info[index].kind; + if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { + return Fail(error, "shared operation has invalid resource kind"); + } + uses_gds |= kind == IR::ResourceKind::Gds; + } + } + } if (uses_gds) { Expect(Kind::Gds); } @@ -249,6 +262,26 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { } } } + const auto shared_access = IR::SharedAccessOf(inst.GetOpcode()); + if (shared_access != IR::SharedAccess::None) { + const auto index = inst.Flags().index; + if (index >= program.values->memory_info.size()) { + return Fail(error, "shared operation has invalid memory metadata"); + } + const auto kind = program.values->memory_info[index].kind; + if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { + return Fail(error, "shared operation has invalid resource kind"); + } + if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) { + requirements.function_lds = true; + } + if (shared_access == IR::SharedAccess::Append || + shared_access == IR::SharedAccess::Consume) { + MarkExactSubgroup(); + requirements.subgroup_shuffle = true; + requirements.subgroup_local_invocation_id = true; + } + } switch (inst.GetOpcode()) { case IR::ValueOpcode::Ballot: MarkExactSubgroup(); break; case IR::ValueOpcode::DppMoveU32: @@ -268,9 +301,7 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { requirements.subgroup_local_invocation_id = true; break; } - case IR::ValueOpcode::Permlane16U32: - case IR::ValueOpcode::GdsDataAppend: - case IR::ValueOpcode::GdsDataConsume: { + case IR::ValueOpcode::Permlane16U32: { MarkExactSubgroup(); requirements.subgroup_shuffle = true; requirements.subgroup_local_invocation_id = true; @@ -282,41 +313,6 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { requirements.subgroup_local_invocation_id = true; break; } - case IR::ValueOpcode::DataAppend: - case IR::ValueOpcode::DataConsume: { - MarkExactSubgroup(); - requirements.subgroup_shuffle = true; - requirements.subgroup_local_invocation_id = true; - [[fallthrough]]; - } - case IR::ValueOpcode::LoadSharedU8: - case IR::ValueOpcode::LoadSharedU16: - case IR::ValueOpcode::LoadSharedU32: - case IR::ValueOpcode::WriteSharedU8: - case IR::ValueOpcode::WriteSharedU16: - case IR::ValueOpcode::WriteSharedU32: - case IR::ValueOpcode::SharedAtomicFMin32: - case IR::ValueOpcode::SharedAtomicFMax32: - case IR::ValueOpcode::SharedAtomicSwap32: - case IR::ValueOpcode::SharedAtomicIAdd32: - case IR::ValueOpcode::SharedAtomicISub32: - case IR::ValueOpcode::SharedAtomicSMin32: - case IR::ValueOpcode::SharedAtomicUMin32: - case IR::ValueOpcode::SharedAtomicSMax32: - case IR::ValueOpcode::SharedAtomicUMax32: - case IR::ValueOpcode::SharedAtomicAnd32: - case IR::ValueOpcode::SharedAtomicOr32: - case IR::ValueOpcode::SharedAtomicXor32: { - const auto index = inst.Flags().index; - if (index >= program.values->memory_info.size()) { - return Fail(error, "shared operation has invalid memory metadata"); - } - if (program.stage != ShaderType::Compute && - program.values->memory_info[index].kind == IR::ResourceKind::Lds) { - requirements.function_lds = true; - } - break; - } case IR::ValueOpcode::LaneId: requirements.subgroup_local_invocation_id = true; break; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 85392dc8a..d68fae67c 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -25,16 +25,14 @@ uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) { const auto merged_two = state.builder.AllocateId(); const auto quad_bits = state.builder.AllocateId(); const auto result = state.builder.AllocateId(); - state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), shifted_one, value, - ConstantU32(state, 1)}); state.builder.AddFunction( - {OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one}); - state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), shifted_two, merged_one, - ConstantU32(state, 2)}); + {OpShiftRightLogical, TypeU32(state), shifted_one, value, ConstantU32(state, 1)}); + state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one}); state.builder.AddFunction( - {OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two}); - state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), quad_bits, merged_two, - ConstantU32(state, 0x11111111u)}); + {OpShiftRightLogical, TypeU32(state), shifted_two, merged_one, ConstantU32(state, 2)}); + state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two}); + state.builder.AddFunction( + {OpBitwiseAnd, TypeU32(state), quad_bits, merged_two, ConstantU32(state, 0x11111111u)}); state.builder.AddFunction( {OpIMul, TypeU32(state), result, quad_bits, ConstantU32(state, 0x0fu)}); return result; @@ -103,7 +101,7 @@ uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) { ConstantU32(state, ScopeSubgroup), active}); const auto low = state.builder.AllocateId(); state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0}); - const auto wqm_low = EmitWqmWordU32(state, low); + const auto wqm_low = EmitWqmWordU32(state, low); const auto lane = EmitSubgroupLocalInvocationId(state); uint32_t mask = wqm_low; uint32_t bit_lane = lane; @@ -120,9 +118,9 @@ uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) { state.builder.AddFunction( {OpBitwiseAnd, TypeU32(state), bit_lane, lane, ConstantU32(state, 31)}); } - const auto bit = state.builder.AllocateId(); - const auto hit = state.builder.AllocateId(); - const auto result = state.builder.AllocateId(); + const auto bit = state.builder.AllocateId(); + const auto hit = state.builder.AllocateId(); + const auto result = state.builder.AllocateId(); state.builder.AddFunction( {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), bit_lane}); state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit}); @@ -531,8 +529,6 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) { case IR::ValueOpcode::GetAddressResource: case IR::ValueOpcode::GetImageResource: case IR::ValueOpcode::GetSamplerResource: - case IR::ValueOpcode::GetLdsResource: - case IR::ValueOpcode::GetGdsResource: case IR::ValueOpcode::MakeImageAddress: return true; default: return false; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 75318c646..a04347aec 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -314,6 +314,7 @@ struct EmitterState { std::array storage_buffer_offsets {}; uint32_t address_memory_variable = 0; uint32_t gds_variable = 0; + uint32_t gds_length = 0; uint32_t push_constant_variable = 0; uint32_t vsharp_storage_variable = 0; uint32_t flattened_srt_variable = 0; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 31034c617..bbfea504e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -141,7 +141,11 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo ctx.Arg(inst, 3)); } if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) { - return ctx.Arg(inst, 1); + if (mem.offset == 0u) { + return ctx.Arg(inst, 0); + } + return Binary(ctx.state, OpIAdd, TypeU32(ctx.state), ctx.Arg(inst, 0), + ConstantU32(ctx.state, mem.offset)); } return AddressByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2)); } @@ -493,35 +497,25 @@ void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor uint32_t AtomicOpcode(IR::ValueOpcode opcode) { switch (opcode) { case IR::ValueOpcode::BufferAtomicSwap32: - case IR::ValueOpcode::SharedAtomicSwap32: - case IR::ValueOpcode::GdsAtomicSwap32: return OpAtomicExchange; + case IR::ValueOpcode::SharedAtomicSwap32: return OpAtomicExchange; case IR::ValueOpcode::BufferAtomicIAdd32: - case IR::ValueOpcode::SharedAtomicIAdd32: - case IR::ValueOpcode::GdsAtomicIAdd32: return OpAtomicIAdd; + case IR::ValueOpcode::SharedAtomicIAdd32: return OpAtomicIAdd; case IR::ValueOpcode::BufferAtomicISub32: - case IR::ValueOpcode::SharedAtomicISub32: - case IR::ValueOpcode::GdsAtomicISub32: return OpAtomicISub; + case IR::ValueOpcode::SharedAtomicISub32: return OpAtomicISub; case IR::ValueOpcode::BufferAtomicSMin32: - case IR::ValueOpcode::SharedAtomicSMin32: - case IR::ValueOpcode::GdsAtomicSMin32: return OpAtomicSMin; + case IR::ValueOpcode::SharedAtomicSMin32: return OpAtomicSMin; case IR::ValueOpcode::BufferAtomicUMin32: - case IR::ValueOpcode::SharedAtomicUMin32: - case IR::ValueOpcode::GdsAtomicUMin32: return OpAtomicUMin; + case IR::ValueOpcode::SharedAtomicUMin32: return OpAtomicUMin; case IR::ValueOpcode::BufferAtomicSMax32: - case IR::ValueOpcode::SharedAtomicSMax32: - case IR::ValueOpcode::GdsAtomicSMax32: return OpAtomicSMax; + case IR::ValueOpcode::SharedAtomicSMax32: return OpAtomicSMax; case IR::ValueOpcode::BufferAtomicUMax32: - case IR::ValueOpcode::SharedAtomicUMax32: - case IR::ValueOpcode::GdsAtomicUMax32: return OpAtomicUMax; + case IR::ValueOpcode::SharedAtomicUMax32: return OpAtomicUMax; case IR::ValueOpcode::BufferAtomicAnd32: - case IR::ValueOpcode::SharedAtomicAnd32: - case IR::ValueOpcode::GdsAtomicAnd32: return OpAtomicAnd; + case IR::ValueOpcode::SharedAtomicAnd32: return OpAtomicAnd; case IR::ValueOpcode::BufferAtomicOr32: - case IR::ValueOpcode::SharedAtomicOr32: - case IR::ValueOpcode::GdsAtomicOr32: return OpAtomicOr; + case IR::ValueOpcode::SharedAtomicOr32: return OpAtomicOr; case IR::ValueOpcode::BufferAtomicXor32: - case IR::ValueOpcode::SharedAtomicXor32: - case IR::ValueOpcode::GdsAtomicXor32: return OpAtomicXor; + case IR::ValueOpcode::SharedAtomicXor32: return OpAtomicXor; default: return 0; } } @@ -614,7 +608,7 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR EmitIfCondition( ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { ctx.state.builder.AddFunction( - {OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)}); + {OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)}); const auto data = ctx.state.builder.AllocateId(); ctx.state.builder.AddFunction( {OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable}); @@ -623,7 +617,7 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR mem.kind, [&](uint32_t old) { const auto old_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), old); const auto compare_f = - Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 3)); + Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2)); const auto data_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), data); const auto compare = Binary(ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan, @@ -637,9 +631,9 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR return 0; } -uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bool append) { +uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) { auto& state = ctx.state; - const auto m0 = ctx.Arg(inst, 1); + const auto m0 = ctx.Arg(inst, 0); const auto base = Binary(state, OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16)); const auto size = Binary(state, OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu)); @@ -647,11 +641,10 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo Binary(state, OpIAdd, TypeU32(state), base, ConstantU32(state, ctx.Memory(inst).offset)); const auto raw_index = Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); - auto mem = ctx.Memory(inst); - mem.kind = gds ? IR::ResourceKind::Gds : IR::ResourceKind::Lds; + const auto mem = ctx.Memory(inst); const auto access = PrepareMemoryResourceAccess(state, mem); const auto index = EmitMemoryElementIndex(state, access, raw_index); - const auto exec = ctx.Arg(inst, 2); + const auto exec = ctx.Arg(inst, 1); const auto ballot = state.builder.AllocateId(); state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot, ConstantU32(state, ScopeSubgroup), exec}); @@ -659,8 +652,8 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo const auto high = state.builder.AllocateId(); state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0}); state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1}); - const auto count_low = state.per_invocation_masks ? low : ctx.Arg(inst, 3); - const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 4); + const auto count_low = state.per_invocation_masks ? low : ctx.Arg(inst, 2); + const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 3); const auto count = Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), count_low), Unary(state, OpBitCount, TypeU32(state), count_high)); @@ -671,17 +664,19 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), first); const auto storage_bounds = EmitMemoryElementInBounds(state, access, index); const auto m0_bounds = - gds ? Binary(state, OpINotEqual, TypeBool(state), size, ConstantU32(state, 0)) + mem.kind == IR::ResourceKind::Gds + ? Binary(state, OpINotEqual, TypeBool(state), size, ConstantU32(state, 0)) : Binary(state, OpULessThan, TypeBool(state), ConstantU32(state, ctx.Memory(inst).offset + 3u), size); const auto condition = AndCondition( state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds))); const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() { const auto value = state.builder.AllocateId(); - state.builder.AddFunction({append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value, - EmitMemoryElementPointer(state, access, index), - ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup), - ConstantU32(state, MemorySemanticsNone), count}); + state.builder.AddFunction( + {append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value, + EmitMemoryElementPointer(state, access, index), + ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? ScopeDevice : ScopeWorkgroup), + ConstantU32(state, MemorySemanticsNone), count}); return value; }); const auto result = state.builder.AllocateId(); @@ -706,9 +701,9 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR:: const MemoryResourceAccess& resource, uint32_t components, FormattedAccess access) { PreparedFormattedMemory plan; - plan.format = BufferFormat(ctx, inst, mem); - plan.info = Format::GetFormatInfo(plan.format); - plan.resource = resource; + plan.format = BufferFormat(ctx, inst, mem); + plan.info = Format::GetFormatInfo(plan.format); + plan.resource = resource; std::array required_components {}; if (access == FormattedAccess::Load) { for (uint32_t output = 0; output < components; output++) { @@ -718,8 +713,8 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR:: } } } else { - for (uint32_t component = 0; - component < std::min(components, plan.info.component_count); component++) { + for (uint32_t component = 0; component < std::min(components, plan.info.component_count); + component++) { required_components[component] = true; } } @@ -812,7 +807,7 @@ uint32_t LoadWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t co mem.formatted ? BufferFormat(ctx, inst, mem) : Prospero::BufferFormat::kInvalid; if (Format::IsKnownFormat(format)) { const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, components, - FormattedAccess::Load); + FormattedAccess::Load); return EmitValueOrDefaultIfCondition( state, plan.in_bounds, TypeU32Composite(state, components), FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() { @@ -842,7 +837,7 @@ void StoreWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo mem.formatted ? BufferFormat(ctx, inst, mem) : Prospero::BufferFormat::kInvalid; if (Format::IsKnownFormat(format)) { const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, components, - FormattedAccess::Store); + FormattedAccess::Store); EmitIfCondition(state, plan.in_bounds, [&]() { for (uint32_t component = 0; component < components; component++) { const auto data = state.builder.AllocateId(); @@ -862,6 +857,54 @@ void StoreWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo }); } +uint32_t LoadWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) { + auto& state = ctx.state; + return EmitValueOrDefaultIfCondition( + state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU32Composite(state, components), + ConstantU32CompositeZero(state, components), [&]() { + const auto mem = ctx.Memory(inst); + const auto resource = PrepareMemoryResourceAccess(state, mem); + const auto base = ByteAddress(ctx, inst, mem); + std::array values {}; + for (uint32_t component = 0; component < components; component++) { + const auto address = component == 0u + ? base + : Binary(state, OpIAdd, TypeU32(state), base, + ConstantU32(state, component * 4u)); + const auto raw_index = Binary(state, OpShiftRightLogical, TypeU32(state), address, + ConstantU32(state, 2)); + const auto access = PrepareMemoryElement(ctx, mem, resource, raw_index); + values[component] = EmitValueOrZeroIfCondition( + state, EmitMemoryElementInBounds(state, access.resource, access.index), + [&]() { return LoadWordInBounds(ctx, access.resource, access.index); }); + } + return ConstructU32Composite(state, components, values); + }); +} + +void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) { + auto& state = ctx.state; + EmitIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto mem = ctx.Memory(inst); + const auto resource = PrepareMemoryResourceAccess(state, mem); + const auto base = ByteAddress(ctx, inst, mem); + for (uint32_t component = 0; component < components; component++) { + const auto address = component == 0u + ? base + : Binary(state, OpIAdd, TypeU32(state), base, + ConstantU32(state, component * 4u)); + const auto raw_index = + Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); + const auto access = PrepareMemoryElement(ctx, mem, resource, raw_index); + EmitIfCondition(state, EmitMemoryElementInBounds(state, access.resource, access.index), + [&]() { + StoreWordInBounds(ctx, access.resource, access.index, + ctx.Arg(inst, component + 1u)); + }); + } + }); +} + } // namespace bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { @@ -876,6 +919,15 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { } return true; } + const auto shared_components = IR::SharedComponentCount(op); + if (shared_components > 1u) { + if (IR::SharedAccessOf(op) == IR::SharedAccess::Read) { + ctx.Define(inst, LoadWideShared(ctx, inst, shared_components)); + } else { + StoreWideShared(ctx, inst, shared_components); + } + return true; + } if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) && ctx.Memory(inst).planning_only) { return true; @@ -917,25 +969,17 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 || op == IR::ValueOpcode::LoadBufferU16 || op == IR::ValueOpcode::LoadBufferU32; - const bool load_shared = - op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadSharedU16 || - op == IR::ValueOpcode::LoadSharedU32 || op == IR::ValueOpcode::LoadGdsU8 || - op == IR::ValueOpcode::LoadGdsU16 || op == IR::ValueOpcode::LoadGdsU32; + const bool load_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Read; if (load_address || load_buffer || load_shared) { - auto mem = ctx.Memory(inst); - if (load_shared) - mem.kind = op == IR::ValueOpcode::LoadGdsU8 || op == IR::ValueOpcode::LoadGdsU16 || - op == IR::ValueOpcode::LoadGdsU32 - ? IR::ResourceKind::Gds - : IR::ResourceKind::Lds; - uint32_t value = 0; + const auto mem = ctx.Memory(inst); + uint32_t value = 0; if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted) value = FormattedLoad(ctx, inst, mem); else if (op == IR::ValueOpcode::LoadAddressU8 || op == IR::ValueOpcode::LoadBufferU8 || - op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadGdsU8) + op == IR::ValueOpcode::LoadSharedU8) value = LoadSubword(ctx, inst, mem, 8, false); else if (op == IR::ValueOpcode::LoadAddressU16 || op == IR::ValueOpcode::LoadBufferU16 || - op == IR::ValueOpcode::LoadSharedU16 || op == IR::ValueOpcode::LoadGdsU16) + op == IR::ValueOpcode::LoadSharedU16) value = LoadSubword(ctx, inst, mem, 16, false); else value = LoadWord(ctx, inst, mem); @@ -948,24 +992,16 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { const bool store_buffer = op == IR::ValueOpcode::StoreBufferU8 || op == IR::ValueOpcode::StoreBufferU16 || op == IR::ValueOpcode::StoreBufferU32; - const bool store_shared = - op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteSharedU16 || - op == IR::ValueOpcode::WriteSharedU32 || op == IR::ValueOpcode::WriteGdsU8 || - op == IR::ValueOpcode::WriteGdsU16 || op == IR::ValueOpcode::WriteGdsU32; + const bool store_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Write; if (store_address || store_buffer || store_shared) { - auto mem = ctx.Memory(inst); - if (store_shared) - mem.kind = op == IR::ValueOpcode::WriteGdsU8 || op == IR::ValueOpcode::WriteGdsU16 || - op == IR::ValueOpcode::WriteGdsU32 - ? IR::ResourceKind::Gds - : IR::ResourceKind::Lds; + const auto mem = ctx.Memory(inst); if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted) FormattedStore(ctx, inst, mem); else if (op == IR::ValueOpcode::StoreAddressU8 || op == IR::ValueOpcode::StoreBufferU8 || - op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteGdsU8) + op == IR::ValueOpcode::WriteSharedU8) StoreSubword(ctx, inst, mem, 8); else if (op == IR::ValueOpcode::StoreAddressU16 || op == IR::ValueOpcode::StoreBufferU16 || - op == IR::ValueOpcode::WriteSharedU16 || op == IR::ValueOpcode::WriteGdsU16) + op == IR::ValueOpcode::WriteSharedU16) StoreSubword(ctx, inst, mem, 16); else StoreWord(ctx, inst, mem); @@ -973,12 +1009,7 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { } const auto atomic_opcode = AtomicOpcode(op); if (atomic_opcode != 0) { - auto mem = ctx.Memory(inst); - if (op >= IR::ValueOpcode::SharedAtomicSwap32 && op <= IR::ValueOpcode::SharedAtomicXor32) - mem.kind = IR::ResourceKind::Lds; - else if (op >= IR::ValueOpcode::GdsAtomicSwap32 && op <= IR::ValueOpcode::GdsAtomicXor32) - mem.kind = IR::ResourceKind::Gds; - ctx.Define(inst, EmitAtomic(ctx, inst, mem, true)); + ctx.Define(inst, EmitAtomic(ctx, inst, ctx.Memory(inst), true)); return true; } if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) { @@ -986,24 +1017,12 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { op == IR::ValueOpcode::BufferAtomicFMax32)); return true; } - if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32 || - op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32) { - auto mem = ctx.Memory(inst); - mem.kind = op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32 - ? IR::ResourceKind::Gds - : IR::ResourceKind::Lds; - SharedFloatAtomic(ctx, inst, mem, - op == IR::ValueOpcode::SharedAtomicFMax32 || - op == IR::ValueOpcode::GdsAtomicFMax32); + if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32) { + SharedFloatAtomic(ctx, inst, ctx.Memory(inst), op == IR::ValueOpcode::SharedAtomicFMax32); return true; } - if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume || - op == IR::ValueOpcode::GdsDataAppend || op == IR::ValueOpcode::GdsDataConsume) { - ctx.Define(inst, AppendConsume(ctx, inst, - op == IR::ValueOpcode::GdsDataAppend || - op == IR::ValueOpcode::GdsDataConsume, - op == IR::ValueOpcode::DataAppend || - op == IR::ValueOpcode::GdsDataAppend)); + if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume) { + ctx.Define(inst, AppendConsume(ctx, inst, op == IR::ValueOpcode::DataAppend)); return true; } if (op == IR::ValueOpcode::SwizzleU32) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index aa34caca2..0f482b74d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -93,7 +93,11 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR:: "GDS binding was not emitted"); } access.object_pointer = state.gds_variable; - break; + if (state.gds_length == 0) { + EXIT("GDS length was not prepared at function entry\n"); + } + access.length = state.gds_length; + return access; case IR::ResourceKind::ScalarAddress: case IR::ResourceKind::Flat: case IR::ResourceKind::Global: diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index fbb0aedd2..0b5f38b25 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -592,9 +592,7 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std: if (std::ranges::any_of(*block, [](const IR::Inst& inst) { return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 || inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 || - inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32 || - inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMin32 || - inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMax32; + inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32; })) { ctx.scratch_u32_variable = state.builder.AllocateId(); break; @@ -631,6 +629,11 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std: TypePointer(state, StorageClassFunction, TypeU32(state)), ctx.scratch_u32_variable, StorageClassFunction}); } + if (state.gds_variable != 0) { + state.gds_length = state.builder.AllocateId(); + state.builder.AddFunction( + {OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0}); + } if (state.pixel_valid_mask_variable != 0) { state.builder.AddFunction( {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)}); diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index de2937e9e..8501cd310 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -96,7 +96,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = { {0x29u, Opcode::DS_AND_RTN_B32, 1, 32}, {0x2au, Opcode::DS_OR_RTN_B32, 1, 32}, {0x2bu, Opcode::DS_XOR_RTN_B32, 1, 32}, {0x2du, Opcode::DS_WRXCHG_RTN_B32, 1, 32}, {0x35u, Opcode::DS_SWIZZLE_B32, 1, 32}, {0x36u, Opcode::DS_READ_B32, 1, 32}, - {0x37u, Opcode::DS_READ2_B32, 2, 32}, {0x38u, Opcode::DS_READ2_B32, 2, 32}, + {0x37u, Opcode::DS_READ2_B32, 2, 32}, {0x38u, Opcode::DS_READ2ST64_B32, 2, 32}, {0x39u, Opcode::DS_READ_I8, 1, 8, true}, {0x3au, Opcode::DS_READ_U8, 1, 8}, {0x3bu, Opcode::DS_READ_I16, 1, 16, true}, {0x3cu, Opcode::DS_READ_U16, 1, 16}, {0x3du, Opcode::DS_CONSUME, 1, 32}, {0x3eu, Opcode::DS_APPEND, 1, 32}, @@ -403,10 +403,11 @@ void DecodeDs(uint32_t pc, std::span code, uint32_t word_index, SetUnsupported(inst, Family::DS, opcode, "DS read addtid data operands are not implemented"); } - if (inst.opcode == Opcode::DS_WRITE2_B32 || opcode == 0x37u) { + if (inst.opcode == Opcode::DS_WRITE2_B32 || inst.opcode == Opcode::DS_READ2_B32) { inst.offset = offset0 * 4u; inst.secondary_offset = offset1 * 4u; - } else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 || opcode == 0x38u) { + } else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 || + inst.opcode == Opcode::DS_READ2ST64_B32) { inst.offset = offset0 * 256u; inst.secondary_offset = offset1 * 256u; } else if (inst.opcode == Opcode::DS_WRITE2_B64 || inst.opcode == Opcode::DS_READ2_B64) { diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 5ecce0a20..999e2c69b 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -658,6 +658,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::DS_READ_I16: case Opcode::DS_READ_U16: case Opcode::DS_READ2_B32: + case Opcode::DS_READ2ST64_B32: case Opcode::DS_READ_B32: case Opcode::DS_READ_B64: case Opcode::DS_READ2_B64: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 9821a711f..21d3b1795 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -490,6 +490,7 @@ enum class Opcode { DS_READ_I16, DS_READ_U16, DS_READ2_B32, + DS_READ2ST64_B32, DS_READ_B32, DS_READ_B64, DS_READ2_B64, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index b782525b0..dedefbb41 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -65,11 +65,6 @@ IR::Value Translator::GetSamplerResource(const IR::MemoryInfo& memory) { GetResourceDword(memory.sampler, 2), GetResourceDword(memory.sampler, 3)}); } -IR::Value Translator::GetSharedResource(IR::ResourceKind kind) { - return ir.Emit(kind == IR::ResourceKind::Gds ? IR::ValueOpcode::GetGdsResource - : IR::ValueOpcode::GetLdsResource); -} - IR::Value Translator::MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base) { std::array components {}; components[0] = ReadRawU32(PlainOperand(base)); @@ -170,22 +165,7 @@ IR::ValueOpcode Translator::BufferAtomicOpcode(IR::Opcode opcode) { } } -IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) { - if (gds) { - switch (opcode) { - case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::GdsAtomicSwap32; - case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::GdsAtomicIAdd32; - case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::GdsAtomicISub32; - case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::GdsAtomicSMin32; - case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::GdsAtomicUMin32; - case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::GdsAtomicSMax32; - case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::GdsAtomicUMax32; - case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::GdsAtomicAnd32; - case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::GdsAtomicOr32; - case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::GdsAtomicXor32; - default: EXIT("invalid GDS atomic opcode"); - } - } +IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode) { switch (opcode) { case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::SharedAtomicSwap32; case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::SharedAtomicIAdd32; @@ -197,7 +177,7 @@ IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) { case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::SharedAtomicAnd32; case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::SharedAtomicOr32; case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::SharedAtomicXor32; - default: EXIT("invalid LDS atomic opcode"); + default: EXIT("invalid shared atomic opcode"); } } @@ -363,13 +343,9 @@ bool Translator::TranslateAtomicMemory(const IR::Instruction& inst) { } case IR::ResourceKind::Lds: case IR::ResourceKind::Gds: { - const bool gds = inst.memory.kind == IR::ResourceKind::Gds; - const auto resource = GetSharedResource(inst.memory.kind); - const auto address = - ir.IAdd(ReadU32(inst.src[1]), IR::U32(IR::Value(inst.memory.offset))); - result = ir.Emit(SharedAtomicOpcode(inst.op, gds), - {resource, address, ReadU32(inst.src[0]), ir.GetExec()}, - AddMemoryInfo(inst)); + const auto address = ReadU32(inst.src[1]); + result = ir.Emit(SharedAtomicOpcode(inst.op), + {address, ReadU32(inst.src[0]), ir.GetExec()}, AddMemoryInfo(inst)); break; } default: return false; @@ -518,10 +494,51 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) { if (!shared && inst.op != IR::Opcode::DsSwizzleB32) { return false; } - const bool gds = inst.memory.kind == IR::ResourceKind::Gds; - const auto resource = shared ? GetSharedResource(inst.memory.kind) : IR::Value {}; - const auto shared_address = [&](uint32_t source) { - return ir.IAdd(ReadU32(inst.src[source]), IR::U32(IR::Value(inst.memory.offset))); + const auto shared_address = [&](uint32_t source) { return ReadU32(inst.src[source]); }; + const auto load_u32 = [&](uint32_t width, IR::U32 address, const IR::Instruction& source) { + IR::ValueOpcode opcode; + switch (width) { + case 1u: opcode = IR::ValueOpcode::LoadSharedU32; break; + case 2u: opcode = IR::ValueOpcode::LoadSharedU32x2; break; + case 3u: opcode = IR::ValueOpcode::LoadSharedU32x3; break; + case 4u: opcode = IR::ValueOpcode::LoadSharedU32x4; break; + default: EXIT("invalid shared load width"); + } + return ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(source)); + }; + const auto extract_u32 = [&](IR::Value value, uint32_t width, uint32_t index) { + if (width == 1u) { + return value; + } + const auto opcode = width == 2u ? IR::ValueOpcode::CompositeExtractU32x2 + : width == 3u ? IR::ValueOpcode::CompositeExtractU32x3 + : IR::ValueOpcode::CompositeExtractU32x4; + return ir.Emit(opcode, {value, IR::Value(index)}); + }; + const auto write_u32 = [&](uint32_t width, IR::U32 address, + const std::array& values, + const IR::Instruction& source) { + switch (width) { + case 1u: + ir.Emit(IR::ValueOpcode::WriteSharedU32, {address, values[0], ir.GetExec()}, + AddMemoryInfo(source)); + break; + case 2u: + ir.Emit(IR::ValueOpcode::WriteSharedU32x2, + {address, values[0], values[1], ir.GetExec()}, AddMemoryInfo(source)); + break; + case 3u: + ir.Emit(IR::ValueOpcode::WriteSharedU32x3, + {address, values[0], values[1], values[2], ir.GetExec()}, + AddMemoryInfo(source)); + break; + case 4u: + ir.Emit(IR::ValueOpcode::WriteSharedU32x4, + {address, values[0], values[1], values[2], values[3], ir.GetExec()}, + AddMemoryInfo(source)); + break; + default: EXIT("invalid shared store width"); + } }; switch (inst.op) { case IR::Opcode::DsReadUbyte: @@ -533,65 +550,109 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) { uint32_t bits; bool sign; if (inst.op == IR::Opcode::DsReadUbyte || inst.op == IR::Opcode::DsReadSbyte) { - opcode = gds ? IR::ValueOpcode::LoadGdsU8 : IR::ValueOpcode::LoadSharedU8; + opcode = IR::ValueOpcode::LoadSharedU8; bits = 8u; sign = inst.op == IR::Opcode::DsReadSbyte; } else if (inst.op == IR::Opcode::DsReadUshort || inst.op == IR::Opcode::DsReadSshort) { - opcode = gds ? IR::ValueOpcode::LoadGdsU16 : IR::ValueOpcode::LoadSharedU16; + opcode = IR::ValueOpcode::LoadSharedU16; bits = 16u; sign = inst.op == IR::Opcode::DsReadSshort; } else { - opcode = gds ? IR::ValueOpcode::LoadGdsU32 : IR::ValueOpcode::LoadSharedU32; - bits = 32u; - sign = false; + const auto width = inst.memory.data_dwords; + const auto loaded = load_u32(width, shared_address(0), inst); + for (uint32_t index = 0; index < width; index++) { + WriteOperand(OffsetOperand(inst.dst, index), extract_u32(loaded, width, index)); + } + return true; } const auto loaded = - ir.Emit(opcode, {resource, shared_address(0), ir.GetExec()}, AddMemoryInfo(inst)); + ir.Emit(opcode, {shared_address(0), ir.GetExec()}, AddMemoryInfo(inst)); WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign)); return true; } + case IR::Opcode::DsRead2B32: { + const auto width = inst.memory.data_dwords / 2u; + const auto address = shared_address(0); + auto first = inst; + first.memory.data_dwords = width; + first.memory.component_count = width; + const auto first_value = load_u32(width, address, first); + IR::Value second_value = first_value; + if (inst.memory.secondary_offset != inst.memory.offset) { + auto second = first; + second.memory.offset = inst.memory.secondary_offset; + second_value = load_u32(width, address, second); + } + for (uint32_t index = 0; index < width; index++) { + WriteOperand(OffsetOperand(inst.dst, index), + extract_u32(first_value, width, index)); + WriteOperand(OffsetOperand(inst.dst, width + index), + extract_u32(second_value, width, index)); + } + return true; + } case IR::Opcode::DsWriteByte: case IR::Opcode::DsWriteShort: case IR::Opcode::DsWriteB32: { - const auto data = ReadU32(inst.src[0]); + const auto width = inst.memory.data_dwords; + std::array values {}; + for (uint32_t index = 0; index < width; index++) { + values[index] = ReadU32(OffsetOperand(inst.src[0], index)); + } + const auto data = IR::U32(values[0]); const auto address = shared_address(1); IR::ValueOpcode opcode; IR::Value value; if (inst.op == IR::Opcode::DsWriteByte) { - opcode = gds ? IR::ValueOpcode::WriteGdsU8 : IR::ValueOpcode::WriteSharedU8; + opcode = IR::ValueOpcode::WriteSharedU8; value = NarrowSubdword(data, 8u); } else if (inst.op == IR::Opcode::DsWriteShort) { - opcode = gds ? IR::ValueOpcode::WriteGdsU16 : IR::ValueOpcode::WriteSharedU16; + opcode = IR::ValueOpcode::WriteSharedU16; value = NarrowSubdword(data, 16u); } else { - opcode = gds ? IR::ValueOpcode::WriteGdsU32 : IR::ValueOpcode::WriteSharedU32; - value = data; + write_u32(width, address, values, inst); + return true; + } + ir.Emit(opcode, {address, value, ir.GetExec()}, AddMemoryInfo(inst)); + return true; + } + case IR::Opcode::DsWrite2B32: { + const auto width = inst.memory.data_dwords / 2u; + const auto address = shared_address(1); + std::array first_values {}; + std::array second_values {}; + for (uint32_t index = 0; index < width; index++) { + first_values[index] = ReadU32(OffsetOperand(inst.src[0], index)); + second_values[index] = ReadU32(OffsetOperand(inst.src[2], index)); + } + auto first = inst; + first.memory.data_dwords = width; + first.memory.component_count = width; + write_u32(width, address, first_values, first); + if (inst.memory.secondary_offset != inst.memory.offset) { + auto second = first; + second.memory.offset = inst.memory.secondary_offset; + write_u32(width, address, second_values, second); } - ir.Emit(opcode, {resource, address, value, ir.GetExec()}, AddMemoryInfo(inst)); return true; } case IR::Opcode::DsMinF32: case IR::Opcode::DsMaxF32: { - const auto opcode = - gds ? (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::GdsAtomicFMin32 - : IR::ValueOpcode::GdsAtomicFMax32) - : (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::SharedAtomicFMin32 - : IR::ValueOpcode::SharedAtomicFMax32); + const auto opcode = inst.op == IR::Opcode::DsMinF32 + ? IR::ValueOpcode::SharedAtomicFMin32 + : IR::ValueOpcode::SharedAtomicFMax32; ir.Emit(opcode, - {resource, shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]), - ir.GetExec()}, + {shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]), ir.GetExec()}, AddMemoryInfo(inst)); return true; } case IR::Opcode::DsAppend: case IR::Opcode::DsConsume: { const bool append = inst.op == IR::Opcode::DsAppend; - const auto opcode = - gds ? (append ? IR::ValueOpcode::GdsDataAppend : IR::ValueOpcode::GdsDataConsume) - : (append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume); + const auto opcode = append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume; WriteOperand(inst.dst, ir.Emit(opcode, - {resource, ReadU32(inst.src[0]), ir.GetExec(), - ir.GetExecLo(), ir.GetExecHi()}, + {ReadU32(inst.src[0]), ir.GetExec(), ir.GetExecLo(), + ir.GetExecHi()}, AddMemoryInfo(inst))); return true; } @@ -601,16 +662,13 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) { const auto base = ir.BitwiseAnd(ReadU32(inst.src[m0_index]), IR::U32(IR::Value(0xffffu))); const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); - const auto address = ir.IAdd(ir.IAdd(base, IR::U32(IR::Value(inst.memory.offset))), - ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u)))); + const auto address = ir.IAdd(base, ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u)))); if (inst.op == IR::Opcode::DsWriteAddtidB32) { ir.Emit(IR::ValueOpcode::WriteSharedU32, - {resource, address, ReadU32(inst.src[0]), ir.GetExec()}, - AddMemoryInfo(inst)); + {address, ReadU32(inst.src[0]), ir.GetExec()}, AddMemoryInfo(inst)); } else { - WriteOperand(inst.dst, - ir.Emit(IR::ValueOpcode::LoadSharedU32, - {resource, address, ir.GetExec()}, AddMemoryInfo(inst))); + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::LoadSharedU32, + {address, ir.GetExec()}, AddMemoryInfo(inst))); } return true; } @@ -676,9 +734,11 @@ bool Translator::TranslateMemoryOperation(const IR::Instruction& inst, case IR::Opcode::DsReadUshort: case IR::Opcode::DsReadSshort: case IR::Opcode::DsReadB32: + case IR::Opcode::DsRead2B32: case IR::Opcode::DsWriteByte: case IR::Opcode::DsWriteShort: case IR::Opcode::DsWriteB32: + case IR::Opcode::DsWrite2B32: case IR::Opcode::DsMinF32: case IR::Opcode::DsMaxF32: case IR::Opcode::DsSwizzleB32: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 670078bd4..6c0510b81 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -768,6 +768,14 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u)); } else if (inst.op == IR::Opcode::BufferStoreDword) { include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u)); + } else if (inst.op == IR::Opcode::DsReadB32 || inst.op == IR::Opcode::DsRead2B32) { + include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u)); + } else if (inst.op == IR::Opcode::DsWriteB32) { + include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u)); + } else if (inst.op == IR::Opcode::DsWrite2B32) { + const auto width = std::max(inst.memory.data_dwords / 2u, 1u); + include_vector(inst.src[0], width); + include_vector(inst.src[2], width); } else if (inst.op == IR::Opcode::LoadInputF32) { include_vector(inst.dst, std::max(inst.input_info.component_count, 1u)); } diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index fd0400a00..5cb43361e 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -75,7 +75,6 @@ private: IR::Value GetScalarAddressResource(uint32_t base); IR::Value GetImageResource(const IR::MemoryInfo& memory); IR::Value GetSamplerResource(const IR::MemoryInfo& memory); - IR::Value GetSharedResource(IR::ResourceKind kind); IR::Value MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base); IR::Value ConstructU32x4(const IR::Operand& base, uint32_t count); void WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask, @@ -85,7 +84,7 @@ private: IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign); IR::Value NarrowSubdword(IR::U32 value, uint32_t bits); IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode); - IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode, bool gds); + IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode); bool IsScalarMemoryLoadOperation(IR::Opcode opcode); ScalarMemorySourceValues ReadScalarMemorySource(const IR::Instruction& inst); bool TranslateScalarMemory(const IR::Instruction& inst, diff --git a/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp b/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp index 0c5997c57..852675c1e 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp @@ -168,42 +168,33 @@ ResourceKind DsMemoryKind(const Decoder::Instruction& decoded) { } bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - const auto ir_op = DsReadOpcode(decoded); - const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; - for (uint32_t i = 0; i < count; i++) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = ir_op; - inst.src_count = 1; - inst.memory = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i); - if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); + const auto ir_op = DsReadOpcode(decoded); + Instruction inst; + inst.pc = decoded.pc; + inst.op = ir_op; + inst.src_count = 1; + inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); + if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || + !LowerSourceOperand(decoded.src0, inst.src[0], error)) { + return false; } + block.instructions.push_back(inst); return true; } bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock& block, uint32_t dwords_per_read, std::string* error) { - const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset}; - for (uint32_t read = 0; read < 2u; read++) { - for (uint32_t dword = 0; dword < dwords_per_read; dword++) { - const auto index = read * dwords_per_read + dword; - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::DsReadB32; - inst.src_count = 1; - inst.memory = - ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[read] + dword * 4u); - if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, index), inst.dst, error) || - !LowerSourceOperand(decoded.src0, inst.src[0], error)) { - return false; - } - block.instructions.push_back(inst); - } + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::DsRead2B32; + inst.src_count = 1; + inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); + inst.memory.data_dwords = dwords_per_read * 2u; + if (!LowerRegisterOperand(decoded.dst, inst.dst, error) || + !LowerSourceOperand(decoded.src0, inst.src[0], error)) { + return false; } + block.instructions.push_back(inst); return true; } @@ -408,45 +399,36 @@ bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock& block, std: } bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { - const auto ir_op = DsWriteOpcode(decoded.data_bits); - const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u; - for (uint32_t i = 0; i < count; i++) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = ir_op; - inst.src_count = 2; - inst.memory = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i); - inst.dst.kind = OperandKind::Null; - if (!LowerSourceOperand(OffsetDecodedRegister(decoded.src1, i), inst.src[0], error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); + const auto ir_op = DsWriteOpcode(decoded.data_bits); + Instruction inst; + inst.pc = decoded.pc; + inst.op = ir_op; + inst.src_count = 2; + inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); + inst.dst.kind = OperandKind::Null; + if (!LowerSourceOperand(decoded.src1, inst.src[0], error) || + !LowerSourceOperand(decoded.src0, inst.src[1], error)) { + return false; } + block.instructions.push_back(inst); return true; } bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock& block, uint32_t dwords_per_write, std::string* error) { - const Decoder::Operand* data[] = {&decoded.src1, &decoded.src2}; - const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset}; - for (uint32_t write = 0; write < 2u; write++) { - for (uint32_t dword = 0; dword < dwords_per_write; dword++) { - Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::DsWriteB32; - inst.src_count = 2; - inst.memory = - ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[write] + dword * 4u); - inst.dst.kind = OperandKind::Null; - if (!LowerSourceOperand(OffsetDecodedRegister(*data[write], dword), inst.src[0], - error) || - !LowerSourceOperand(decoded.src0, inst.src[1], error)) { - return false; - } - block.instructions.push_back(inst); - } + Instruction inst; + inst.pc = decoded.pc; + inst.op = Opcode::DsWrite2B32; + inst.src_count = 3; + inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded)); + inst.memory.data_dwords = dwords_per_write * 2u; + inst.dst.kind = OperandKind::Null; + if (!LowerSourceOperand(decoded.src1, inst.src[0], error) || + !LowerSourceOperand(decoded.src0, inst.src[1], error) || + !LowerSourceOperand(decoded.src2, inst.src[2], error)) { + return false; } + block.instructions.push_back(inst); return true; } @@ -618,7 +600,8 @@ bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& blo return LowerDsWriteAddtidB32(decoded, block, error); case Decoder::Opcode::DS_READ_ADDTID_B32: return LowerDsReadAddtidB32(decoded, block, error); - case Decoder::Opcode::DS_READ2_B32: return LowerDsRead2(decoded, block, 1, error); + case Decoder::Opcode::DS_READ2_B32: + case Decoder::Opcode::DS_READ2ST64_B32: return LowerDsRead2(decoded, block, 1, error); case Decoder::Opcode::DS_READ2_B64: case Decoder::Opcode::DS_READ2ST64_B64: return LowerDsRead2(decoded, block, 2, error); case Decoder::Opcode::DS_READ_I8: @@ -765,6 +748,7 @@ bool IsMemoryOpcode(Decoder::Opcode opcode) { case Decoder::Opcode::DS_READ_I16: case Decoder::Opcode::DS_READ_U16: case Decoder::Opcode::DS_READ2_B32: + case Decoder::Opcode::DS_READ2ST64_B32: case Decoder::Opcode::DS_READ_B32: case Decoder::Opcode::DS_READ_B64: case Decoder::Opcode::DS_READ2_B64: diff --git a/src/graphics/shader/recompiler/ir/Type.cpp b/src/graphics/shader/recompiler/ir/Type.cpp index 52f25126b..ac7e14a2a 100644 --- a/src/graphics/shader/recompiler/ir/Type.cpp +++ b/src/graphics/shader/recompiler/ir/Type.cpp @@ -31,8 +31,6 @@ std::string TypeName(Type type) { "AddressResource", "ImageResource", "SamplerResource", - "LdsResource", - "GdsResource", "ImageAddress", }; const auto bits = static_cast(type); diff --git a/src/graphics/shader/recompiler/ir/Type.h b/src/graphics/shader/recompiler/ir/Type.h index 6195ca89b..055b2697a 100644 --- a/src/graphics/shader/recompiler/ir/Type.h +++ b/src/graphics/shader/recompiler/ir/Type.h @@ -28,9 +28,7 @@ enum class Type : uint32_t { AddressResource = 1u << 18u, ImageResource = 1u << 19u, SamplerResource = 1u << 20u, - LdsResource = 1u << 21u, - GdsResource = 1u << 22u, - ImageAddress = 1u << 23u, + ImageAddress = 1u << 21u, }; constexpr Type operator|(Type lhs, Type rhs) { diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.cpp b/src/graphics/shader/recompiler/ir/ValueProgram.cpp index 4ec2736ff..2f7ec7d65 100644 --- a/src/graphics/shader/recompiler/ir/ValueProgram.cpp +++ b/src/graphics/shader/recompiler/ir/ValueProgram.cpp @@ -338,6 +338,34 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st ValueOpcodeName(inst.GetOpcode()))); } } + const auto shared_components = SharedComponentCount(inst.GetOpcode()); + if (shared_components != 0u) { + const auto memory_index = inst.Flags().index; + if (memory_index >= program.memory_info.size()) { + return Fail(error, fmt::format("{} has an invalid memory-info index", + ValueOpcodeName(inst.GetOpcode()))); + } + const auto& memory = program.memory_info[memory_index]; + if ((memory.kind != ResourceKind::Lds && memory.kind != ResourceKind::Gds) || + memory.resource != 0u || memory.sampler != 0u || memory.component_count == 0u || + memory.component_index >= memory.component_count) { + return Fail(error, fmt::format("{} has invalid shared-memory metadata", + ValueOpcodeName(inst.GetOpcode()))); + } + uint32_t expected_bits = 32u; + if (inst.GetOpcode() == ValueOpcode::LoadSharedU8 || + inst.GetOpcode() == ValueOpcode::WriteSharedU8) { + expected_bits = 8u; + } else if (inst.GetOpcode() == ValueOpcode::LoadSharedU16 || + inst.GetOpcode() == ValueOpcode::WriteSharedU16) { + expected_bits = 16u; + } + if (memory.data_bits != expected_bits || memory.data_dwords != shared_components || + (shared_components > 1u && memory.component_index != 0u)) { + return Fail(error, fmt::format("{} has inconsistent shared-memory width", + ValueOpcodeName(inst.GetOpcode()))); + } + } uint32_t composite_components = 0u; switch (inst.GetOpcode()) { case ValueOpcode::CompositeExtractU32x2: composite_components = 2u; break; @@ -358,10 +386,11 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st ValueOpcodeName(inst.GetOpcode()))); } if (fixed_signature && arg.GetType() != ArgTypeOf(inst.GetOpcode(), arg_index)) { - return Fail(error, fmt::format("{} argument {} has type {}, expected {}", - ValueOpcodeName(inst.GetOpcode()), arg_index, - TypeName(arg.GetType()), - TypeName(ArgTypeOf(inst.GetOpcode(), arg_index)))); + return Fail(error, + fmt::format("{} argument {} has type {}, expected {}", + ValueOpcodeName(inst.GetOpcode()), arg_index, + TypeName(arg.GetType()), + TypeName(ArgTypeOf(inst.GetOpcode(), arg_index)))); } if (const auto* definition = arg.TryInstruction(); definition != nullptr && !instructions.contains(definition)) { diff --git a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp index b3623b8fb..6798e8bb5 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp @@ -392,19 +392,20 @@ constexpr LowerMap LOWER_OPS[] = { {Decoder::Opcode::DS_READ_U8, Opcode::DsReadUbyte}, {Decoder::Opcode::DS_READ_I16, Opcode::DsReadSshort}, {Decoder::Opcode::DS_READ_U16, Opcode::DsReadUshort}, - {Decoder::Opcode::DS_READ2_B32, Opcode::DsReadB32}, + {Decoder::Opcode::DS_READ2_B32, Opcode::DsRead2B32}, + {Decoder::Opcode::DS_READ2ST64_B32, Opcode::DsRead2B32}, {Decoder::Opcode::DS_READ_B32, Opcode::DsReadB32}, {Decoder::Opcode::DS_READ_B64, Opcode::DsReadB32}, - {Decoder::Opcode::DS_READ2_B64, Opcode::DsReadB32}, - {Decoder::Opcode::DS_READ2ST64_B64, Opcode::DsReadB32}, + {Decoder::Opcode::DS_READ2_B64, Opcode::DsRead2B32}, + {Decoder::Opcode::DS_READ2ST64_B64, Opcode::DsRead2B32}, {Decoder::Opcode::DS_READ_B96, Opcode::DsReadB32}, {Decoder::Opcode::DS_READ_B128, Opcode::DsReadB32}, {Decoder::Opcode::DS_WRITE_B8, Opcode::DsWriteByte}, {Decoder::Opcode::DS_WRITE_B16, Opcode::DsWriteShort}, - {Decoder::Opcode::DS_WRITE2_B32, Opcode::DsWriteB32}, - {Decoder::Opcode::DS_WRITE2ST64_B32, Opcode::DsWriteB32}, - {Decoder::Opcode::DS_WRITE2_B64, Opcode::DsWriteB32}, - {Decoder::Opcode::DS_WRITE2ST64_B64, Opcode::DsWriteB32}, + {Decoder::Opcode::DS_WRITE2_B32, Opcode::DsWrite2B32}, + {Decoder::Opcode::DS_WRITE2ST64_B32, Opcode::DsWrite2B32}, + {Decoder::Opcode::DS_WRITE2_B64, Opcode::DsWrite2B32}, + {Decoder::Opcode::DS_WRITE2ST64_B64, Opcode::DsWrite2B32}, {Decoder::Opcode::DS_WRITE_B32, Opcode::DsWriteB32}, {Decoder::Opcode::DS_WRITE_B64, Opcode::DsWriteB32}, {Decoder::Opcode::DS_WRITE_B96, Opcode::DsWriteB32}, diff --git a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc index 5738c4e77..bba995922 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc @@ -331,9 +331,11 @@ IR_OPCODE(DsReadSbyte, General, Memory) IR_OPCODE(DsReadUshort, General, Memory) IR_OPCODE(DsReadSshort, General, Memory) IR_OPCODE(DsReadB32, General, Memory) +IR_OPCODE(DsRead2B32, General, Memory) IR_OPCODE(DsWriteByte, General, Memory) IR_OPCODE(DsWriteShort, General, Memory) IR_OPCODE(DsWriteB32, General, Memory) +IR_OPCODE(DsWrite2B32, General, Memory) IR_OPCODE(DsMinF32, General, Memory) IR_OPCODE(DsMaxF32, General, Memory) IR_OPCODE(DsSwizzleB32, General, Memory) diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index 05ec0e5e7..86ae86adc 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -42,8 +42,6 @@ constexpr Type BufferResource = Type::BufferResource; constexpr Type AddressResource = Type::AddressResource; constexpr Type ImageResource = Type::ImageResource; constexpr Type SamplerResource = Type::SamplerResource; -constexpr Type LdsResource = Type::LdsResource; -constexpr Type GdsResource = Type::GdsResource; constexpr Type ImageAddress = Type::ImageAddress; constexpr std::array(ValueOpcode::Count)> MetaTable = {{ @@ -76,52 +74,23 @@ bool HasSideEffects(ValueOpcode opcode) { if (buffer_access == BufferAccess::Write || buffer_access == BufferAccess::Atomic) { return true; } + const auto shared_access = SharedAccessOf(opcode); + if (shared_access == SharedAccess::Write || shared_access == SharedAccess::Atomic || + shared_access == SharedAccess::Append || shared_access == SharedAccess::Consume) { + return true; + } switch (opcode) { case ValueOpcode::Reference: case ValueOpcode::ReferenceU32: case ValueOpcode::StoreAddressU8: case ValueOpcode::StoreAddressU16: case ValueOpcode::StoreAddressU32: - case ValueOpcode::WriteSharedU8: - case ValueOpcode::WriteSharedU16: - case ValueOpcode::WriteSharedU32: - case ValueOpcode::WriteGdsU8: - case ValueOpcode::WriteGdsU16: - case ValueOpcode::WriteGdsU32: - case ValueOpcode::SharedAtomicFMin32: - case ValueOpcode::SharedAtomicFMax32: - case ValueOpcode::GdsAtomicFMin32: - case ValueOpcode::GdsAtomicFMax32: - case ValueOpcode::SharedAtomicSwap32: - case ValueOpcode::SharedAtomicIAdd32: - case ValueOpcode::SharedAtomicISub32: - case ValueOpcode::SharedAtomicSMin32: - case ValueOpcode::SharedAtomicUMin32: - case ValueOpcode::SharedAtomicSMax32: - case ValueOpcode::SharedAtomicUMax32: - case ValueOpcode::SharedAtomicAnd32: - case ValueOpcode::SharedAtomicOr32: - case ValueOpcode::SharedAtomicXor32: - case ValueOpcode::GdsAtomicSwap32: - case ValueOpcode::GdsAtomicIAdd32: - case ValueOpcode::GdsAtomicISub32: - case ValueOpcode::GdsAtomicSMin32: - case ValueOpcode::GdsAtomicUMin32: - case ValueOpcode::GdsAtomicSMax32: - case ValueOpcode::GdsAtomicUMax32: - case ValueOpcode::GdsAtomicAnd32: - case ValueOpcode::GdsAtomicOr32: - case ValueOpcode::GdsAtomicXor32: case ValueOpcode::ImageAtomicIAdd32: case ValueOpcode::ImageAtomicUMin32: case ValueOpcode::ImageAtomicUMax32: case ValueOpcode::ImageAtomicAnd32: case ValueOpcode::ImageAtomicOr32: case ValueOpcode::ImageAtomicXor32: - case ValueOpcode::DataAppend: - case ValueOpcode::DataConsume: - case ValueOpcode::GdsDataAppend: - case ValueOpcode::GdsDataConsume: case ValueOpcode::ImageWrite: case ValueOpcode::SetAttribute: case ValueOpcode::Barrier: return true; @@ -172,4 +141,48 @@ uint32_t BufferComponentCount(ValueOpcode opcode) { } } +SharedAccess SharedAccessOf(ValueOpcode opcode) { + switch (opcode) { + case ValueOpcode::LoadSharedU8: + case ValueOpcode::LoadSharedU16: + case ValueOpcode::LoadSharedU32: + case ValueOpcode::LoadSharedU32x2: + case ValueOpcode::LoadSharedU32x3: + case ValueOpcode::LoadSharedU32x4: return SharedAccess::Read; + case ValueOpcode::WriteSharedU8: + case ValueOpcode::WriteSharedU16: + case ValueOpcode::WriteSharedU32: + case ValueOpcode::WriteSharedU32x2: + case ValueOpcode::WriteSharedU32x3: + case ValueOpcode::WriteSharedU32x4: return SharedAccess::Write; + case ValueOpcode::SharedAtomicFMin32: + case ValueOpcode::SharedAtomicFMax32: + case ValueOpcode::SharedAtomicSwap32: + case ValueOpcode::SharedAtomicIAdd32: + case ValueOpcode::SharedAtomicISub32: + case ValueOpcode::SharedAtomicSMin32: + case ValueOpcode::SharedAtomicUMin32: + case ValueOpcode::SharedAtomicSMax32: + case ValueOpcode::SharedAtomicUMax32: + case ValueOpcode::SharedAtomicAnd32: + case ValueOpcode::SharedAtomicOr32: + case ValueOpcode::SharedAtomicXor32: return SharedAccess::Atomic; + case ValueOpcode::DataAppend: return SharedAccess::Append; + case ValueOpcode::DataConsume: return SharedAccess::Consume; + default: return SharedAccess::None; + } +} + +uint32_t SharedComponentCount(ValueOpcode opcode) { + switch (opcode) { + case ValueOpcode::LoadSharedU32x2: + case ValueOpcode::WriteSharedU32x2: return 2u; + case ValueOpcode::LoadSharedU32x3: + case ValueOpcode::WriteSharedU32x3: return 3u; + case ValueOpcode::LoadSharedU32x4: + case ValueOpcode::WriteSharedU32x4: return 4u; + default: return SharedAccessOf(opcode) == SharedAccess::None ? 0u : 1u; + } +} + } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h index 135afa50d..5636e0d1c 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h @@ -17,6 +17,7 @@ enum class ValueOpcode { }; enum class BufferAccess { None, Read, Write, Atomic }; +enum class SharedAccess { None, Read, Write, Atomic, Append, Consume }; struct DppMoveFlags { uint16_t control = 0; @@ -56,6 +57,8 @@ static_assert(std::is_trivially_copyable_v); [[nodiscard]] bool HasSideEffects(ValueOpcode opcode); [[nodiscard]] BufferAccess BufferAccessOf(ValueOpcode opcode); [[nodiscard]] uint32_t BufferComponentCount(ValueOpcode opcode); +[[nodiscard]] SharedAccess SharedAccessOf(ValueOpcode opcode); +[[nodiscard]] uint32_t SharedComponentCount(ValueOpcode opcode); [[nodiscard]] std::string_view ValueOpcodeName(ValueOpcode opcode); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index e3cd5b6fb..a9269951f 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -202,8 +202,6 @@ VALUE_OPCODE(GetBufferResource, BufferResource, U32, U32, U32, U32) VALUE_OPCODE(GetAddressResource, AddressResource, U32, U32) VALUE_OPCODE(GetImageResource, ImageResource, U32, U32, U32, U32, U32, U32, U32, U32) VALUE_OPCODE(GetSamplerResource, SamplerResource, U32, U32, U32, U32) -VALUE_OPCODE(GetLdsResource, LdsResource) -VALUE_OPCODE(GetGdsResource, GdsResource) VALUE_OPCODE(MakeImageAddress, ImageAddress, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32) @@ -240,46 +238,32 @@ VALUE_OPCODE(BufferAtomicXor32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicFMin32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicFMax32, U32, BufferResource, U32, U32, U32, U32, U1) -VALUE_OPCODE(LoadSharedU8, U8, LdsResource, U32, U1) -VALUE_OPCODE(LoadSharedU16, U16, LdsResource, U32, U1) -VALUE_OPCODE(LoadSharedU32, U32, LdsResource, U32, U1) -VALUE_OPCODE(WriteSharedU8, Void, LdsResource, U32, U8, U1) -VALUE_OPCODE(WriteSharedU16, Void, LdsResource, U32, U16, U1) -VALUE_OPCODE(WriteSharedU32, Void, LdsResource, U32, U32, U1) -VALUE_OPCODE(LoadGdsU8, U8, GdsResource, U32, U1) -VALUE_OPCODE(LoadGdsU16, U16, GdsResource, U32, U1) -VALUE_OPCODE(LoadGdsU32, U32, GdsResource, U32, U1) -VALUE_OPCODE(WriteGdsU8, Void, GdsResource, U32, U8, U1) -VALUE_OPCODE(WriteGdsU16, Void, GdsResource, U32, U16, U1) -VALUE_OPCODE(WriteGdsU32, Void, GdsResource, U32, U32, U1) -VALUE_OPCODE(SharedAtomicFMin32, Void, LdsResource, U32, U32, U32, U1) -VALUE_OPCODE(SharedAtomicFMax32, Void, LdsResource, U32, U32, U32, U1) -VALUE_OPCODE(GdsAtomicFMin32, Void, GdsResource, U32, U32, U32, U1) -VALUE_OPCODE(GdsAtomicFMax32, Void, GdsResource, U32, U32, U32, U1) -VALUE_OPCODE(SharedAtomicSwap32, U32, LdsResource, U32, U32, U1) -VALUE_OPCODE(SharedAtomicIAdd32, U32, LdsResource, U32, U32, U1) -VALUE_OPCODE(SharedAtomicISub32, U32, LdsResource, U32, U32, U1) -VALUE_OPCODE(SharedAtomicSMin32, U32, LdsResource, U32, U32, U1) -VALUE_OPCODE(SharedAtomicUMin32, U32, LdsResource, U32, U32, U1) -VALUE_OPCODE(SharedAtomicSMax32, U32, LdsResource, U32, U32, U1) -VALUE_OPCODE(SharedAtomicUMax32, U32, LdsResource, U32, U32, U1) -VALUE_OPCODE(SharedAtomicAnd32, U32, LdsResource, U32, U32, U1) -VALUE_OPCODE(SharedAtomicOr32, U32, LdsResource, U32, U32, U1) -VALUE_OPCODE(SharedAtomicXor32, U32, LdsResource, U32, U32, U1) -VALUE_OPCODE(GdsAtomicSwap32, U32, GdsResource, U32, U32, U1) -VALUE_OPCODE(GdsAtomicIAdd32, U32, GdsResource, U32, U32, U1) -VALUE_OPCODE(GdsAtomicISub32, U32, GdsResource, U32, U32, U1) -VALUE_OPCODE(GdsAtomicSMin32, U32, GdsResource, U32, U32, U1) -VALUE_OPCODE(GdsAtomicUMin32, U32, GdsResource, U32, U32, U1) -VALUE_OPCODE(GdsAtomicSMax32, U32, GdsResource, U32, U32, U1) -VALUE_OPCODE(GdsAtomicUMax32, U32, GdsResource, U32, U32, U1) -VALUE_OPCODE(GdsAtomicAnd32, U32, GdsResource, U32, U32, U1) -VALUE_OPCODE(GdsAtomicOr32, U32, GdsResource, U32, U32, U1) -VALUE_OPCODE(GdsAtomicXor32, U32, GdsResource, U32, U32, U1) -VALUE_OPCODE(DataAppend, U32, LdsResource, U32, U1, U32, U32) -VALUE_OPCODE(DataConsume, U32, LdsResource, U32, U1, U32, U32) -VALUE_OPCODE(GdsDataAppend, U32, GdsResource, U32, U1, U32, U32) -VALUE_OPCODE(GdsDataConsume, U32, GdsResource, U32, U1, U32, U32) +VALUE_OPCODE(LoadSharedU8, U8, U32, U1) +VALUE_OPCODE(LoadSharedU16, U16, U32, U1) +VALUE_OPCODE(LoadSharedU32, U32, U32, U1) +VALUE_OPCODE(LoadSharedU32x2, U32x2, U32, U1) +VALUE_OPCODE(LoadSharedU32x3, U32x3, U32, U1) +VALUE_OPCODE(LoadSharedU32x4, U32x4, U32, U1) +VALUE_OPCODE(WriteSharedU8, Void, U32, U8, U1) +VALUE_OPCODE(WriteSharedU16, Void, U32, U16, U1) +VALUE_OPCODE(WriteSharedU32, Void, U32, U32, U1) +VALUE_OPCODE(WriteSharedU32x2, Void, U32, U32, U32, U1) +VALUE_OPCODE(WriteSharedU32x3, Void, U32, U32, U32, U32, U1) +VALUE_OPCODE(WriteSharedU32x4, Void, U32, U32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicISub32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicSMin32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicUMin32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicSMax32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1) +VALUE_OPCODE(DataAppend, U32, U32, U1, U32, U32) +VALUE_OPCODE(DataConsume, U32, U32, U1, U32, U32) VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1) VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress) diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp index 10e04184f..7cb41bf6a 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp @@ -115,20 +115,25 @@ void AddBinding(BindingLayout& layout, DescriptorBindingKind kind, {kind, static_cast(layout.descriptors.size()), std::move(resources)}); } -bool UsesGds(const Program& program) { +bool UsesGds(const Program& program, bool& uses_gds) { + uses_gds = false; for (const auto* block: program.values->blocks) { for (const auto& inst: *block) { - if (inst.GetOpcode() == ValueOpcode::GetGdsResource && inst.HasUses()) { - return true; + if (SharedAccessOf(inst.GetOpcode()) == SharedAccess::None) { + continue; } - for (size_t index = 0; index < inst.NumArgs(); index++) { - if (inst.Arg(index).GetType() == Type::GdsResource) { - return true; - } + const auto index = inst.Flags().index; + if (index >= program.values->memory_info.size()) { + return false; } + const auto kind = program.values->memory_info[index].kind; + if (kind != ResourceKind::Lds && kind != ResourceKind::Gds) { + return false; + } + uses_gds |= kind == ResourceKind::Gds; } } - return false; + return true; } } // namespace @@ -220,7 +225,14 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std } AddBinding(next, DescriptorBindingKind::Samplers, std::move(resources)); } - if (UsesGds(program)) { + bool uses_gds = false; + if (!UsesGds(program, uses_gds)) { + if (error != nullptr) { + *error = "typed shader contains invalid shared-memory metadata"; + } + return false; + } + if (uses_gds) { AddBinding(next, DescriptorBindingKind::Gds); } if (!program.info.addresses.empty()) { diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index cac430c92..e4fa1ed1f 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -256,8 +256,8 @@ void TestInvariantIndirectImageMaterialization() { fixture->program.values->descriptor_sources[source] .indirect_image.has_value(), "indirect image source was not retained for runtime proof"); - const auto image_handle = std::ranges::find_if( - *fixture->block, [](const Inst &inst) { + const auto image_handle = + std::ranges::find_if(*fixture->block, [](const Inst &inst) { return inst.GetOpcode() == ValueOpcode::GetImageResource; }); Check(image_handle != fixture->block->end() && @@ -326,11 +326,12 @@ void TestInvariantIndirectImageMaterialization() { memory.words[(0x2020u - memory.base) / 4u + 3u] = image_descriptor[3] ^ (1u << 28u); ResourceSnapshot null_snapshot; - Check(MaterializeResources(fixture->program, runtime, null_snapshot, &error) && - null_snapshot.indirect_images.empty() && - std::ranges::all_of(null_snapshot.images[0].dwords, - [](uint32_t dword) { return dword == 0u; }), - "stale typed null image descriptors were not canonicalized"); + Check( + MaterializeResources(fixture->program, runtime, null_snapshot, &error) && + null_snapshot.indirect_images.empty() && + std::ranges::all_of(null_snapshot.images[0].dwords, + [](uint32_t dword) { return dword == 0u; }), + "stale typed null image descriptors were not canonicalized"); for (uint32_t dword = 0; dword < image_descriptor.size(); dword++) { memory.words[(0x2000u - memory.base) / 4u + dword] = @@ -380,11 +381,12 @@ void TestInvariantIndirectImageMaterialization() { ResourceSnapshot capacity_snapshot; for (const uint32_t records : {1u, 3u}) { user_data[2] = records; - Check(MaterializeResources(fixture->program, runtime, capacity_snapshot, - &error) && - ValidateResourceSpecialization(fixture->program, - capacity_snapshot, &error), - "runtime indirect key mapping rejected a fitting material-table size"); + Check( + MaterializeResources(fixture->program, runtime, capacity_snapshot, + &error) && + ValidateResourceSpecialization(fixture->program, capacity_snapshot, + &error), + "runtime indirect key mapping rejected a fitting material-table size"); } user_data[2] = 2u; memory.words[(0x2020u - memory.base) / 4u] = @@ -396,11 +398,12 @@ void TestInvariantIndirectImageMaterialization() { image_descriptor[dword]; } memory.words[(0x1000u - memory.base + 68u) / 4u] = 2u; - Check(!MaterializeResources(fixture->program, runtime, rebound_snapshot, - &error) && - error.find("candidate topology") != std::string::npos && - same_snapshot(rebound_snapshot, collapsed_snapshot), - "larger indirect candidate topology reused or mutated a cached snapshot"); + Check( + !MaterializeResources(fixture->program, runtime, rebound_snapshot, + &error) && + error.find("candidate topology") != std::string::npos && + same_snapshot(rebound_snapshot, collapsed_snapshot), + "larger indirect candidate topology reused or mutated a cached snapshot"); auto memory_backed = MakeIndirectImageFixture(false, 0u, true); memory_backed->PlanAndTrack(); @@ -985,14 +988,14 @@ void TestExecMaskedFlatAddressProvenance() { Fixture fixture; const auto low_root = fixture.UserData(0); const auto high_root = fixture.UserData(1); - const auto active = fixture.Emit( - ValueOpcode::INotEqual32, {fixture.UserData(2), Value(0u)}); + const auto active = + fixture.Emit(ValueOpcode::INotEqual32, {fixture.UserData(2), Value(0u)}); const auto inactive_low = fixture.Emit(ValueOpcode::UndefU32); const auto inactive_high = fixture.Emit(ValueOpcode::UndefU32); - const auto low = fixture.Emit(ValueOpcode::SelectU32, - {active, low_root, inactive_low}); - const auto high = fixture.Emit(ValueOpcode::SelectU32, - {active, high_root, inactive_high}); + const auto low = + fixture.Emit(ValueOpcode::SelectU32, {active, low_root, inactive_low}); + const auto high = + fixture.Emit(ValueOpcode::SelectU32, {active, high_root, inactive_high}); const auto address = fixture.Address(low, high, 0xa4); MemoryInfo flat; flat.kind = ResourceKind::Flat; @@ -1015,19 +1018,18 @@ void TestExecMaskedFlatAddressProvenance() { "exec-masked FLAT address materialized the wrong user-data root"); Fixture mismatch; - const auto mismatch_active = mismatch.Emit( - ValueOpcode::INotEqual32, {mismatch.UserData(2), Value(0u)}); - const auto other_active = mismatch.Emit(ValueOpcode::LogicalNot, - {mismatch_active}); + const auto mismatch_active = mismatch.Emit(ValueOpcode::INotEqual32, + {mismatch.UserData(2), Value(0u)}); + const auto other_active = + mismatch.Emit(ValueOpcode::LogicalNot, {mismatch_active}); const auto mismatch_low = mismatch.Emit( - ValueOpcode::SelectU32, - {mismatch_active, mismatch.UserData(0), - mismatch.Emit(ValueOpcode::UndefU32)}); + ValueOpcode::SelectU32, {mismatch_active, mismatch.UserData(0), + mismatch.Emit(ValueOpcode::UndefU32)}); const auto mismatch_high = mismatch.Emit( - ValueOpcode::SelectU32, - {mismatch_active, mismatch.UserData(1), - mismatch.Emit(ValueOpcode::UndefU32)}); - const auto mismatch_address = mismatch.Address(mismatch_low, mismatch_high, 0xa4); + ValueOpcode::SelectU32, {mismatch_active, mismatch.UserData(1), + mismatch.Emit(ValueOpcode::UndefU32)}); + const auto mismatch_address = + mismatch.Address(mismatch_low, mismatch_high, 0xa4); mismatch.Emit(ValueOpcode::LoadAddressU8, {mismatch_address, mismatch_low, mismatch_high, other_active}, mismatch.AddMemory(flat, 0xa4)); @@ -1086,9 +1088,10 @@ void TestShaderInfoAndBindingLayout() { {Value(static_cast(StageInputKind::GlobalInvocationId)), Value(2u)}); fixture.Emit(ValueOpcode::BitwiseXor32, {Value(1u), Value(2u)}); - const auto gds = fixture.Emit(ValueOpcode::GetGdsResource); - fixture.Emit(ValueOpcode::WriteGdsU32, - {gds, Value(0u), Value(1u), Value(true)}); + MemoryInfo gds; + gds.kind = ResourceKind::Gds; + fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), Value(1u), Value(true)}, + fixture.AddMemory(gds, 8)); fixture.PlanAndTrack(); ShaderComputeInputInfo compute{}; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 16942e86c..add8843e3 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -974,7 +974,7 @@ void CheckRectListShaders() { shaders.control.size() > 1 && shaders.control[1] == 0x00010500u && shaders.evaluation.size() > 1 && shaders.evaluation[1] == 0x00010500u, - "shadPS4-compatible vector selection requires SPIR-V 1.5"); + "shadPS4-compatible vector selection requires SPIR-V 1.5"); ValidateSpirv(name, shaders.control); ValidateSpirv(name, shaders.evaluation); @@ -1127,8 +1127,8 @@ CompiledShader CompileCase(const TestCase &test) { image != result.program.info.images.end() && image->mip_count == test.expected_storage_mip_descriptors, "runtime descriptor range did not specialize the mip count"); - const auto resource = static_cast( - image - result.program.info.images.begin()); + const auto resource = + static_cast(image - result.program.info.images.begin()); const auto binding = std::ranges::find_if( result.program.bindings.descriptors, [&](const auto &group) { return std::ranges::find(group.resources, resource) != @@ -1156,8 +1156,9 @@ CompiledShader CompileCase(const TestCase &test) { shader_data != nullptr, "offset-only shader data did not use its storage fallback"); std::vector storage_spirv; - if (!ShaderRecompiler::Spirv::EmitProgram( - result.program, result.resources, options.input_info, storage_spirv, &error)) { + if (!ShaderRecompiler::Spirv::EmitProgram(result.program, result.resources, + options.input_info, storage_spirv, + &error)) { Fail(test.name, "SPIR-V emit", error.c_str()); } result.spirv = std::move(storage_spirv); @@ -2209,7 +2210,8 @@ public: sizeof(source_words)); append_dma(0, memory_copy_dst, 0, memory_src, sizeof(source_words)); append_dma(3, l2_copy_dst, 3, memory_src, sizeof(source_words)); - append_dma(2, nowhere_dst, 3, nowhere_dst, sizeof(nowhere_words), 1u << 31u); + append_dma(2, nowhere_dst, 3, nowhere_dst, sizeof(nowhere_words), + 1u << 31u); Require("GpuCommandLane", "DMA_DATA packet assembly", dma_cursor == dma_commands.size(), "DMA_DATA packet stream has the wrong size"); @@ -2223,7 +2225,7 @@ public: buffer_cache.IsRegionRegistered(clean_cached_fill, sizeof(source_words)) && !buffer_cache.HasGpuDirtyBytes(nowhere_dst, - sizeof(nowhere_words)) && + sizeof(nowhere_words)) && buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(source_words)), "DMA prefetch modified ownership or did not establish a dirty " @@ -2611,14 +2613,15 @@ public: depth_only_info.format = depth_stencil_info.pixel_format; const auto depth_only = depth_stencil.FindView(depth_only_info); auto combined_info = depth_only_info; - combined_info.aspect = vk::ImageAspectFlagBits::eDepth | - vk::ImageAspectFlagBits::eStencil; + combined_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; combined_info.mapping = {}; combined_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; const auto combined = depth_stencil.FindView(combined_info); Require(name, "depth/stencil aspects", depth_only != nullptr && combined != nullptr && - depth_only != combined && depth_stencil.views.views.size() == 2 && + depth_only != combined && + depth_stencil.views.views.size() == 2 && depth_stencil.views.views[0].info.aspect == vk::ImageAspectFlagBits::eDepth && depth_stencil.views.views[1].info.aspect == @@ -3592,7 +3595,7 @@ public: }; // A formatted Buffer read must use the private - // shadPS4-shaped image-copy path. Use a request larger + // shadPS4-shaped image-copy path. Use a request larger // than the stream shortcut and poison guest backing // after upload so stale CPU staging cannot accidentally // satisfy the content check. @@ -4533,10 +4536,11 @@ public: texture_cache.FindImage(metadata_depth_a); const auto metadata_depth_b_id = texture_cache.FindImage(metadata_depth_b); - Require(name, "pending DCC/HTile separation", - !texture_cache.TryConsumeDccFill(base + metadata_a, 0x80, 0) && - !texture_cache.IsMeta(base + metadata_a), - "an unclassified PS5 DCC fill was exposed as registered metadata"); + Require( + name, "pending DCC/HTile separation", + !texture_cache.TryConsumeDccFill(base + metadata_a, 0x80, 0) && + !texture_cache.IsMeta(base + metadata_a), + "an unclassified PS5 DCC fill was exposed as registered metadata"); const auto metadata_depth_a_view = texture_cache.FindDepthTarget(metadata_depth_a_id, metadata_depth_a); const auto metadata_depth_b_view = @@ -4568,11 +4572,12 @@ public: texture_cache.IsMeta(base + metadata_b) && texture_cache.IsMetaCleared(base + metadata_b, 0), "image discovery retired aliased metadata or its live depth image"); - Require(name, "registered DCC/HTile separation", - texture_cache.TouchMeta(base + metadata_a, 0, false) && - !texture_cache.TryConsumeDccFill(base + metadata_a, 0x80, 0) && - !texture_cache.IsMetaCleared(base + metadata_a, 0), - "a PS5 DCC fill changed registered HTile state"); + Require( + name, "registered DCC/HTile separation", + texture_cache.TouchMeta(base + metadata_a, 0, false) && + !texture_cache.TryConsumeDccFill(base + metadata_a, 0x80, 0) && + !texture_cache.IsMetaCleared(base + metadata_a, 0), + "a PS5 DCC fill changed registered HTile state"); Require( name, "metadata first-touch state", texture_cache.TouchMeta(base + metadata_b, 0, true) && @@ -6356,17 +6361,15 @@ public: color.desc.info.metadata.kind == ImageMetadataKind::Dcc && color.desc.info.metadata.range.address == dcc_address && color.metadata_clear_supported && - texture_cache.IsMetaCleared(dcc_address, 7, - &dcc_clear_value) && - dcc_clear_value == 0 && - !texture_cache.ClearMeta(dcc_address) && + texture_cache.IsMetaCleared(dcc_address, 7, &dcc_clear_value) && + dcc_clear_value == 0 && !texture_cache.ClearMeta(dcc_address) && color.desc.view_info.type == vk::ImageViewType::e2D && color.desc.view_info.layer_count == 1 && image.backing.image_type == vk::ImageType::e3D && static_cast(image.backing.flags & vk::ImageCreateFlagBits::e2DArrayCompatible) && image.usage.render_target && image.IsGpuModified(), - "dimension=2/depth=31 did not create the SDK-defined 32x32x32 " + "dimension=2/depth=31 did not create the SDK-defined 32x32x32 " "backing and 2D " "attachment slice"); @@ -6389,18 +6392,18 @@ public: sliced_rendering.num_layers == 1 && sliced_rendering.color_attachments[0].is_clear && sliced_rendering.color_attachments[0].clear_value == - vk::ClearColorValue {}.uint32 && + vk::ClearColorValue{}.uint32 && !texture_cache.IsMetaCleared(dcc_address, 7), "a nonzero 3D attachment slice was treated as a Vulkan array layer"); dcc_clear_value = 0; - Require(name, "registered DCC code state", - texture_cache.TryConsumeDccFill(dcc_address, 0x1000, 0xffffffffu) && - !texture_cache.IsMetaCleared(dcc_address, 7, - &dcc_clear_value) && - dcc_clear_value == 0xffffffffu && - texture_cache.TryConsumeDccFill(dcc_address, 0x1000, 0) && - texture_cache.IsMetaCleared(dcc_address, 7), - "DCC fill codes were not classified as clear or uncompressed"); + Require( + name, "registered DCC code state", + texture_cache.TryConsumeDccFill(dcc_address, 0x1000, 0xffffffffu) && + !texture_cache.IsMetaCleared(dcc_address, 7, &dcc_clear_value) && + dcc_clear_value == 0xffffffffu && + texture_cache.TryConsumeDccFill(dcc_address, 0x1000, 0) && + texture_cache.IsMetaCleared(dcc_address, 7), + "DCC fill codes were not classified as clear or uncompressed"); auto storage_desc = color.desc; storage_desc.type = BindingType::Storage; @@ -6629,11 +6632,10 @@ public: .channel_type = Prospero::ChannelType::kFloat, .channel_order = Prospero::ChannelOrder::kStandard}); registers.SetColorAttrib2(0, {.height = 63, .width = 63}); - registers.SetColorAttrib3( - 0, {.tile_mode = Prospero::TileMode::kDepth, - .dimension = 1, - .cmask_pipe_aligned = true, - .dcc_pipe_aligned = true}); + registers.SetColorAttrib3(0, {.tile_mode = Prospero::TileMode::kDepth, + .dimension = 1, + .cmask_pipe_aligned = true, + .dcc_pipe_aligned = true}); registers.SetRenderTargetMask(0x0f); scheduler.Begin(registers, user_config, shaders); @@ -6660,20 +6662,21 @@ public: scheduler.DrainPriorityOperations(); std::vector cleared(allocation_size); Require(name, "clear guest backing", - Libs::LibKernel::Memory::TryWriteBacking( - base, cleared.data(), allocation_size), + Libs::LibKernel::Memory::TryWriteBacking(base, cleared.data(), + allocation_size), "depth-tiled color backing could not be cleared"); - Require(name, "readback queue", - TextureCacheTestAccess::TryDownload(texture_cache, color.image_id), - "depth-tiled color target could not be queued for readback"); + Require( + name, "readback queue", + TextureCacheTestAccess::TryDownload(texture_cache, color.image_id), + "depth-tiled color target could not be queued for readback"); RenderExecutorTestAccess::ResetBindings(executor); scheduler.Finish(); scheduler.DrainPriorityOperations(); std::vector observed(expected.size()); Require(name, "readback backing", - Libs::LibKernel::Memory::TryReadBacking( - base, observed.data(), allocation_size), + Libs::LibKernel::Memory::TryReadBacking(base, observed.data(), + allocation_size), "depth-tiled color readback is unavailable"); bool recovered = true; for (uint32_t y = 0; y < height; ++y) { @@ -7264,27 +7267,27 @@ public: writable_alias_bindings.vertex); RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), *writable_alias_bindings.pixel); - Require(name, "forced-general descriptor capture", - writable_alias_bindings.vertex.resources.images[0].layout == - vk::ImageLayout::eGeneral && - writable_alias_bindings.pixel->resources.images[0].layout == - vk::ImageLayout::eGeneral && - DescriptorCacheTestAccess::MakeImageInfo( - writable_alias_bindings.vertex.resources.images[0]) - .imageLayout == vk::ImageLayout::eGeneral && - DescriptorCacheTestAccess::MakeImageInfo( - writable_alias_bindings.pixel->resources.images[0]) - .imageLayout == vk::ImageLayout::eGeneral && - texture_cache.GetImage(storage_id).backing.state.layout == - vk::ImageLayout::eGeneral && - texture_cache.GetImage(storage_id) - .backing.state.access_mask == - (vk::AccessFlagBits2::eShaderRead | - vk::AccessFlagBits2::eShaderWrite | - vk::AccessFlagBits2::eColorAttachmentRead | - vk::AccessFlagBits2::eColorAttachmentWrite), - "sampled/storage aliases did not retain the promoted general " - "layout and writable access in both generated descriptors"); + Require( + name, "forced-general descriptor capture", + writable_alias_bindings.vertex.resources.images[0].layout == + vk::ImageLayout::eGeneral && + writable_alias_bindings.pixel->resources.images[0].layout == + vk::ImageLayout::eGeneral && + DescriptorCacheTestAccess::MakeImageInfo( + writable_alias_bindings.vertex.resources.images[0]) + .imageLayout == vk::ImageLayout::eGeneral && + DescriptorCacheTestAccess::MakeImageInfo( + writable_alias_bindings.pixel->resources.images[0]) + .imageLayout == vk::ImageLayout::eGeneral && + texture_cache.GetImage(storage_id).backing.state.layout == + vk::ImageLayout::eGeneral && + texture_cache.GetImage(storage_id).backing.state.access_mask == + (vk::AccessFlagBits2::eShaderRead | + vk::AccessFlagBits2::eShaderWrite | + vk::AccessFlagBits2::eColorAttachmentRead | + vk::AccessFlagBits2::eColorAttachmentWrite), + "sampled/storage aliases did not retain the promoted general " + "layout and writable access in both generated descriptors"); RenderExecutorTestAccess::ResetBindings(executor); const std::array split_mip_data{0x01020304u, 0x05060708u, @@ -7564,8 +7567,7 @@ public: vk::ImageAspectFlagBits::eColor && sampled_depth_binding.desc.view_info.usage == vk::ImageUsageFlagBits::eSampled && - sampled_depth_image.info.pixel_format == - vk::Format::eD32Sfloat && + sampled_depth_image.info.pixel_format == vk::Format::eD32Sfloat && sampled_depth_image.backing.format == vk::Format::eD32Sfloat && sampled_depth_image.backing.image == sampled_depth_backing && sampled_depth_view != nullptr && normalized_depth_view, @@ -11225,8 +11227,8 @@ TestCase ScalarShiftCountsMaskLowBits() { code, {}, {1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u}, - {O::S_MOV_B32, O::S_LSHL_B32, O::S_LSHR_B32, O::S_ASHR_I32, O::V_MOV_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_LSHL_B32, O::S_LSHR_B32, O::S_ASHR_I32, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase Rdna2ScalarOpcodes() { @@ -11250,8 +11252,8 @@ TestCase Rdna2ScalarOpcodes() { code, {}, {0x11u, 0x11u, 7u, 7u}, - {O::S_MOV_B32, O::S_BITSET1_B32, O::S_SETREG_B32, O::S_ADD_I32, O::S_SLEEP, - O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_BITSET1_B32, O::S_SETREG_B32, O::S_ADD_I32, + O::S_SLEEP, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarTrapDisabledFallsThroughExactRaw() { @@ -11338,10 +11340,10 @@ TestCase ScalarExtendedArithmetic() { code, {}, {0, 12, 15, 0xfffffffeu, 0xfffffffbu, 3, 3, 0xfffffffeu, 5, 33}, - {O::S_MOV_B32, O::S_ADD_U32, O::S_ADDC_U32, O::S_ADD_I32, O::S_SUB_I32, - O::S_MIN_I32, O::S_MAX_I32, O::S_MIN_U32, O::S_MAX_U32, O::S_ABS_I32, - O::S_MOVK_I32, O::S_MULK_I32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_ADD_U32, O::S_ADDC_U32, O::S_ADD_I32, + O::S_SUB_I32, O::S_MIN_I32, O::S_MAX_I32, O::S_MIN_U32, O::S_MAX_U32, + O::S_ABS_I32, O::S_MOVK_I32, O::S_MULK_I32, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarArithmeticSccCarryBorrowOverflow() { @@ -11403,8 +11405,9 @@ TestCase ScalarArithmeticSccCarryBorrowOverflow() { code, {}, {1, 0, 1, 0, 1, 0, 1, 0}, - {O::S_MOV_B32, O::S_SUB_U32, O::S_ADD_I32, O::S_SUB_I32, O::S_CMP_EQ_U32, - O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_SUB_U32, O::S_ADD_I32, O::S_SUB_I32, + O::S_CMP_EQ_U32, O::S_CSELECT_B32, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarMinMaxSccComparisonEdges() { @@ -11447,8 +11450,8 @@ TestCase ScalarMinMaxSccComparisonEdges() { {}, {1, 0, 1, 0, 1, 0, 1, 0}, {O::S_MOV_B32, O::S_MIN_I32, O::S_MAX_I32, O::S_MIN_U32, O::S_MAX_U32, - O::S_CMP_EQ_U32, O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + O::S_CMP_EQ_U32, O::S_CSELECT_B32, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarAbsI32UpdatesScc() { @@ -11482,8 +11485,8 @@ TestCase ScalarAbsI32UpdatesScc() { code, {}, {0, 0, 5, 1}, - {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_ABS_I32, O::S_CSELECT_B32, O::V_MOV_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_ABS_I32, O::S_CSELECT_B32, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarShiftLeftAddSccCarryEdges() { @@ -11526,9 +11529,9 @@ TestCase ScalarShiftLeftAddSccCarryEdges() { code, {}, {1, 0, 1, 0, 1, 0, 1, 0}, - {O::S_MOV_B32, O::S_LSHL1_ADD_U32, O::S_LSHL2_ADD_U32, O::S_LSHL3_ADD_U32, - O::S_LSHL4_ADD_U32, O::S_CMP_EQ_U32, O::S_CSELECT_B32, O::V_MOV_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_LSHL1_ADD_U32, O::S_LSHL2_ADD_U32, + O::S_LSHL3_ADD_U32, O::S_LSHL4_ADD_U32, O::S_CMP_EQ_U32, + O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarCompareOps() { @@ -11578,9 +11581,10 @@ TestCase ScalarCompareOps() { code, {}, {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1}, - {O::S_MOV_B32, O::S_CMP_EQ_I32, O::S_CMP_LG_I32, O::S_CMP_GT_I32, O::S_CMP_GE_I32, - O::S_CMP_LT_I32, O::S_CMP_LE_I32, O::S_CMP_LG_U32, O::S_CMP_GT_U32, O::S_CMP_GE_U32, - O::S_CMP_LE_U32, O::S_CMP_EQ_U64, O::S_CMP_LG_U64, O::S_CSELECT_B32, O::V_MOV_B32, + {O::S_MOV_B32, O::S_CMP_EQ_I32, O::S_CMP_LG_I32, O::S_CMP_GT_I32, + O::S_CMP_GE_I32, O::S_CMP_LT_I32, O::S_CMP_LE_I32, O::S_CMP_LG_U32, + O::S_CMP_GT_U32, O::S_CMP_GE_U32, O::S_CMP_LE_U32, O::S_CMP_EQ_U64, + O::S_CMP_LG_U64, O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } @@ -11619,9 +11623,10 @@ TestCase ScalarShiftAddAndMaskOps() { {}, {7, 11, 19, 35, 0xfffffffeu, 1, 0x7fffffeeu, 0x7ffffffeu, 0xf00000ffu, 0xf000000fu}, - {O::S_MOV_B32, O::S_LSHL1_ADD_U32, O::S_LSHL2_ADD_U32, O::S_LSHL3_ADD_U32, - O::S_LSHL4_ADD_U32, O::S_ASHR_I32, O::S_MUL_HI_U32, O::S_NOT_B64, O::S_WQM_B64, - O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_LSHL1_ADD_U32, O::S_LSHL2_ADD_U32, + O::S_LSHL3_ADD_U32, O::S_LSHL4_ADD_U32, O::S_ASHR_I32, + O::S_MUL_HI_U32, O::S_NOT_B64, O::S_WQM_B64, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarNotB64UpdatesScc() { @@ -11657,8 +11662,8 @@ TestCase ScalarNotB64UpdatesScc() { code, {}, {0, 1, 0, 0, 1, 0}, - {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_NOT_B64, O::S_CSELECT_B32, O::V_MOV_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_NOT_B64, O::S_CSELECT_B32, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarFlbitI32B64Gpu() { @@ -11691,8 +11696,8 @@ TestCase ScalarFlbitI32B64Gpu() { code, {}, {0, 48, 0xffffffffu, 60}, - {O::S_MOV_B32, O::S_FLBIT_I32_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_FLBIT_I32_B64, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarSaveExecOps() { @@ -11732,8 +11737,9 @@ TestCase ScalarSaveExecOps() { 0xffffffffu, 0xffffffffu, 0x00000003u, 0xffffffffu, 0x00000003u, 0x00000002u, 0xffffffffu, 1}, {O::S_MOV_B32, O::S_AND_SAVEEXEC_B64, O::S_ORN2_SAVEEXEC_B64, - O::S_ANDN1_SAVEEXEC_B64, O::S_AND_SAVEEXEC_B32, O::S_ANDN1_SAVEEXEC_B32, - O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + O::S_ANDN1_SAVEEXEC_B64, O::S_AND_SAVEEXEC_B32, + O::S_ANDN1_SAVEEXEC_B32, O::S_MOV_B64, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() { @@ -11848,10 +11854,10 @@ TestCase ScalarBitfieldPack() { 0xddddbbbbu, 0xccccbbbbu, 0xccccaaaau, 1, 1, 40, 5, 0xffffffffu, 0, 1}, {O::S_MOV_B32, O::S_BREV_B32, O::S_BCNT1_I32_B32, O::S_BCNT1_I32_B64, - O::S_FF1_I32_B64, O::S_BITREPLICATE_B64_B32, O::S_BFM_B32, O::S_BFE_U32, - O::S_PACK_LL_B32_B16, O::S_PACK_LH_B32_B16, O::S_PACK_HH_B32_B16, O::S_BITCMP0_B32, - O::S_BITCMP1_B32, O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + O::S_FF1_I32_B64, O::S_BITREPLICATE_B64_B32, O::S_BFM_B32, + O::S_BFE_U32, O::S_PACK_LL_B32_B16, O::S_PACK_LH_B32_B16, + O::S_PACK_HH_B32_B16, O::S_BITCMP0_B32, O::S_BITCMP1_B32, + O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarBrevB32PreservesScc() { @@ -11885,8 +11891,8 @@ TestCase ScalarBrevB32PreservesScc() { code, {}, {0x80000000u, 0, 0, 1}, - {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_BREV_B32, O::S_CSELECT_B32, O::V_MOV_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_BREV_B32, O::S_CSELECT_B32, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarBfeI32CapturedRawSignExtends() { @@ -11912,8 +11918,9 @@ TestCase ScalarBfeI32CapturedRawSignExtends() { test.code = std::move(code); test.initial = {0xf0000000u, 0, 0, 0, 0}; test.expected = {0xf0000000u, 0xffffffffu, 1, 0, 0}; - test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_BFE_I32, - O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, + test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, + O::S_BFE_I32, O::S_CSELECT_B32, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.required_spirv = {"OpBitFieldSExtract"}; return test; @@ -11942,8 +11949,8 @@ TestCase BitfieldExtractWidthPastEndEdges() { code, {}, {0x00000f00u, 1, 0x0000000fu}, - {O::S_MOV_B32, O::S_BFE_U32, O::V_MOV_B32, O::V_BFE_U32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_BFE_U32, O::V_MOV_B32, O::V_BFE_U32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase Scalar64BitOps() { @@ -11990,10 +11997,11 @@ TestCase Scalar64BitOps() { 0xf00ff00fu, 0x0f0f0f0fu, 0x00ff00ffu, 0xf0f0f0f0u, 0x0ff00ff0u, 0xff0f0f0fu, 0x0000ff00u, 0xfffffff0u, 0x000000ffu, 0xffffffffu, 0x7fffffffu, 0x000000f0u, 0, 0x0f0f0f0fu, 0x00ff00ffu}, - {O::S_MOV_B32, O::S_MOV_B64, O::S_AND_B64, O::S_ANDN2_B64, O::S_OR_B64, - O::S_ORN2_B64, O::S_XOR_B64, O::S_NAND_B64, O::S_NOR_B64, O::S_XNOR_B64, - O::S_LSHL_B64, O::S_LSHR_B64, O::S_BFM_B64, O::S_BFE_U64, O::S_CMP_EQ_U32, - O::S_CSELECT_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_MOV_B64, O::S_AND_B64, O::S_ANDN2_B64, + O::S_OR_B64, O::S_ORN2_B64, O::S_XOR_B64, O::S_NAND_B64, + O::S_NOR_B64, O::S_XNOR_B64, O::S_LSHL_B64, O::S_LSHR_B64, + O::S_BFM_B64, O::S_BFE_U64, O::S_CMP_EQ_U32, O::S_CSELECT_B64, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarAndn2B64SccBranch() { @@ -12065,7 +12073,8 @@ TestCase VectorVop3MoveAppliesFloatSourceModifiers() { test.name = "VectorVop3MoveAppliesFloatSourceModifiers"; test.code = code; test.expected = {0xc0000000u}; - test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.required_spirv = {"OpFNegate"}; return test; } @@ -12133,13 +12142,16 @@ TestCase VectorIntegerOps() { 32, 32, 0x0fffffffu, 0x0fffffffu, 0xffffffffu, 0xffffffffu, 0xf0f0f0f0u, 0xf0000000u, 1, 30, 7, 0xfffffff8u}, - {O::V_MOV_B32, O::V_MIN_I32, O::V_MAX_I32, O::V_MIN_U32, - O::V_MAX_U32, O::V_ADD_NC_U32, O::V_SUB_NC_U32, O::V_SUBREV_NC_U32, - O::V_MUL_U32_U24, O::V_MUL_I32_I24, O::V_AND_B32, O::V_OR_B32, - O::V_XOR_B32, O::V_XNOR_B32, O::V_LSHL_B32, O::V_LSHLREV_B32, - O::V_LSHR_B32, O::V_LSHRREV_B32, O::V_ASHR_I32, O::V_ASHRREV_I32, - O::V_NOT_B32, O::V_BFREV_B32, O::V_FFBL_B32, O::V_FFBH_U32, - O::V_CMP_EQ_U32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_MIN_I32, O::V_MAX_I32, + O::V_MIN_U32, O::V_MAX_U32, O::V_ADD_NC_U32, + O::V_SUB_NC_U32, O::V_SUBREV_NC_U32, O::V_MUL_U32_U24, + O::V_MUL_I32_I24, O::V_AND_B32, O::V_OR_B32, + O::V_XOR_B32, O::V_XNOR_B32, O::V_LSHL_B32, + O::V_LSHLREV_B32, O::V_LSHR_B32, O::V_LSHRREV_B32, + O::V_ASHR_I32, O::V_ASHRREV_I32, O::V_NOT_B32, + O::V_BFREV_B32, O::V_FFBL_B32, O::V_FFBH_U32, + O::V_CMP_EQ_U32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase Vop2SdwaSubNcExactByte2Destination() { @@ -12156,7 +12168,8 @@ TestCase Vop2SdwaSubNcExactByte2Destination() { test.name = "Vop2SdwaSubNcExactByte2Destination"; test.code = code; test.expected = {0x000b0002u}; - test.opcodes = {O::V_MOV_B32, O::V_SUB_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_SUB_NC_U32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; return test; } @@ -12179,7 +12192,7 @@ TestCase Vop2SdwaAddNcCapturedHighWordDestination() { test.initial = {0x00010002u, 0}; test.expected = {0x00010002u, 0x0006c3d4u}; test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::V_MOV_B32, - O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.required_spirv = {"OpIAdd", "OpShiftLeftLogical", "OpBitwiseOr"}; return test; } @@ -12190,8 +12203,7 @@ TestCase Vop2SdwaAshrrevCapturedWord0SignExtends() { std::vector code; AppendBufferLoadDword(&code, 1, 30); code.push_back(0x303202f9u); - code.push_back( - 0x0c860686u); // v_ashrrev_i32 v25, 6, sign_extend(v1.word0) + code.push_back(0x0c860686u); // v_ashrrev_i32 v25, 6, sign_extend(v1.word0) AppendStoreVgpr(&code, 25, 1); AppendEnd(&code); @@ -12277,8 +12289,8 @@ TestCase Vop3Med3I16Captured() { test.name = "Vop3Med3I16Captured"; test.code = std::move(code); test.expected = {0xfc18cafeu}; - test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_MED3_I16, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_MED3_I16, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.decoded_counts = {{"0x00000020: V_MED3_I16 v3.sdwa(sel=5,sext=0), " "v5.opsel(lo=1,hi=0,neghi=0), s25, s27\n", 1}}; @@ -12307,7 +12319,8 @@ TestCase Vop2SdwaMinU32PreservesWordDestination() { test.name = "Vop2SdwaMinU32PreservesWordDestination"; test.code = code; test.expected = {0xa1b20007u, 0xa1b2001fu}; - test.opcodes = {O::V_MOV_B32, O::V_MIN_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_MIN_U32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.required_spirv = {"OpULessThan", "OpSelect"}; return test; } @@ -12336,8 +12349,9 @@ TestCase VectorShiftCountsMaskLowBits() { code, {}, {1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u}, - {O::V_MOV_B32, O::V_LSHL_B32, O::V_LSHLREV_B32, O::V_LSHR_B32, O::V_LSHRREV_B32, - O::V_ASHR_I32, O::V_ASHRREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_LSHL_B32, O::V_LSHLREV_B32, O::V_LSHR_B32, + O::V_LSHRREV_B32, O::V_ASHR_I32, O::V_ASHRREV_I32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorVop3IntegerOps() { @@ -12429,14 +12443,17 @@ TestCase VectorVop3IntegerOps() { 5, 0xffffffffu, 1}, - {O::V_MOV_B32, O::V_ADD3_U32, O::V_MIN3_I32, O::V_MAX3_I32, - O::V_MED3_I32, O::V_MIN3_U32, O::V_MAX3_U32, O::V_MED3_U32, - O::V_SAD_U32, O::V_LSHL_ADD_U32, O::V_ADD_LSHL_U32, O::V_XAD_U32, - O::V_LSHL_OR_B32, O::V_AND_OR_B32, O::V_OR3_B32, O::V_XOR3_B32, - O::V_BFE_U32, O::V_BFE_I32, O::V_BFI_B32, O::V_ALIGNBIT_B32, - O::V_BFM_B32, O::V_MUL_LO_U32, O::V_MUL_HI_U32, O::V_MUL_LO_I32, - O::V_MUL_HI_I32, O::V_MAD_I32_I24, O::V_MAD_U32_U24, O::V_ADD_I32, - O::V_SUB_I32, O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_ADD3_U32, O::V_MIN3_I32, + O::V_MAX3_I32, O::V_MED3_I32, O::V_MIN3_U32, + O::V_MAX3_U32, O::V_MED3_U32, O::V_SAD_U32, + O::V_LSHL_ADD_U32, O::V_ADD_LSHL_U32, O::V_XAD_U32, + O::V_LSHL_OR_B32, O::V_AND_OR_B32, O::V_OR3_B32, + O::V_XOR3_B32, O::V_BFE_U32, O::V_BFE_I32, + O::V_BFI_B32, O::V_ALIGNBIT_B32, O::V_BFM_B32, + O::V_MUL_LO_U32, O::V_MUL_HI_U32, O::V_MUL_LO_I32, + O::V_MUL_HI_I32, O::V_MAD_I32_I24, O::V_MAD_U32_U24, + O::V_ADD_I32, O::V_SUB_I32, O::V_SUBREV_I32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorBfeI32SignExtendsField() { @@ -12478,12 +12495,13 @@ TestCase VectorAlignByteUsesFiveBitByteOffset() { } AppendEnd(&code); - return {"VectorAlignByteUsesFiveBitByteOffset", - code, - {}, - {0x55667788u, 0x44556677u, 0x22334455u, 0x11223344u, 0x00112233u, - 0x00000011u, 0u, 0u}, - {O::V_MOV_B32, O::V_ALIGNBYTE_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return { + "VectorAlignByteUsesFiveBitByteOffset", + code, + {}, + {0x55667788u, 0x44556677u, 0x22334455u, 0x11223344u, 0x00112233u, + 0x00000011u, 0u, 0u}, + {O::V_MOV_B32, O::V_ALIGNBYTE_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorCarryAndBitCountOps() { @@ -12538,8 +12556,12 @@ TestCase VectorMbcntUsesThreadMask() { test.name = "VectorMbcntUsesThreadMask"; test.code = code; test.expected = {0, 1, 2, 3, 0, 1, 2, 3}; - test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, - O::V_MBCNT_LO_U32_B32, O::V_MBCNT_HI_U32_B32, O::BUFFER_STORE_DWORD, + test.opcodes = {O::V_MOV_B32, + O::V_LSHLREV_B32, + O::V_ADD_NC_U32, + O::V_MBCNT_LO_U32_B32, + O::V_MBCNT_HI_U32_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; @@ -12571,8 +12593,9 @@ TestCase VectorAddcUsesPerLaneCarryIn() { test.name = "VectorAddcUsesPerLaneCarryIn"; test.code = code; test.expected = std::vector(8, 1); - test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_CMP_T_U32, - O::V_ADDC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, + O::V_CMP_T_U32, O::V_ADDC_U32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -12605,8 +12628,9 @@ TestCase VectorAddcWritesPerLaneCarryOut() { test.name = "VectorAddcWritesPerLaneCarryOut"; test.code = code; test.expected = std::vector(8, 0x0fu); - test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_CMP_F_U32, - O::V_ADDC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, + O::V_CMP_F_U32, O::V_ADDC_U32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -12645,10 +12669,11 @@ TestCase VectorSubrevCoCiU32ExactRawOnGpu() { test.name = "VectorSubrevCoCiU32ExactRawOnGpu"; test.code = code; test.expected = {0xdeadbeefu, 0, 2, 3, 0, 0, 0, 0}; - test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, - O::V_CMP_LT_U32, O::S_MOV_B64, O::V_CMP_NE_U32, - O::V_SUBREV_CO_CI_U32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, + O::V_ADD_NC_U32, O::V_CMP_LT_U32, + O::S_MOV_B64, O::V_CMP_NE_U32, + O::V_SUBREV_CO_CI_U32, O::V_CNDMASK_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.required_spirv = {"OpISub", "OpUGreaterThan"}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; @@ -12685,10 +12710,11 @@ TestCase VectorVop3BSubrevCoCiUsesEncodedMasks() { test.name = "VectorVop3BSubrevCoCiUsesEncodedMasks"; test.code = code; test.expected = {0xfffffffeu, 0xfffffffeu, 0, 1, 1, 1, 0, 0}; - test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, - O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_MOV_B32, - O::V_SUBREV_CO_CI_U32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, + O::V_ADD_NC_U32, O::V_CMP_EQ_U32, + O::S_MOV_B64, O::S_MOV_B32, + O::V_SUBREV_CO_CI_U32, O::V_CNDMASK_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -12791,7 +12817,7 @@ TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() { test.name = "VectorVop3BSubCoU32UsesRdna2Opcode310"; test.code = code; test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 9, 9, 9, 9}; - test.opcodes = {O::V_MOV_B32, O::V_CMP_EQ_U32, O::V_CNDMASK_B32, + test.opcodes = {O::V_MOV_B32, O::V_CMP_EQ_U32, O::V_CNDMASK_B32, O::V_SUB_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; @@ -12865,10 +12891,11 @@ TestCase VectorLaneAndPackedOps() { {0x12345678u, 0x12345678u, 0x12345678u, 0x12345678u, 0x40003c00u, 0x46004400u, 0x48004200u, 0x40003c00u, 0x44004200u, 0x48804400u, 0x40000000u, 0xef015678u}, - {O::V_MOV_B32, O::V_READFIRSTLANE_B32, O::V_READLANE_B32, O::V_WRITELANE_B32, - O::V_PERMLANE16_B32, O::V_CVT_PKRTZ_F16_F32, O::V_PK_ADD_F16, O::V_PK_MUL_F16, - O::V_PK_MIN_F16, O::V_PK_MAX_F16, O::V_PK_FMA_F16, O::V_LDEXP_F32, - O::V_CVT_PK_U16_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_READFIRSTLANE_B32, O::V_READLANE_B32, + O::V_WRITELANE_B32, O::V_PERMLANE16_B32, O::V_CVT_PKRTZ_F16_F32, + O::V_PK_ADD_F16, O::V_PK_MUL_F16, O::V_PK_MIN_F16, O::V_PK_MAX_F16, + O::V_PK_FMA_F16, O::V_LDEXP_F32, O::V_CVT_PK_U16_U32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase Vop3pOpselHiUsesArchitecturalSourceBits() { @@ -12916,11 +12943,12 @@ TestCase CvtPkU8F32PacksSelectedByte() { } AppendEnd(&code); - return {"CvtPkU8F32PacksSelectedByte", - code, - {}, - {0x1122330cu, 0x1122ff44u, 0x11003344u, 0x00223344u}, - {O::V_MOV_B32, O::V_CVT_PK_U8_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return { + "CvtPkU8F32PacksSelectedByte", + code, + {}, + {0x1122330cu, 0x1122ff44u, 0x11003344u, 0x00223344u}, + {O::V_MOV_B32, O::V_CVT_PK_U8_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase CvtPkrtzF16F32SubnormalRoundsTowardZero() { @@ -12937,7 +12965,8 @@ TestCase CvtPkrtzF16F32SubnormalRoundsTowardZero() { code, {}, {0x80010001u}, - {O::V_MOV_B32, O::V_CVT_PKRTZ_F16_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_CVT_PKRTZ_F16_F32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase PackedMinMaxF16NanAndSignedZeroEdges() { @@ -12965,8 +12994,8 @@ TestCase PackedMinMaxF16NanAndSignedZeroEdges() { code, {}, {0x80004000u, 0x00004000u, 0x7f017f01u, 0x7f017f01u}, - {O::V_MOV_B32, O::V_PK_MIN_F16, O::V_PK_MAX_F16, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_PK_MIN_F16, O::V_PK_MAX_F16, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorMinMaxF16Ops() { @@ -13008,8 +13037,9 @@ TestCase VectorMinMaxF16Ops() { {}, {0x66664200u, 0x77773c00u, 0x8888bc00u, 0xaaaa0000u, 0x55554000u, 0x12344000u, 0x87653c00u, 0x99994000u}, - {O::V_MOV_B32, O::V_ADD_F16, O::V_SUB_F16, O::V_SUBREV_F16, O::V_MUL_F16, - O::V_MAX_F16, O::V_MIN_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_ADD_F16, O::V_SUB_F16, O::V_SUBREV_F16, + O::V_MUL_F16, O::V_MAX_F16, O::V_MIN_F16, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase VectorCvtU16F16Sdwa() { @@ -13237,7 +13267,7 @@ TestCase VectorPermlane16FetchInactiveZero() { test.name = "VectorPermlane16FetchInactiveZero"; test.code = code; test.expected = {0}; - test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, O::V_PERMLANE16_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; @@ -13263,7 +13293,7 @@ TestCase VectorPermlane16FetchInactiveFi() { test.name = "VectorPermlane16FetchInactiveFi"; test.code = code; test.expected = {11}; - test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, O::V_PERMLANE16_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; @@ -13288,8 +13318,8 @@ TestCase VectorDppQuadPermuteReverse() { test.name = "VectorDppQuadPermuteReverse"; test.code = code; test.expected = {103, 102, 101, 100, 107, 106, 105, 104}; - test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 8; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -13315,8 +13345,8 @@ TestCase VectorDppBankMaskPreservesDestination() { test.code = code; test.expected = {0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, 104, 105, 106, 107}; - test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 8; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -13341,8 +13371,8 @@ TestCase VectorDppBoundsControlZeroPreservesDestination() { test.name = "VectorDppBoundsControlZeroPreservesDestination"; test.code = code; test.expected = {0xaaaaaaaau, 100, 101, 102, 103, 104, 105, 106}; - test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 8; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -13459,9 +13489,9 @@ TestCase VectorFloatSpecialOps() { {0x41400000u, 0x80017fffu, 0x8000ffffu, 0xaaaa4700u, 0x4700bbbbu, 0x40800000u, 0x3f800000u, 0xc0000000u, 0x40c00000u}, {O::V_MOV_B32, O::V_DOT2C_F32_F16, O::V_CVT_PKNORM_I16_F32, - O::V_CVT_PKNORM_U16_F32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16, O::V_CUBEID_F32, - O::V_CUBESC_F32, O::V_CUBETC_F32, O::V_CUBEMA_F32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + O::V_CVT_PKNORM_U16_F32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16, + O::V_CUBEID_F32, O::V_CUBESC_F32, O::V_CUBETC_F32, O::V_CUBEMA_F32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase MadMixF16LiteralHalfSourceUsesOpsel() { @@ -13485,8 +13515,8 @@ TestCase MadMixF16LiteralHalfSourceUsesOpsel() { code, {}, {0xaaaa4200u, 0x40002222u}, - {O::V_MOV_B32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase MadMixF16NegHiIsAbsAndNegIsIndependent() { @@ -13510,8 +13540,8 @@ TestCase MadMixF16NegHiIsAbsAndNegIsIndependent() { code, {}, {0xaaaa4200u, 0xbc002222u}, - {O::V_MOV_B32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorVop3FmaF16UsesRdna2Opcode34b() { @@ -13556,8 +13586,9 @@ TestCase VectorFloatControlContractPreservesInfNan() { test.name = "VectorFloatControlContractPreservesInfNan"; test.code = std::move(code); test.expected = {1u}; - test.opcodes = {O::S_BFM_B32, O::V_MOV_B32, O::V_MUL_F32, O::V_CMP_CLASS_F32, - O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::S_BFM_B32, O::V_MOV_B32, O::V_MUL_F32, + O::V_CMP_CLASS_F32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.required_spirv = {"SPV_KHR_float_controls", "SignedZeroInfNanPreserve 32"}; return test; @@ -13634,10 +13665,11 @@ TestCase VectorFloatArithmeticOps() { 0x40a00000u, 0x40800000u, 0x40c00000u, 0x41000000u, 0x40400000u, 0x40400000u, 0xbf800000u, 0x3f800000u, 0x3f800000u, 0x40800000u, 0xbf800000u, 0x40c00000u, 0x40000000u, 0x40400000u}, - {O::S_MOV_B32, O::V_MOV_B32, O::V_ADD_F32, O::V_SUB_F32, O::V_SUBREV_F32, - O::V_MUL_F32, O::V_MIN_F32, O::V_MAX_F32, O::V_MAC_F32, O::V_MAD_F32, - O::V_FMA_F32, O::V_MIN3_F32, O::V_MAX3_F32, O::V_MED3_F32, O::V_MADMK_F32, - O::V_MADAK_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::V_MOV_B32, O::V_ADD_F32, O::V_SUB_F32, + O::V_SUBREV_F32, O::V_MUL_F32, O::V_MIN_F32, O::V_MAX_F32, + O::V_MAC_F32, O::V_MAD_F32, O::V_FMA_F32, O::V_MIN3_F32, + O::V_MAX3_F32, O::V_MED3_F32, O::V_MADMK_F32, O::V_MADAK_F32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorMinMaxF32NanAndSignedZeroEdges() { @@ -13671,8 +13703,8 @@ TestCase VectorMinMaxF32NanAndSignedZeroEdges() { {}, {0x40000000u, 0x40000000u, 0x80000000u, 0x00000000u, 0x7fe00001u, 0x7fe00001u, 0x40000000u, 0x40000000u}, - {O::V_MOV_B32, O::V_MIN_F32, O::V_MAX_F32, O::V_MIN3_F32, O::V_MAX3_F32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_MIN_F32, O::V_MAX_F32, O::V_MIN3_F32, + O::V_MAX3_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorMed3F32NanUsesMin3Path() { @@ -13768,15 +13800,15 @@ TestCase VectorFloatConversionOps() { 0x40400000u, 0x40000000u, 0x40000000u, 0x40000000u, 0x40000000u, 0x3f000000u, 0x40000000u, 0, 0x3f800000u, 0xbd800000u, 0x3f800000u, 0x3f800000u, 0x40000000u}, - {O::V_MOV_B32, O::V_CVT_F32_I32, O::V_CVT_F32_U32, - O::V_CVT_U32_F32, O::V_CVT_I32_F32, O::V_CVT_F16_F32, + {O::V_MOV_B32, O::V_CVT_F32_I32, O::V_CVT_F32_U32, + O::V_CVT_U32_F32, O::V_CVT_I32_F32, O::V_CVT_F16_F32, O::V_CVT_F32_F16, O::V_CVT_RPI_I32_F32, O::V_CVT_FLR_I32_F32, - O::V_CVT_OFF_F32_I4, O::V_CVT_F32_UBYTE0, O::V_CVT_F32_UBYTE1, - O::V_CVT_F32_UBYTE2, O::V_CVT_F32_UBYTE3, O::V_RCP_F32, - O::V_FRACT_F32, O::V_TRUNC_F32, O::V_CEIL_F32, - O::V_RNDNE_F32, O::V_FLOOR_F32, O::V_EXP_F32, - O::V_LOG_F32, O::V_RSQ_F32, O::V_SQRT_F32, - O::V_SIN_F32, O::V_COS_F32, O::BUFFER_STORE_DWORD, + O::V_CVT_OFF_F32_I4, O::V_CVT_F32_UBYTE0, O::V_CVT_F32_UBYTE1, + O::V_CVT_F32_UBYTE2, O::V_CVT_F32_UBYTE3, O::V_RCP_F32, + O::V_FRACT_F32, O::V_TRUNC_F32, O::V_CEIL_F32, + O::V_RNDNE_F32, O::V_FLOOR_F32, O::V_EXP_F32, + O::V_LOG_F32, O::V_RSQ_F32, O::V_SQRT_F32, + O::V_SIN_F32, O::V_COS_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } @@ -13826,8 +13858,9 @@ TestCase VectorFrexpF32Edges() { 0x3f7ffffeu, 0x3f000000u, 0x7f800000u, 0x7fc00000u, 0xff800000u, 0x7fa00001u, }; - test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FREXP_EXP_I32_F32, - O::V_FREXP_MANT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, + O::V_FREXP_EXP_I32_F32, O::V_FREXP_MANT_F32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.decoded_counts = {{"V_FREXP_EXP_I32_F32", inputs.size()}, {"V_FREXP_MANT_F32", inputs.size()}}; test.required_spirv = {"FindUMsb", "OpBitFieldUExtract", @@ -13865,8 +13898,8 @@ TestCase CvtF32ToIntSaturatesNaNAndOutOfRange() { {}, {0, 0, 0xffffffffu, 0xffffffffu, 0, 0x7fffffffu, 0x80000000u, 0x7fffffffu}, - {O::V_MOV_B32, O::V_CVT_U32_F32, O::V_CVT_I32_F32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_CVT_U32_F32, O::V_CVT_I32_F32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorSpecialF32FlushesDenormalInputs() { @@ -13888,8 +13921,8 @@ TestCase VectorSpecialF32FlushesDenormalInputs() { code, {}, {0xff800000u, 0x7f800000u, 0x7f800000u, 0x00000000u}, - {O::V_MOV_B32, O::V_LOG_F32, O::V_RCP_F32, O::V_RSQ_F32, O::V_SQRT_F32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_LOG_F32, O::V_RCP_F32, O::V_RSQ_F32, + O::V_SQRT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorRcpIflagF32IntegerReciprocal() { @@ -13937,12 +13970,12 @@ TestCase VectorSinCosMaxFiniteSpecialCases() { AppendStoreVgpr(&code, 2, 1); AppendEnd(&code); - return { - "VectorSinCosMaxFiniteSpecialCases", - code, - {}, - {0x00000000u, 0x3f800000u}, - {O::V_MOV_B32, O::V_SIN_F32, O::V_COS_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"VectorSinCosMaxFiniteSpecialCases", + code, + {}, + {0x00000000u, 0x3f800000u}, + {O::V_MOV_B32, O::V_SIN_F32, O::V_COS_F32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase VectorCompareOps() { @@ -14031,15 +14064,18 @@ TestCase VectorCompareOps() { code, {}, expected, - {O::V_MOV_B32, O::V_CNDMASK_B32, O::V_CMP_F_F32, O::V_CMP_LT_F32, - O::V_CMP_EQ_F32, O::V_CMP_LE_F32, O::V_CMP_GT_F32, O::V_CMP_LG_F32, - O::V_CMP_GE_F32, O::V_CMP_O_F32, O::V_CMP_U_F32, O::V_CMP_NGE_F32, - O::V_CMP_NLG_F32, O::V_CMP_NGT_F32, O::V_CMP_NLE_F32, O::V_CMP_NEQ_F32, - O::V_CMP_NLT_F32, O::V_CMP_TRU_F32, O::V_CMP_F_I32, O::V_CMP_LT_I32, - O::V_CMP_EQ_I32, O::V_CMP_LE_I32, O::V_CMP_GT_I32, O::V_CMP_NE_I32, - O::V_CMP_GE_I32, O::V_CMP_T_I32, O::V_CMP_F_U32, O::V_CMP_LT_U32, - O::V_CMP_EQ_U32, O::V_CMP_LE_U32, O::V_CMP_GT_U32, O::V_CMP_NE_U32, - O::V_CMP_GE_U32, O::V_CMP_T_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_CNDMASK_B32, O::V_CMP_F_F32, + O::V_CMP_LT_F32, O::V_CMP_EQ_F32, O::V_CMP_LE_F32, + O::V_CMP_GT_F32, O::V_CMP_LG_F32, O::V_CMP_GE_F32, + O::V_CMP_O_F32, O::V_CMP_U_F32, O::V_CMP_NGE_F32, + O::V_CMP_NLG_F32, O::V_CMP_NGT_F32, O::V_CMP_NLE_F32, + O::V_CMP_NEQ_F32, O::V_CMP_NLT_F32, O::V_CMP_TRU_F32, + O::V_CMP_F_I32, O::V_CMP_LT_I32, O::V_CMP_EQ_I32, + O::V_CMP_LE_I32, O::V_CMP_GT_I32, O::V_CMP_NE_I32, + O::V_CMP_GE_I32, O::V_CMP_T_I32, O::V_CMP_F_U32, + O::V_CMP_LT_U32, O::V_CMP_EQ_U32, O::V_CMP_LE_U32, + O::V_CMP_GT_U32, O::V_CMP_NE_U32, O::V_CMP_GE_U32, + O::V_CMP_T_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorVop3CompareNeU64OnGpu() { @@ -14066,8 +14102,8 @@ TestCase VectorVop3CompareNeU64OnGpu() { code, {}, {0, 1}, - {O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_NE_U64, O::V_CNDMASK_B32, O::S_MOV_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_NE_U64, O::V_CNDMASK_B32, + O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorVop3CompareGtU64OnGpu() { @@ -14135,8 +14171,8 @@ TestCase VectorVop3CompareEqI64OnGpu() { code, {}, {1, 0}, - {O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_I64, O::V_CNDMASK_B32, O::S_MOV_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_I64, O::V_CNDMASK_B32, + O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorCompareClassF32() { @@ -14177,8 +14213,8 @@ TestCase VectorCompareClassF32() { code, {}, expected, - {O::V_MOV_B32, O::V_CMP_CLASS_F32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_CMP_CLASS_F32, O::V_CNDMASK_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase VectorCompareF16Ops() { @@ -14229,10 +14265,11 @@ TestCase VectorCompareF16Ops() { code, {}, expected, - {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_LT_F16, O::V_CMP_EQ_F16, O::V_CMP_LE_F16, - O::V_CMP_GT_F16, O::V_CMP_LG_F16, O::V_CMP_GE_F16, O::V_CMP_NEQ_F16, - O::V_CMPX_LT_F16, O::V_CMPX_EQ_F16, O::V_CMPX_LE_F16, O::V_CMPX_GT_F16, - O::V_CMPX_GE_F16, O::V_CMPX_NEQ_F16, O::V_CMPX_NLT_F16, O::V_CNDMASK_B32, + {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_LT_F16, O::V_CMP_EQ_F16, + O::V_CMP_LE_F16, O::V_CMP_GT_F16, O::V_CMP_LG_F16, O::V_CMP_GE_F16, + O::V_CMP_NEQ_F16, O::V_CMPX_LT_F16, O::V_CMPX_EQ_F16, + O::V_CMPX_LE_F16, O::V_CMPX_GT_F16, O::V_CMPX_GE_F16, + O::V_CMPX_NEQ_F16, O::V_CMPX_NLT_F16, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } @@ -14253,8 +14290,8 @@ TestCase Vop2SdwaCndmaskSourceModifier() { code, {}, {0xbf800000u}, - {O::V_MOV_B32, O::V_CMP_T_U32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + {O::V_MOV_B32, O::V_CMP_T_U32, O::V_CNDMASK_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase Vop2SdwaCndmaskFullDestinationWithSubDwordSource() { @@ -14277,7 +14314,7 @@ TestCase Vop2SdwaCndmaskFullDestinationWithSubDwordSource() { test.name = "Vop2SdwaCndmaskFullDestinationWithSubDwordSource"; test.code = code; test.expected = {0x00001234u, 0x55667788u}; - test.opcodes = {O::V_MOV_B32, O::V_CMP_F_U32, O::V_CMP_T_U32, + test.opcodes = {O::V_MOV_B32, O::V_CMP_F_U32, O::V_CMP_T_U32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.required_spirv = {"OpSelect"}; return test; @@ -14303,8 +14340,9 @@ TestCase Vop3CndmaskUsesSgprMaskLaneBits() { test.name = "Vop3CndmaskUsesSgprMaskLaneBits"; test.code = code; test.expected = {20, 10, 10, 10, 20, 10, 10, 10}; - test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_CMP_F_U32, - O::S_MOV_B32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, + O::V_CMP_F_U32, O::S_MOV_B32, O::V_CNDMASK_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -14383,13 +14421,16 @@ TestCase VectorCompareExecOps() { code, {}, std::vector(out, 1), - {O::S_MOV_B32, O::V_MOV_B32, O::V_CMPX_LT_F32, O::V_CMPX_EQ_F32, - O::V_CMPX_LE_F32, O::V_CMPX_GT_F32, O::V_CMPX_LG_F32, O::V_CMPX_GE_F32, - O::V_CMPX_NGE_F32, O::V_CMPX_NLG_F32, O::V_CMPX_NGT_F32, O::V_CMPX_NLE_F32, - O::V_CMPX_NEQ_F32, O::V_CMPX_NLT_F32, O::V_CMPX_LT_I32, O::V_CMPX_EQ_I32, - O::V_CMPX_LE_I32, O::V_CMPX_GT_I32, O::V_CMPX_NE_I32, O::V_CMPX_GE_I32, - O::V_CMPX_LT_U32, O::V_CMPX_EQ_U32, O::V_CMPX_LE_U32, O::V_CMPX_GT_U32, - O::V_CMPX_NE_U32, O::V_CMPX_GE_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::V_MOV_B32, O::V_CMPX_LT_F32, + O::V_CMPX_EQ_F32, O::V_CMPX_LE_F32, O::V_CMPX_GT_F32, + O::V_CMPX_LG_F32, O::V_CMPX_GE_F32, O::V_CMPX_NGE_F32, + O::V_CMPX_NLG_F32, O::V_CMPX_NGT_F32, O::V_CMPX_NLE_F32, + O::V_CMPX_NEQ_F32, O::V_CMPX_NLT_F32, O::V_CMPX_LT_I32, + O::V_CMPX_EQ_I32, O::V_CMPX_LE_I32, O::V_CMPX_GT_I32, + O::V_CMPX_NE_I32, O::V_CMPX_GE_I32, O::V_CMPX_LT_U32, + O::V_CMPX_EQ_U32, O::V_CMPX_LE_U32, O::V_CMPX_GT_U32, + O::V_CMPX_NE_U32, O::V_CMPX_GE_U32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase VectorVop3FloatCompareNegSourceModifier() { @@ -14510,8 +14551,8 @@ TestCase SimpleLoop() { code, {}, {4}, - {O::S_MOV_B32, O::S_CMP_LT_U32, O::S_CBRANCH_SCC0, O::S_ADD_U32, O::S_BRANCH, - O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::S_CMP_LT_U32, O::S_CBRANCH_SCC0, O::S_ADD_U32, + O::S_BRANCH, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase BranchVccnzUsesWholeMask() { @@ -14536,7 +14577,7 @@ TestCase BranchVccnzUsesWholeMask() { test.code = code; test.expected = std::vector(8, 42); test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, - O::V_CMP_EQ_U32, O::S_CBRANCH_VCCNZ, O::S_BRANCH, + O::V_CMP_EQ_U32, O::S_CBRANCH_VCCNZ, O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; @@ -14603,9 +14644,9 @@ TestCase ScalarMemoryLoadVariants() { expected, {O::S_MOV_B32, O::S_LOAD_DWORD, O::S_LOAD_DWORDX2, O::S_LOAD_DWORDX4, O::S_LOAD_DWORDX8, O::S_LOAD_DWORDX16, O::S_BUFFER_LOAD_DWORD, - O::S_BUFFER_LOAD_DWORDX2, O::S_BUFFER_LOAD_DWORDX4, O::S_BUFFER_LOAD_DWORDX8, - O::S_BUFFER_LOAD_DWORDX16, O::V_MOV_B32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + O::S_BUFFER_LOAD_DWORDX2, O::S_BUFFER_LOAD_DWORDX4, + O::S_BUFFER_LOAD_DWORDX8, O::S_BUFFER_LOAD_DWORDX16, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase ScalarLoadSignedImmediateOffsetAddsSoffset() { @@ -14635,11 +14676,12 @@ TestCase BufferLoadStore() { code.push_back(EncodeVop1(0x01, 31, InlineU32(4))); AppendBufferStoreDword(&code, 0, 31); AppendEnd(&code); - return {"BufferLoadStore", - code, - {0x11223344u, 0}, - {0x11223344u, 0x11223344u}, - {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return { + "BufferLoadStore", + code, + {0x11223344u, 0}, + {0x11223344u, 0x11223344u}, + {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset() { @@ -14653,11 +14695,12 @@ TestCase BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset() { AppendStoreVgpr(&code, 0, 0); AppendEnd(&code); - return {"BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset", - code, - {0x11111111u, 0x22222222u, 0x33333333u}, - {0x33333333u, 0x22222222u, 0x33333333u}, - {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return { + "BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset", + code, + {0x11111111u, 0x22222222u, 0x33333333u}, + {0x33333333u, 0x22222222u, 0x33333333u}, + {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset() { @@ -14688,11 +14731,12 @@ TestCase BufferLoadDwordNoAddressFlagsIgnoresVaddr() { AppendStoreVgpr(&code, 0, 3); AppendEnd(&code); - return {"BufferLoadDwordNoAddressFlagsIgnoresVaddr", - code, - {0x11111111u, 0x22222222u, 0x33333333u, 0}, - {0x11111111u, 0x22222222u, 0x33333333u, 0x11111111u}, - {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return { + "BufferLoadDwordNoAddressFlagsIgnoresVaddr", + code, + {0x11111111u, 0x22222222u, 0x33333333u, 0}, + {0x11111111u, 0x22222222u, 0x33333333u, 0x11111111u}, + {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase BufferLoadDwordIdxenUsesDescriptorStride() { @@ -14794,7 +14838,8 @@ TestCase BufferOffsetsUsePackedLaneAndStorageFallback() { test.storage_buffer_range_dwords = 1; test.storage_buffer_offsets = {0, 12}; test.force_shader_data_storage = true; - test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; return test; } @@ -14828,8 +14873,9 @@ TestCase BufferLoadVariants() { 0x11223344u, 0x55667788u, 0x99aabbccu, 0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u}, {O::BUFFER_LOAD_UBYTE, O::BUFFER_LOAD_SBYTE, O::BUFFER_LOAD_USHORT, - O::BUFFER_LOAD_SSHORT, O::BUFFER_LOAD_DWORDX2, O::BUFFER_LOAD_DWORDX3, - O::BUFFER_LOAD_DWORDX4, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + O::BUFFER_LOAD_SSHORT, O::BUFFER_LOAD_DWORDX2, + O::BUFFER_LOAD_DWORDX3, O::BUFFER_LOAD_DWORDX4, O::V_MOV_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase BufferLoadDwordx4SnapshotsOverlappingAddress() { @@ -14927,8 +14973,8 @@ TestCase BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail() { test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; test.expected = {0x33333333u, 0x44444444u, 0u, 0u}; test.storage_buffer_range_dwords = 4; - test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; return test; } @@ -14973,8 +15019,7 @@ TestCase BufferLoadFormatXyzwRejectsPartialRecord() { test.expected = {0u, 0u, 0u, 0u}; test.storage_buffer_range_dwords = 4; test.user_data = MakeStructuredStorageBufferData( - 0, 4, false, - BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -15001,8 +15046,7 @@ TestCase BufferStoreFormatXyzwDropsPartialRecord() { test.expected = test.initial; test.storage_buffer_range_dwords = 4; test.user_data = MakeStructuredStorageBufferData( - 0, 4, false, - BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM}; return test; @@ -15024,11 +15068,10 @@ TestCase BufferLoadFormatXChecksOnlyTransferredComponent() { test.expected = {0x44444444u, 0x22222222u, 0x33333333u, 0x44444444u}; test.storage_buffer_range_dwords = 4; test.user_data = MakeStructuredStorageBufferData( - 0, 4, false, - BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; - test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_X, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; return test; } @@ -15049,8 +15092,7 @@ TestCase BufferStoreFormatXChecksOnlyTransferredComponent() { test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0xaaaaaaaau}; test.storage_buffer_range_dwords = 4; test.user_data = MakeStructuredStorageBufferData( - 0, 4, false, - BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM}; return test; @@ -15073,11 +15115,10 @@ TestCase BufferLoadFormatXyChecksOnlyTransferredComponents() { test.expected = {0x33333333u, 0x44444444u, 0x33333333u, 0x44444444u}; test.storage_buffer_range_dwords = 4; test.user_data = MakeStructuredStorageBufferData( - 0, 4, false, - BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; - test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; return test; } @@ -15099,8 +15140,7 @@ TestCase BufferStoreFormatXyChecksOnlyTransferredComponents() { test.expected = {0x11111111u, 0x22222222u, 0xaaaaaaaau, 0xbbbbbbbbu}; test.storage_buffer_range_dwords = 4; test.user_data = MakeStructuredStorageBufferData( - 0, 4, false, - BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM}; return test; @@ -15139,8 +15179,8 @@ TestCase BufferStoreVariants() { {0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u, 0x88888888u, 0x99999999u}, {O::V_MOV_B32, O::BUFFER_STORE_BYTE, O::BUFFER_STORE_SHORT, - O::BUFFER_STORE_DWORDX2, O::BUFFER_STORE_DWORDX3, O::BUFFER_STORE_DWORDX4, - O::S_ENDPGM}}; + O::BUFFER_STORE_DWORDX2, O::BUFFER_STORE_DWORDX3, + O::BUFFER_STORE_DWORDX4, O::S_ENDPGM}}; } TestCase BufferFormatVariants() { @@ -15194,8 +15234,8 @@ TestCase BufferLoadFormatXyzwSnapshotsOverlappingAddress() { 0, 0, 0, 0}; test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; - test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.user_data = MakeStructuredStorageBufferData( 16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; @@ -15276,8 +15316,9 @@ TestCase BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress() { test.initial = {0xaaaaaaaa, 0xbbbbbbbb, 0xcccccccc, 0xdddddddd, 0, 0, 0, 0}; test.expected = {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu, 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; - test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_FORMAT_XYZW, - O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = { + O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_FORMAT_XYZW, + O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.user_data = MakeStructuredStorageBufferData( 16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; @@ -15314,7 +15355,8 @@ TestCase BufferFormatStoreVariants() { {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u, 0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u}, {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::BUFFER_STORE_FORMAT_XY, - O::BUFFER_STORE_FORMAT_XYZ, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM}}; + O::BUFFER_STORE_FORMAT_XYZ, O::BUFFER_STORE_FORMAT_XYZW, + O::S_ENDPGM}}; } TestCase BufferLoadFormatXResource8UintZeroExtendsByte() { @@ -15593,8 +15635,8 @@ TestCase TBufferLoadVariants() { {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u, 0x11121314u, 0x21222324u, 0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}, {O::TBUFFER_LOAD_FORMAT_X, O::TBUFFER_LOAD_FORMAT_XY, - O::TBUFFER_LOAD_FORMAT_XYZ, O::TBUFFER_LOAD_FORMAT_XYZW, O::V_MOV_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + O::TBUFFER_LOAD_FORMAT_XYZ, O::TBUFFER_LOAD_FORMAT_XYZW, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase TBufferLoadFormatXyzwSnapshotsOverlappingAddress() { @@ -15620,8 +15662,8 @@ TestCase TBufferLoadFormatXyzwSnapshotsOverlappingAddress() { 0, 0, 0, 0}; test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; - test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.user_data = MakeStructuredStorageBufferData(16, 2); test.has_user_data = true; return test; @@ -15647,8 +15689,8 @@ TestCase TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress() { test.code = std::move(code); test.initial = {0x44332211u, 0, 0, 0, 0, 0, 0, 0}; test.expected = {0x44332211u, 0, 0, 0, 0x11u, 0x22u, 0x33u, 0x44u}; - test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.user_data = MakeStructuredStorageBufferData(4, 8); test.has_user_data = true; return test; @@ -15685,7 +15727,8 @@ TestCase TBufferLoadFormatX8UintZeroExtendsByte() { code, {0x11223344u, 0}, {0x00000044u, 0}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatX8888UintExtractsFirstByte() { @@ -15702,7 +15745,8 @@ TestCase TBufferLoadFormatX8888UintExtractsFirstByte() { code, {0x44332211u, 0}, {0x00000011u, 0}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatXIdxenUsesDescriptorStride() { @@ -15741,7 +15785,8 @@ TestCase TBufferLoadFormatX16FloatConvertsToFloat() { code, {0x00003c00u, 0}, {0x3f800000u, 0}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatXSintSignExtendsSubDword() { @@ -15762,7 +15807,8 @@ TestCase TBufferLoadFormatXSintSignExtendsSubDword() { code, {0x00000080u, 0x00008001u}, {0xffffff80u, 0xffff8001u}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferStoreFormatXSintWritesSubDword() { @@ -15801,12 +15847,12 @@ TestCase TBufferLoadFormatXy88IntegerComponents() { } AppendEnd(&code); - return { - "TBufferLoadFormatXy8_8IntegerComponents", - code, - {0x0000807fu, 0x00007f80u, 0, 0}, - {0x0000007fu, 0x00000080u, 0xffffff80u, 0x0000007fu}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXy8_8IntegerComponents", + code, + {0x0000807fu, 0x00007f80u, 0, 0}, + {0x0000007fu, 0x00000080u, 0xffffff80u, 0x0000007fu}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferStoreFormatXy88IntegerComponents() { @@ -15847,12 +15893,12 @@ TestCase TBufferLoadFormatXy1616IntegerComponents() { } AppendEnd(&code); - return { - "TBufferLoadFormatXy16_16IntegerComponents", - code, - {0x80017fffu, 0x7fff8000u, 0, 0}, - {0x00007fffu, 0x00008001u, 0xffff8000u, 0x00007fffu}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXy16_16IntegerComponents", + code, + {0x80017fffu, 0x7fff8000u, 0, 0}, + {0x00007fffu, 0x00008001u, 0xffff8000u, 0x00007fffu}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferStoreFormatXy1616IntegerComponents() { @@ -15890,12 +15936,12 @@ TestCase TBufferLoadFormatXyz16161616UintLoadsHalfwords() { } AppendEnd(&code); - return { - "TBufferLoadFormatXyz16_16_16_16UintLoadsHalfwords", - code, - {0x22221111u, 0x44443333u, 0}, - {0x00001111u, 0x00002222u, 0x00003333u}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZ, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXyz16_16_16_16UintLoadsHalfwords", + code, + {0x22221111u, 0x44443333u, 0}, + {0x00001111u, 0x00002222u, 0x00003333u}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZ, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatXy88UnormConvertsToFloat() { @@ -15909,12 +15955,12 @@ TestCase TBufferLoadFormatXy88UnormConvertsToFloat() { AppendStoreVgpr(&code, 1, 1); AppendEnd(&code); - return { - "TBufferLoadFormatXy8_8UnormConvertsToFloat", - code, - {0x0000ff80u, 0}, - {0x3f008081u, 0x3f800000u}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXy8_8UnormConvertsToFloat", + code, + {0x0000ff80u, 0}, + {0x3f008081u, 0x3f800000u}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatXy88SnormConvertsToFloat() { @@ -15928,12 +15974,12 @@ TestCase TBufferLoadFormatXy88SnormConvertsToFloat() { AppendStoreVgpr(&code, 1, 1); AppendEnd(&code); - return { - "TBufferLoadFormatXy8_8SnormConvertsToFloat", - code, - {0x00007f80u, 0}, - {0xbf800000u, 0x3f800000u}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXy8_8SnormConvertsToFloat", + code, + {0x00007f80u, 0}, + {0xbf800000u, 0x3f800000u}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatXy1616UnormConvertsToFloat() { @@ -15947,12 +15993,12 @@ TestCase TBufferLoadFormatXy1616UnormConvertsToFloat() { AppendStoreVgpr(&code, 1, 1); AppendEnd(&code); - return { - "TBufferLoadFormatXy16_16UnormConvertsToFloat", - code, - {0xffff8000u, 0}, - {0x3f000080u, 0x3f800000u}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXy16_16UnormConvertsToFloat", + code, + {0xffff8000u, 0}, + {0x3f000080u, 0x3f800000u}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents() { @@ -15966,12 +16012,12 @@ TestCase TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents() { AppendStoreVgpr(&code, 1, 1); AppendEnd(&code); - return { - "TBufferLoadFormatXy8_8_8_8UnormConvertsFirstTwoComponents", - code, - {0x44332211u, 0xdeadbeefu}, - {0x3d888889u, 0x3e088889u}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXy8_8_8_8UnormConvertsFirstTwoComponents", + code, + {0x44332211u, 0xdeadbeefu}, + {0x3d888889u, 0x3e088889u}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatXyzw8888UintExtractsBytes() { @@ -15986,12 +16032,12 @@ TestCase TBufferLoadFormatXyzw8888UintExtractsBytes() { } AppendEnd(&code); - return { - "TBufferLoadFormatXyzw8_8_8_8UintExtractsBytes", - code, - {0x44332211u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu}, - {0x00000011u, 0x00000022u, 0x00000033u, 0x00000044u}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXyzw8_8_8_8UintExtractsBytes", + code, + {0x44332211u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu}, + {0x00000011u, 0x00000022u, 0x00000033u, 0x00000044u}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatXyzw1010102SnormConvertsToFloat() { @@ -16006,12 +16052,12 @@ TestCase TBufferLoadFormatXyzw1010102SnormConvertsToFloat() { } AppendEnd(&code); - return { - "TBufferLoadFormatXyzw10_10_10_2SnormConvertsToFloat", - code, - {0x800801ffu, 0, 0, 0}, - {0x3f800000u, 0xbf800000u, 0, 0xbf800000u}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXyzw10_10_10_2SnormConvertsToFloat", + code, + {0x800801ffu, 0, 0, 0}, + {0x3f800000u, 0xbf800000u, 0, 0xbf800000u}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatXyz111110FloatUnpacks() { @@ -16026,12 +16072,12 @@ TestCase TBufferLoadFormatXyz111110FloatUnpacks() { } AppendEnd(&code); - return { - "TBufferLoadFormatXyz11_11_10FloatUnpacks", - code, - {0x781e03c0u, 0, 0}, - {0x3f800000u, 0x3f800000u, 0x3f800000u}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZ, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXyz11_11_10FloatUnpacks", + code, + {0x781e03c0u, 0, 0}, + {0x3f800000u, 0x3f800000u, 0x3f800000u}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZ, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferLoadFormatXyzw3232FloatZerosMissingComponents() { @@ -16046,12 +16092,12 @@ TestCase TBufferLoadFormatXyzw3232FloatZerosMissingComponents() { } AppendEnd(&code); - return { - "TBufferLoadFormatXyzw32_32FloatZerosMissingComponents", - code, - {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}, - {0x11111111u, 0x22222222u, 0, 0}, - {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return {"TBufferLoadFormatXyzw32_32FloatZerosMissingComponents", + code, + {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}, + {0x11111111u, 0x22222222u, 0, 0}, + {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents() { @@ -16104,7 +16150,8 @@ TestCase TBufferStoreVariants() { {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u, 0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u}, {O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_X, O::TBUFFER_STORE_FORMAT_XY, - O::TBUFFER_STORE_FORMAT_XYZ, O::TBUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM}}; + O::TBUFFER_STORE_FORMAT_XYZ, O::TBUFFER_STORE_FORMAT_XYZW, + O::S_ENDPGM}}; } TestCase FlatLoadVariants() { @@ -16145,8 +16192,8 @@ TestCase FlatLoadVariants() { 0x11223344u}, {O::V_MOV_B32, O::FLAT_LOAD_UBYTE, O::FLAT_LOAD_SBYTE, O::FLAT_LOAD_USHORT, O::FLAT_LOAD_SSHORT, O::FLAT_LOAD_DWORD, - O::FLAT_LOAD_DWORDX2, O::FLAT_LOAD_DWORDX3, O::FLAT_LOAD_DWORDX4, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + O::FLAT_LOAD_DWORDX2, O::FLAT_LOAD_DWORDX3, + O::FLAT_LOAD_DWORDX4, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; test.flat_memory_base = 0; return test; } @@ -16172,9 +16219,9 @@ TestCase BranchVccnzUsesCarryProducedWholeMask() { test.name = "BranchVccnzUsesCarryProducedWholeMask"; test.code = code; test.expected = std::vector(8, 42); - test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, + test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_CMP_F_U32, O::V_ADDC_U32, O::S_CBRANCH_VCCNZ, - O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -16252,8 +16299,8 @@ TestCase GlobalSignedImmediateRebasesBeforeSaddr() { code, {0x11111111u, 0x22222222u, 0x12345678u}, {0x12345678u}, - {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase FlatSegmentIgnoresSaddrAndMasksOffsetMsb() { @@ -16282,8 +16329,8 @@ TestCase FlatSegmentIgnoresSaddrAndMasksOffsetMsb() { test.code = code; test.initial = initial; test.expected = {0x11111111u, 0x11111111u}; - test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.flat_memory_base = 0; return test; } @@ -16331,8 +16378,8 @@ TestCase FlatStoreVariants() { 0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u, 0x88888888u, 0x99999999u, 0x11111111u, 0}, {O::V_MOV_B32, O::FLAT_STORE_BYTE, O::FLAT_STORE_SHORT, - O::FLAT_STORE_DWORD, O::FLAT_STORE_DWORDX2, O::FLAT_STORE_DWORDX3, - O::FLAT_STORE_DWORDX4, O::S_ENDPGM}}; + O::FLAT_STORE_DWORD, O::FLAT_STORE_DWORDX2, + O::FLAT_STORE_DWORDX3, O::FLAT_STORE_DWORDX4, O::S_ENDPGM}}; test.flat_memory_base = 0; return test; } @@ -16398,8 +16445,8 @@ TestCase DsReadWriteVariants() { {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_B32, O::DS_WRITE_B8, O::DS_READ_U8, O::DS_READ_I8, O::DS_WRITE_B16, O::DS_READ_U16, O::DS_READ_I16, O::DS_WRITE_B64, O::DS_READ_B64, O::DS_WRITE_B96, - O::DS_READ_B96, O::DS_WRITE_B128, O::DS_READ_B128, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + O::DS_READ_B96, O::DS_WRITE_B128, O::DS_READ_B128, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase DsReadWrite2Variants() { @@ -16462,8 +16509,126 @@ TestCase DsReadWrite2Variants() { 0x12345678u}, {O::V_MOV_B32, O::DS_WRITE2_B32, O::DS_READ2_B32, O::DS_WRITE_B64, O::DS_READ2_B64, O::DS_WRITE2_B64, O::DS_WRITE2ST64_B32, - O::DS_WRITE2ST64_B64, O::DS_READ2ST64_B64, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}}; + O::DS_READ2ST64_B32, O::DS_WRITE2ST64_B64, O::DS_READ2ST64_B64, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; +} + +TestCase DsWideReadSnapshotsOverlappingAddress() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 6, 0x11111111u); + AppendVMovLiteral(&code, 7, 0x22222222u); + code.push_back(EncodeDs0(0x4d, 0)); + code.push_back(EncodeDs1(0, 6, 1)); + AppendVMovLiteral(&code, 8, 0x33333333u); + AppendVMovLiteral(&code, 9, 0x44444444u); + code.push_back(EncodeDs0(0x4d, 16)); + code.push_back(EncodeDs1(0, 8, 1)); + code.push_back(EncodeDs0(0x77, 2u << 8u)); + code.push_back(EncodeDs1Ex(1, 0, 0, 1)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 1u + i, i); + } + AppendEnd(&code); + + return {"DsWideReadSnapshotsOverlappingAddress", + code, + std::vector(4, 0), + {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}, + {O::V_MOV_B32, O::DS_WRITE_B64, O::DS_READ2_B64, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; +} + +TestCase DsReadWrite2EqualOffsetsUseData0() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x11111111u); + AppendVMovLiteral(&code, 3, 0x22222222u); + code.push_back(EncodeDs0(0x0e, (1u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(0, 3, 2, 1)); + AppendVMovLiteral(&code, 4, 0x33333333u); + AppendVMovLiteral(&code, 5, 0x44444444u); + AppendVMovLiteral(&code, 6, 0x55555555u); + AppendVMovLiteral(&code, 7, 0x66666666u); + code.push_back(EncodeDs0(0x4e, (2u << 8u) | 2u)); + code.push_back(EncodeDs1Ex(0, 6, 4, 1)); + code.push_back(EncodeDs0(0x37, (1u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(8, 0, 0, 1)); + code.push_back(EncodeDs0(0x77, (2u << 8u) | 2u)); + code.push_back(EncodeDs1Ex(10, 0, 0, 1)); + for (u32 i = 0; i < 6; i++) { + AppendStoreVgpr(&code, 8u + i, i); + } + AppendEnd(&code); + + return {"DsReadWrite2EqualOffsetsUseData0", + code, + std::vector(6, 0), + {0x11111111u, 0x11111111u, 0x33333333u, 0x44444444u, 0x33333333u, + 0x44444444u}, + {O::V_MOV_B32, O::DS_WRITE2_B32, O::DS_WRITE2_B64, O::DS_READ2_B32, + O::DS_READ2_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; +} + +TestCase DsWideLdsPartialBounds() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x11111111u); + AppendVMovLiteral(&code, 3, 0x22222222u); + AppendVMovLiteral(&code, 4, 0x33333333u); + AppendVMovLiteral(&code, 5, 0x44444444u); + code.push_back(EncodeDs0(0xdf)); + code.push_back(EncodeDs1(0, 2, 1)); + code.push_back(EncodeDs0(0xff)); + code.push_back(EncodeDs1(6, 0, 1)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 6u + i, i); + } + AppendEnd(&code); + + TestCase test{"DsWideLdsPartialBounds", + code, + std::vector(4, 0), + {0x11111111u, 0x22222222u, 0x33333333u, 0}, + {O::V_MOV_B32, O::DS_WRITE_B128, O::DS_READ_B128, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + test.compute_info.lds_size_dwords = 3; + return test; +} + +TestCase DsWideGdsPartialBounds() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x11111111u); + AppendVMovLiteral(&code, 3, 0x22222222u); + AppendVMovLiteral(&code, 4, 0x33333333u); + AppendVMovLiteral(&code, 5, 0x44444444u); + code.push_back(EncodeDs0(0xdf, 0, true)); + code.push_back(EncodeDs1(0, 2, 1)); + code.push_back(EncodeDs0(0xff, 0, true)); + code.push_back(EncodeDs1(6, 0, 1)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 6u + i, i); + } + AppendEnd(&code); + + TestCase test{"DsWideGdsPartialBounds", + code, + std::vector(4, 0), + {0x11111111u, 0x22222222u, 0x33333333u, 0}, + {O::V_MOV_B32, O::DS_WRITE_B128, O::DS_READ_B128, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + test.gds_initial = {0, 0, 0}; + test.expected_gds = {0x11111111u, 0x22222222u, 0x33333333u}; + return test; } TestCase DsAtomicNoReturnVariants() { @@ -16494,9 +16659,10 @@ TestCase DsAtomicNoReturnVariants() { code, std::vector(9, 0), {15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u}, - {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_ADD_U32, O::DS_SUB_U32, O::DS_MIN_I32, - O::DS_MAX_I32, O::DS_MIN_U32, O::DS_MAX_U32, O::DS_AND_B32, O::DS_OR_B32, - O::DS_XOR_B32, O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_ADD_U32, O::DS_SUB_U32, + O::DS_MIN_I32, O::DS_MAX_I32, O::DS_MIN_U32, O::DS_MAX_U32, + O::DS_AND_B32, O::DS_OR_B32, O::DS_XOR_B32, O::DS_READ_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase DsAtomicReturnVariants() { @@ -16534,9 +16700,10 @@ TestCase DsAtomicReturnVariants() { {10, 10, 0xfffffff0u, 0xfffffff0u, 10, 10, 0xf0f0u, 0xf000u, 0xf00fu, 10, 15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u, 99}, {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_ADD_RTN_U32, O::DS_SUB_RTN_U32, - O::DS_MIN_RTN_I32, O::DS_MAX_RTN_I32, O::DS_MIN_RTN_U32, O::DS_MAX_RTN_U32, - O::DS_AND_RTN_B32, O::DS_OR_RTN_B32, O::DS_XOR_RTN_B32, O::DS_WRXCHG_RTN_B32, - O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + O::DS_MIN_RTN_I32, O::DS_MAX_RTN_I32, O::DS_MIN_RTN_U32, + O::DS_MAX_RTN_U32, O::DS_AND_RTN_B32, O::DS_OR_RTN_B32, + O::DS_XOR_RTN_B32, O::DS_WRXCHG_RTN_B32, O::DS_READ_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase DsMiscVariants() { @@ -16582,10 +16749,17 @@ TestCase DsMiscVariants() { test.code = code; test.initial = std::vector(4, 0); test.expected = {0x40000000u, 0x40400000u, 0x12345678u, 0xabcdef01u}; - test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, - O::DS_MIN_F32, O::DS_MAX_F32, O::DS_READ_B32, - O::DS_SWIZZLE_B32, O::DS_WRITE_ADDTID_B32, O::DS_READ_ADDTID_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, + O::S_MOV_B32, + O::DS_WRITE_B32, + O::DS_MIN_F32, + O::DS_MAX_F32, + O::DS_READ_B32, + O::DS_SWIZZLE_B32, + O::DS_WRITE_ADDTID_B32, + O::DS_READ_ADDTID_B32, + O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.compute_info.threads_num[0] = 1; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -16625,8 +16799,9 @@ TestCase DsFloatMinMaxUsesSeparateCompareOperand() { test.code = code; test.initial = std::vector(2, 0); test.expected = {0x41100000u, 0x3f800000u}; - test.opcodes = {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_MIN_F32, O::DS_MAX_F32, - O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_MIN_F32, + O::DS_MAX_F32, O::DS_READ_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.compute_info.threads_num[0] = 1; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -16684,10 +16859,11 @@ TestCase BufferAtomicVariants() { {100, 15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u, 10, 10, 10, 0xfffffff0u, 10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu}, - {O::V_MOV_B32, O::BUFFER_ATOMIC_SWAP, O::BUFFER_ATOMIC_ADD, O::BUFFER_ATOMIC_SUB, - O::BUFFER_ATOMIC_SMIN, O::BUFFER_ATOMIC_UMIN, O::BUFFER_ATOMIC_SMAX, - O::BUFFER_ATOMIC_UMAX, O::BUFFER_ATOMIC_AND, O::BUFFER_ATOMIC_OR, - O::BUFFER_ATOMIC_XOR, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::BUFFER_ATOMIC_SWAP, O::BUFFER_ATOMIC_ADD, + O::BUFFER_ATOMIC_SUB, O::BUFFER_ATOMIC_SMIN, O::BUFFER_ATOMIC_UMIN, + O::BUFFER_ATOMIC_SMAX, O::BUFFER_ATOMIC_UMAX, O::BUFFER_ATOMIC_AND, + O::BUFFER_ATOMIC_OR, O::BUFFER_ATOMIC_XOR, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase BufferAtomicGlc0DoesNotReturnOldValue() { @@ -16700,11 +16876,12 @@ TestCase BufferAtomicGlc0DoesNotReturnOldValue() { AppendStoreVgpr(&code, 0, 1); AppendEnd(&code); - return {"BufferAtomicGlc0DoesNotReturnOldValue", - code, - {10, 0}, - {15, 5}, - {O::V_MOV_B32, O::BUFFER_ATOMIC_ADD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + return { + "BufferAtomicGlc0DoesNotReturnOldValue", + code, + {10, 0}, + {15, 5}, + {O::V_MOV_B32, O::BUFFER_ATOMIC_ADD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase BufferAtomicFMinExactRawGlcModes() { @@ -16770,7 +16947,8 @@ TestCase BufferAtomicFMinSpecialValues() { 0x80000000u, 0x00000000u, 0x00000000u, 0x80000001u, 0x40800000u, 0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u, 0x80000000u, 0x00000000u, 0x00000001u, 0x00000000u}, - {O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase BufferAtomicFMinContendedWorkgroup() { @@ -16787,7 +16965,8 @@ TestCase BufferAtomicFMinContendedWorkgroup() { test.code = code; test.initial = {0x42c80000u}; // 100.0 test.expected = {0x00000000u}; // min(100.0, 0.0 .. 63.0) - test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN, O::S_ENDPGM}; + test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN, + O::S_ENDPGM}; test.compute_info.threads_num[0] = 64; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -16875,7 +17054,8 @@ TestCase BufferAtomicFMaxSpecialValues() { 0x7f800000u, 0xff800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u, 0x3f800000u, 0x80000000u, 0x00000000u, 0x80000001u, 0x80000000u, 0x00000000u, 0x00000001u}, - {O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}}; } TestCase BufferAtomicFMaxContendedWorkgroup() { @@ -16892,7 +17072,8 @@ TestCase BufferAtomicFMaxContendedWorkgroup() { test.code = code; test.initial = {0xc2c80000u}; // -100.0 test.expected = {0x427c0000u}; // max(-100.0, 0.0 .. 63.0) - test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX, O::S_ENDPGM}; + test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX, + O::S_ENDPGM}; test.compute_info.threads_num[0] = 64; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -16944,8 +17125,9 @@ TestCase ImageLoadVariants() { test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, 4}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::IMAGE_LOAD_MIP, - O::IMAGE_GET_RESINFO, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, + O::IMAGE_LOAD_MIP, O::IMAGE_GET_RESINFO, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.sampled_image_rgba = image; return test; } @@ -16978,8 +17160,8 @@ TestCase DsAppendConsumeUsesEncodedLdsSelector() { code, {}, {10, 74, 0, 10}, - {O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_B32, O::DS_APPEND, - O::DS_CONSUME, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + {O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_B32, + O::DS_APPEND, O::DS_CONSUME, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } TestCase DsAppendUsesEncodedGdsSelector() { @@ -17008,12 +17190,12 @@ TestCase DsAppendUsesEncodedGdsSelector() { AppendStoreVgpr(&code, 5, 5); AppendEnd(&code); - TestCase test{ - "DsAppendGdsSelector", - code, - {}, - {10, 74, 20, 84, 40, 104}, - {O::S_MOV_B32, O::DS_APPEND, O::DS_CONSUME, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; + TestCase test{"DsAppendGdsSelector", + code, + {}, + {10, 74, 20, 84, 40, 104}, + {O::S_MOV_B32, O::DS_APPEND, O::DS_CONSUME, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; test.gds_initial = {10, 20, 30, 40}; test.expected_gds = {10, 20, 30, 40}; return test; @@ -17045,8 +17227,9 @@ TestCase DsGdsSubdwordAndAtomicWrites() { TestCase test; test.name = "DsGdsSubdwordAndAtomics"; test.code = code; - test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32, O::DS_WRITE_B8, - O::DS_WRITE_B16, O::DS_ADD_U32, O::DS_MIN_F32, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32, + O::DS_WRITE_B8, O::DS_WRITE_B16, O::DS_ADD_U32, + O::DS_MIN_F32, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -17072,7 +17255,8 @@ TestCase ImageLoadR32UintUsesIntegerSampledImage() { test.name = "ImageLoadR32UintUsesIntegerSampledImage"; test.code = code; test.expected = {0xdeadbeefu}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.sampled_image_rgba.resize(16); test.sampled_image_rgba[6] = 0xdeadbeefu; test.sampled_image_format = vk::Format::eR32Uint; @@ -17098,7 +17282,8 @@ TestCase ImageLoad1DUsesScalarCoordinate() { test.name = "ImageLoad1DUsesScalarCoordinate"; test.code = code; test.expected = {0xdeadbeefu}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.image_width = 4; test.image_height = 1; test.sampled_image_rgba = {0, 0, 0xdeadbeefu, 0}; @@ -17129,7 +17314,8 @@ TestCase ImageLoad1DArrayUsesLayerCoordinate() { test.name = "ImageLoad1DArrayUsesLayerCoordinate"; test.code = code; test.expected = {0xcafebabeu}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.image_width = 4; test.image_height = 1; test.sampled_image_rgba = {0, 0, 0, 0, 0, 0, 0xcafebabeu, 0}; @@ -17160,7 +17346,8 @@ TestCase ImageLoad1DArrayDescriptorUsesSelectedLayer() { test.name = "ImageLoad1DArrayDescriptorUsesSelectedLayer"; test.code = code; test.expected = {0xcafebabeu}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.image_width = 4; test.image_height = 1; test.sampled_image_rgba = { @@ -17204,7 +17391,8 @@ TestCase ImageLoadMipUsesVaddr2Lod2D() { test.name = "ImageLoadMipUsesVaddr2Lod2D"; test.code = code; test.expected = {0x40000000u}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD_MIP, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD_MIP, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.sampled_image_rgba_mips = {base, mip1}; return test; } @@ -17233,7 +17421,8 @@ TestCase ImageLoadMipNsaUsesSelectedAddressVgprs() { test.name = "ImageLoadMipNsaUsesSelectedAddressVgprs"; test.code = code; test.expected = {0x40a00000u}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD_MIP, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD_MIP, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.sampled_image_rgba_mips = {base, mip1}; return test; } @@ -17259,7 +17448,8 @@ TestCase ImageLoadA16UintCoordsOnGpu() { test.name = "ImageLoadA16UintCoordsOnGpu"; test.code = code; test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.sampled_image_rgba = image; test.required_spirv = {"OpShiftRightLogical", "OpBitwiseAnd"}; test.forbidden_spirv = {"UnpackHalf2x16"}; @@ -17360,9 +17550,10 @@ TestCase ImageSampleAndGather() { 0x3f800000u, 0x3f800000u, 0}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::IMAGE_GET_LOD, - O::IMAGE_GATHER4_LZ, O::IMAGE_GATHER4_LZ_O, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = { + O::V_MOV_B32, O::IMAGE_SAMPLE, O::IMAGE_GET_LOD, + O::IMAGE_GATHER4_LZ, O::IMAGE_GATHER4_LZ_O, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.sampled_image_rgba = image; return test; } @@ -17388,7 +17579,8 @@ TestCase ImageSampleA16SamplerCoordsOnGpu() { test.name = "ImageSampleA16SamplerCoordsOnGpu"; test.code = code; test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.sampled_image_rgba = image; test.required_spirv = {"UnpackHalf2x16"}; return test; @@ -17408,7 +17600,8 @@ TestCase ImageSampleOpcodeAliasUsesNormalCoords() { TestCase test; test.name = "ImageSampleOpcodeAliasUsesNormalCoords"; test.code = code; - test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.required_spirv = {"OpImageSampleExplicitLod"}; test.forbidden_spirv = {"UnpackHalf2x16"}; test.compile_only = true; @@ -17438,7 +17631,8 @@ TestCase ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu() { test.name = "ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu"; test.code = code; test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.sampled_image_rgba = image; test.required_spirv = {"UnpackHalf2x16"}; test.forbidden_spirv = {"OpBitFieldSExtract"}; @@ -17460,7 +17654,8 @@ TestCase ImageSampleA16CompareBiasRdna2AddressOrder() { TestCase test; test.name = "ImageSampleA16CompareBiasRdna2AddressOrder"; test.code = code; - test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.required_spirv = {"OpImageSampleDrefExplicitLod", "UnpackHalf2x16"}; test.compile_only = true; return test; @@ -17501,9 +17696,10 @@ TestCase ImageGatherCompareOpcodes() { TestCase test; test.name = "ImageGatherCompareOpcodes"; test.code = code; - test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_C, O::IMAGE_GATHER4_C_LZ, - O::IMAGE_GATHER4_C_O, O::IMAGE_GATHER4_C_LZ_O, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = { + O::V_MOV_B32, O::IMAGE_GATHER4_C, O::IMAGE_GATHER4_C_LZ, + O::IMAGE_GATHER4_C_O, O::IMAGE_GATHER4_C_LZ_O, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; test.required_spirv = {"OpImageDrefGather", "OpBitFieldSExtract"}; test.compile_only = true; return test; @@ -17572,14 +17768,14 @@ void CheckIndirectImageKeySwitch() { program.values->block_info.push_back({.id = 0}); auto &key = block->AppendNewInst(ValueOpcode::LaneId); - auto &image = block->AppendNewInst( - ValueOpcode::GetImageResource, - {Value(&key), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), - Value(0u), Value(0u)}); + auto &image = + block->AppendNewInst(ValueOpcode::GetImageResource, + {Value(&key), Value(0u), Value(0u), Value(0u), + Value(0u), Value(0u), Value(0u), Value(0u)}); image.SetFlags(0u); - auto &sampler = block->AppendNewInst( - ValueOpcode::GetSamplerResource, - {Value(0u), Value(0u), Value(0u), Value(0u)}); + auto &sampler = + block->AppendNewInst(ValueOpcode::GetSamplerResource, + {Value(0u), Value(0u), Value(0u), Value(0u)}); sampler.SetFlags(0u); auto &address = block->AppendNewInst( ValueOpcode::MakeImageAddress, @@ -17600,8 +17796,7 @@ void CheckIndirectImageKeySwitch() { std::memcpy(&memory_flag_bits, &memory_flags, sizeof(memory_flags)); auto &sample = block->AppendNewInst( ValueOpcode::ImageSampleRaw, - {Value(&image), Value(&sampler), Value(&address)}, - memory_flag_bits); + {Value(&image), Value(&sampler), Value(&address)}, memory_flag_bits); auto &sample_x = block->AppendNewInst(ValueOpcode::CompositeExtractU32x4, {Value(&sample), Value(0u)}); block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(&sample_x)}); @@ -17630,15 +17825,14 @@ void CheckIndirectImageKeySwitch() { program.info.sampled_pairs.push_back({0u, 0u, 0x10f0u}); std::string error; - Require(name, "binding layout", - AllocateBindings(program, {}, &error), error.c_str()); + Require(name, "binding layout", AllocateBindings(program, {}, &error), + error.c_str()); ResourceSnapshot snapshot{}; DescriptorValue descriptor{}; descriptor.dword_count = 8; descriptor.dwords[0] = 0x20u; descriptor.dwords[1] = - static_cast(Prospero::BufferFormat::k32_32_32_32Float) - << 20u; + static_cast(Prospero::BufferFormat::k32_32_32_32Float) << 20u; descriptor.dwords[2] = 3u | (3u << 14u); descriptor.dwords[3] = DstSel(4, 5, 6, 7) | @@ -17697,9 +17891,9 @@ TestCase ImageStoreMipSelectsPpsa01340Descriptor() { test.name = "ImageStoreMipSelectsPpsa01340Descriptor"; test.code = std::move(code); test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE_MIP, O::S_ENDPGM}; - const std::array descriptor{ - 0x0294dc00u, 0xc4700000u, 0x00b3c13fu, 0x91b31facu, - 0x00000000u, 0x00700030u, 0xb07b0000u, 0x0002ac3cu}; + const std::array descriptor{0x0294dc00u, 0xc4700000u, 0x00b3c13fu, + 0x91b31facu, 0x00000000u, 0x00700030u, + 0xb07b0000u, 0x0002ac3cu}; std::copy(descriptor.begin(), descriptor.end(), test.user_data.begin()); test.has_user_data = true; test.compile_only = true; @@ -17943,8 +18137,9 @@ TestCase ImageAtomicVariants() { test.name = "ImageAtomicVariants"; test.code = code; test.expected = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu}; - test.opcodes = {O::V_MOV_B32, O::IMAGE_ATOMIC_ADD, O::IMAGE_ATOMIC_UMIN, - O::IMAGE_ATOMIC_AND, O::IMAGE_ATOMIC_OR, O::IMAGE_ATOMIC_XOR, + test.opcodes = {O::V_MOV_B32, O::IMAGE_ATOMIC_ADD, + O::IMAGE_ATOMIC_UMIN, O::IMAGE_ATOMIC_AND, + O::IMAGE_ATOMIC_OR, O::IMAGE_ATOMIC_XOR, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.required_spirv = {"OpImageTexelPointer", "R32ui"}; test.forbidden_spirv = {"OpTypeSampler", "OpTypeSampledImage", @@ -18007,8 +18202,8 @@ GraphicsCase GraphicsInterpolationExport() { return {"GraphicsInterpolationExport", code, {0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u}, - {O::V_INTERP_P1_F32, O::V_INTERP_P2_F32, O::V_INTERP_MOV_F32, O::V_MOV_B32, - O::EXP, O::S_ENDPGM}}; + {O::V_INTERP_P1_F32, O::V_INTERP_P2_F32, O::V_INTERP_MOV_F32, + O::V_MOV_B32, O::EXP, O::S_ENDPGM}}; } GraphicsCase GraphicsFlatInterpolatorExport() { @@ -18055,8 +18250,8 @@ GraphicsCase GraphicsDsAddtidScratchExport() { return {"GraphicsDsAddtidScratchExport", code, {0x3f000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u}, - {O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_ADDTID_B32, O::DS_READ_ADDTID_B32, - O::EXP, O::S_ENDPGM}}; + {O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_ADDTID_B32, + O::DS_READ_ADDTID_B32, O::EXP, O::S_ENDPGM}}; } GraphicsCase GraphicsDirectSgprPushConstantExport() { @@ -18217,8 +18412,8 @@ GraphicsCase GraphicsBranchPathFinalVmExportDiscardsInactiveExec() { return {"GraphicsBranchPathFinalVmExportDiscardsInactiveExec", code, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, - {O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_CMP_EQ_U32, O::S_CBRANCH_SCC1, - O::S_MOV_B32, O::S_ENDPGM}}; + {O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_CMP_EQ_U32, + O::S_CBRANCH_SCC1, O::S_MOV_B32, O::S_ENDPGM}}; } TestCase MultipleWorkitemsGlobalId() { @@ -18238,8 +18433,8 @@ TestCase MultipleWorkitemsGlobalId() { test.name = "MultipleWorkitemsGlobalId"; test.code = code; test.expected = {64, 65, 66, 67, 68, 69, 70, 71}; - test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, - O::S_ENDPGM}; + test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.compute_info.threads_num[0] = 4; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -18267,7 +18462,7 @@ TestCase DispatcherIrreducibleControlFlow() { test.code = code; test.expected = {7}; test.opcodes = {O::S_CBRANCH_SCC1, O::V_MOV_B32, O::BUFFER_STORE_DWORD, - O::S_CBRANCH_SCC0, O::S_BRANCH, O::S_ENDPGM}; + O::S_CBRANCH_SCC0, O::S_BRANCH, O::S_ENDPGM}; return test; } @@ -18459,6 +18654,10 @@ std::vector MakeCases() { AddCase(DsAppendUsesEncodedGdsSelector); AddCase(DsGdsSubdwordAndAtomicWrites); AddCase(DsReadWrite2Variants); + AddCase(DsWideReadSnapshotsOverlappingAddress); + AddCase(DsReadWrite2EqualOffsetsUseData0); + AddCase(DsWideLdsPartialBounds); + AddCase(DsWideGdsPartialBounds); AddCase(DsAtomicNoReturnVariants); AddCase(DsAtomicReturnVariants); AddCase(DsMiscVariants); @@ -19129,8 +19328,8 @@ void CheckSampledColorViews() { {"sampled-invalid-selector", "sampled-incompatible-format", "sampled-invalid-high", "sampled-depth-format", "sampled-depth-swizzle", "storage-incompatible-format", "storage-kind", "storage-no-write", - "storage-nonuint-atomic", "storage-compare", - "storage-dimension", "volume-mip-count", "volume-slice-range"}) { + "storage-nonuint-atomic", "storage-compare", "storage-dimension", + "volume-mip-count", "volume-slice-range"}) { std::string command = std::string("\"") + path + "\" --image-view-death " + kind; std::vector mutable_command(command.begin(), command.end()); @@ -20087,12 +20286,10 @@ void CheckBasicStorageTextureDescriptor() { Require("BasicStorageTexture", "PPSA01340 mip-range descriptor", mip_range.Base40() == 0x294dc0000ull && mip_range.Width5() + 1u == 1280 && - mip_range.Height5() + 1u == 720 && - mip_range.Depth() + 1u == 1 && + mip_range.Height5() + 1u == 720 && mip_range.Depth() + 1u == 1 && mip_range.BaseLevel() == 1 && mip_range.LastLevel() == 3 && mip_range.MaxMip() == 3 && - mip_range.Format() == - Prospero::BufferFormat::k16_16_16_16Float && + mip_range.Format() == Prospero::BufferFormat::k16_16_16_16Float && mip_range.TileMode() == Prospero::TileMode::kRenderTarget && mip_range.DstSelXYZW() == DstSel(4, 5, 6, 7), "PPSA01340 mip-range storage descriptor fixture is malformed"); @@ -20415,14 +20612,27 @@ void CheckBasicStorageTextureDescriptor() { Require("BasicStorageTexture", "host", GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, "GetModuleFileName failed"); - for (const char *kind : - {"resource", "type", "standard256b-volume", - "base-mip-out-of-resource", "dynamic-mip-out-of-resource", - "inverted-mip-range", "swizzle", "linear-rgb1-read", "bgra-read", - "r16-float-read", "r8-unorm-read", "yzwx-read", "reserved-swizzle", - "array-base-out-of-range", "reserved", "uint-format", - "uint-resource-float-format", "atomic-format", "depth-tile-read", - "depth-tile-extent", "depth-tile-fmask"}) { + for (const char *kind : {"resource", + "type", + "standard256b-volume", + "base-mip-out-of-resource", + "dynamic-mip-out-of-resource", + "inverted-mip-range", + "swizzle", + "linear-rgb1-read", + "bgra-read", + "r16-float-read", + "r8-unorm-read", + "yzwx-read", + "reserved-swizzle", + "array-base-out-of-range", + "reserved", + "uint-format", + "uint-resource-float-format", + "atomic-format", + "depth-tile-read", + "depth-tile-extent", + "depth-tile-fmask"}) { std::string command = std::string("\"") + path + "\" --storage-texture-descriptor-death " + kind; std::vector mutable_command(command.begin(), command.end()); @@ -21330,8 +21540,7 @@ void CheckReferenceClockScale() { void CheckClipControlDepthClipState() { HW::ClipControl clip; - Require("ClipControlDepthClipState", "default", - clip.IsZClipEnabled(), + Require("ClipControlDepthClipState", "default", clip.IsZClipEnabled(), "default paired Z clipping was not enabled"); clip.min_z_clip_disable = true; @@ -21341,13 +21550,11 @@ void CheckClipControlDepthClipState() { clip.min_z_clip_disable = false; clip.max_z_clip_disable = true; - Require("ClipControlDepthClipState", "asymmetric far", - !clip.IsZClipEnabled(), + Require("ClipControlDepthClipState", "asymmetric far", !clip.IsZClipEnabled(), "far-plane disable did not disable paired host Z clipping"); clip.min_z_clip_disable = true; - Require("ClipControlDepthClipState", "both disabled", - !clip.IsZClipEnabled(), + Require("ClipControlDepthClipState", "both disabled", !clip.IsZClipEnabled(), "paired Z-clip disable was not represented"); std::printf("[host] %-32s ok\n", "ClipControlDepthClipState"); } diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index b962118aa..6186cf6c0 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -976,7 +976,7 @@ void TestSpirvRequirementsAnalysis() { block->AppendNewInst(ValueOpcode::ImageGatherRaw, {Value(0u), Value(0u), Value(0u)}); auto &shared = block->AppendNewInst(ValueOpcode::LoadSharedU32, - {Value(0u), Value(0u), Value(true)}); + {Value(0u), Value(true)}); shared.SetFlags(MemoryFlags{.index = 0}); auto &export_value = block->AppendNewInst(ValueOpcode::SetAttribute, {Value(0u), Value(true)}); @@ -1095,11 +1095,9 @@ void TestNativeSubgroupPolicy() { } else if (opcode == ValueOpcode::Ballot) { block->AppendNewInst(opcode, {Value(true)}); } else if (opcode == ValueOpcode::DataAppend) { - auto &resource = block->AppendNewInst(ValueOpcode::GetLdsResource); program.values->memory_info.push_back({.kind = ResourceKind::Lds}); - auto &append = - block->AppendNewInst(opcode, {Value(&resource), Value(0u), - Value(true), Value(1u), Value(0u)}); + auto &append = block->AppendNewInst( + opcode, {Value(0u), Value(true), Value(1u), Value(0u)}); append.SetFlags(MemoryFlags{.index = 0}); } std::string error; @@ -1748,6 +1746,7 @@ void TestNewShaderRecompilerScalarVectorAlu() { Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); CheckSpirvBinaryValidates(result.spirv); + } void TestNewShaderRecompilerVop3LaneReadDestinationEncoding() { @@ -4102,8 +4101,6 @@ void TestNewShaderRecompilerMemoryFamilyLowering() { "MIMG sample did not lower to IR"); Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad"); Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); Check(SpirvContainsOpcode(result.spirv, 95), @@ -5329,8 +5326,6 @@ void TestNewShaderRecompilerVintrpLowering() { Check(ProgramInputCount(result.program, ShaderRecompiler::IR::StageInputKind::Parameter) == 2, "VINTRP pixel shader did not reflect parameter inputs"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad"); Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); Check(SpirvContainsOpcode(result.spirv, 81), @@ -5608,8 +5603,6 @@ void TestNewShaderRecompilerWideMemoryLowering() { "global segment load did not lower with global metadata"); Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad"); Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); Check(SpirvContainsOpcode(result.spirv, 194), @@ -5807,8 +5800,6 @@ void TestNewShaderRecompilerMubufFormatLowering() { "MUBUF formatted stores were not preserved as native-width operations"); Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad"); Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); CheckSpirvBinaryValidates(result.spirv); @@ -6283,18 +6274,60 @@ void TestNewShaderRecompilerDsReadWrite2Lowering() { 0xbf810000u, }; + ShaderRecompiler::Decoder::Program decoded; + ShaderRecompiler::CFG::Graph graph; + ShaderRecompiler::IR::Program lowered; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, + &error) && + ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) && + ShaderRecompiler::IR::LowerProgram( + decoded, graph, ShaderType::Compute, 64, lowered, &error), + error.c_str()); + uint32_t pair_reads = 0; + uint32_t pair_writes = 0; + for (const auto &block : lowered.blocks) { + for (const auto &inst : block.instructions) { + pair_reads += inst.op == ShaderRecompiler::IR::Opcode::DsRead2B32; + pair_writes += inst.op == ShaderRecompiler::IR::Opcode::DsWrite2B32; + } + } + Check(pair_reads == 3u && pair_writes == 1u, + "DS read/write2 was split before typed SSA"); + ShaderRecompiler::IR::ValueProgram typed; + ShaderComputeInputInfo compute{}; + Check(ShaderRecompiler::Frontend::TranslateProgram(lowered, typed, nullptr, + nullptr, &compute, &error), + error.c_str()); + uint32_t scalar_loads = 0; + uint32_t vector_loads = 0; + uint32_t scalar_stores = 0; + for (const auto *block : typed.blocks) { + for (const auto &inst : *block) { + scalar_loads += + inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::LoadSharedU32; + vector_loads += inst.GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::LoadSharedU32x2; + scalar_stores += + inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::WriteSharedU32; + } + } + Check(scalar_loads == 4u && vector_loads == 2u && scalar_stores == 2u, + "DS read/write2 did not use the native scalar/x2 transaction shape"); + auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; ShaderRecompiler::CompileResult result; - std::string error; Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "ds_write2_b32"), + Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE2_B32"), "new decoder did not decode old-backed DS write2"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b32"), + Check(Common::ContainsStr(result.decoded_dump, "DS_READ2_B32"), "new decoder did not decode old-backed DS read2"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b64"), + Check(Common::ContainsStr(result.decoded_dump, "DS_READ2ST64_B32"), + "new decoder did not preserve the DS read2st64 opcode identity"); + Check(Common::ContainsStr(result.decoded_dump, "DS_READ2_B64"), "new decoder did not decode old-backed DS read2 b64"); Check(Common::ContainsStr(result.decoded_dump, "offset=4"), "DS write2 decode did not scale offset0 to bytes"); @@ -6316,24 +6349,6 @@ void TestNewShaderRecompilerDsReadWrite2Lowering() { "DS read/write2 decode did not preserve two-dword metadata"); Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), "DS read2 b64 decode did not preserve four-dword metadata"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v60"), - "DS write2 did not lower first dword through shared LDS store IR"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v61"), - "DS write2 did not lower second dword through shared LDS store IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v70"), - "DS read2 did not lower first dword through shared LDS load IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v71"), - "DS read2 did not lower second dword through shared LDS load IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v72"), - "DS read2 st64 did not lower first dword through shared LDS load IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v73"), - "DS read2 st64 did not lower second dword through shared LDS load IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v80"), - "DS read2 b64 did not lower first dword through shared LDS load IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v83"), - "DS read2 b64 did not lower fourth dword through shared LDS load IR"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad"); Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); Check(SpirvContainsOpcode(result.spirv, 65), @@ -6420,49 +6435,80 @@ void TestNewShaderRecompilerDsWideAndAtomicLowering() { 0xbf810000u, }; + ShaderRecompiler::Decoder::Program decoded; + ShaderRecompiler::CFG::Graph graph; + ShaderRecompiler::IR::Program lowered; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, + &error) && + ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error) && + ShaderRecompiler::IR::LowerProgram( + decoded, graph, ShaderType::Compute, 64, lowered, &error), + error.c_str()); + uint32_t native_reads = 0; + uint32_t native_writes = 0; + for (const auto &block : lowered.blocks) { + for (const auto &inst : block.instructions) { + native_reads += inst.op == ShaderRecompiler::IR::Opcode::DsReadB32; + native_writes += inst.op == ShaderRecompiler::IR::Opcode::DsWriteB32; + } + } + Check(native_reads == 3u && native_writes == 3u, + "wide DS transfers were split before typed SSA"); + ShaderRecompiler::IR::ValueProgram typed; + ShaderComputeInputInfo compute{}; + Check(ShaderRecompiler::Frontend::TranslateProgram(lowered, typed, nullptr, + nullptr, &compute, &error), + error.c_str()); + std::array load_widths{}; + std::array store_widths{}; + for (const auto *block : typed.blocks) { + for (const auto &inst : *block) { + using ShaderRecompiler::IR::ValueOpcode; + if (inst.GetOpcode() == ValueOpcode::LoadSharedU32x2) + load_widths[0]++; + if (inst.GetOpcode() == ValueOpcode::LoadSharedU32x3) + load_widths[1]++; + if (inst.GetOpcode() == ValueOpcode::LoadSharedU32x4) + load_widths[2]++; + if (inst.GetOpcode() == ValueOpcode::WriteSharedU32x2) + store_widths[0]++; + if (inst.GetOpcode() == ValueOpcode::WriteSharedU32x3) + store_widths[1]++; + if (inst.GetOpcode() == ValueOpcode::WriteSharedU32x4) + store_widths[2]++; + } + } + Check(load_widths == std::array{1u, 1u, 1u} && + store_widths == std::array{1u, 1u, 1u}, + "wide DS transfers retained scalar sibling operations"); + auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; ShaderRecompiler::CompileResult result; - std::string error; Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "ds_write_b64"), + Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B64"), "new decoder did not decode DS b64 write"); - Check(Common::ContainsStr(result.decoded_dump, "ds_write_b96"), + Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B96"), "new decoder did not decode DS b96 write"); - Check(Common::ContainsStr(result.decoded_dump, "ds_write_b128"), + Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B128"), "new decoder did not decode DS b128 write"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_b64"), + Check(Common::ContainsStr(result.decoded_dump, "DS_READ_B64"), "new decoder did not decode DS b64 read"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_b96"), + Check(Common::ContainsStr(result.decoded_dump, "DS_READ_B96"), "new decoder did not decode DS b96 read"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_b128"), + Check(Common::ContainsStr(result.decoded_dump, "DS_READ_B128"), "new decoder did not decode DS b128 read"); - Check(Common::ContainsStr(result.decoded_dump, "ds_min_u32"), + Check(Common::ContainsStr(result.decoded_dump, "DS_MIN_U32"), "new decoder did not decode DS min atomic"); - Check(Common::ContainsStr(result.decoded_dump, "ds_max_u32"), + Check(Common::ContainsStr(result.decoded_dump, "DS_MAX_U32"), "new decoder did not decode DS max atomic"); - Check(Common::ContainsStr(result.decoded_dump, "ds_or_b32"), + Check(Common::ContainsStr(result.decoded_dump, "DS_OR_B32"), "new decoder did not decode DS or atomic"); Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), "new decoder did not expose DS wide width metadata"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v10"), - "DS b64 write did not expand to first dword store"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v19"), - "DS b128 write did not expand to last dword store"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v20"), - "DS b64 read did not expand to first dword load"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v31"), - "DS b128 read did not expand to last dword load"); - Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 null, v32"), - "DS min atomic did not lower to shared atomic IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 null, v33"), - "DS max atomic did not lower to shared atomic IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 null, v34"), - "DS or atomic did not lower to shared atomic IR"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad"); Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); Check(SpirvContainsOpcode(result.spirv, 237), @@ -6570,6 +6616,7 @@ void TestNewShaderRecompilerDsAddtidLowering() { Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); CheckSpirvBinaryValidates(result.spirv); + } void TestNewShaderRecompilerDsFloatMinMaxLowering() { @@ -7838,17 +7885,16 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { auto program = result.program; IR::IREmitter definition(program.values->blocks[0]); - const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU32x3, - {IR::Value(1u), IR::Value(2u), - IR::Value(3u)}); + const auto vector = + definition.Emit(IR::ValueOpcode::CompositeConstructU32x3, + {IR::Value(1u), IR::Value(2u), IR::Value(3u)}); IR::IREmitter use(program.values->blocks[1]); - use.Emit(IR::ValueOpcode::CompositeExtractU32x3, - {vector, IR::Value(2u)}); + use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)}); Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str()); std::vector spirv; - Check(Spirv::EmitProgram(program, result.resources, options.input_info, - spirv, &error), + Check(Spirv::EmitProgram(program, result.resources, options.input_info, spirv, + &error), error.c_str()); CheckSpirvBinaryValidates(spirv); const auto before = MeasureSpirv(result.spirv); @@ -8533,7 +8579,8 @@ void TestWqmConstantPropagation() { "WqmU64 constant propagation crossed a quad or dword boundary"); } - const auto check_invalid_signature = [](ValueOpcode opcode, Value valid, Value invalid) { + const auto check_invalid_signature = [](ValueOpcode opcode, Value valid, + Value invalid) { ValueProgram malformed; malformed.block_storage.push_back(std::make_unique()); malformed.blocks.push_back(malformed.block_storage.back().get()); @@ -8547,7 +8594,8 @@ void TestWqmConstantPropagation() { "value IR accepted a WQM operand that violates opcode metadata"); }; check_invalid_signature(ValueOpcode::WqmU64, Value(uint64_t{1}), Value(true)); - check_invalid_signature(ValueOpcode::WqmMask, Value(true), Value(uint64_t{1})); + check_invalid_signature(ValueOpcode::WqmMask, Value(true), + Value(uint64_t{1})); } void TestNativeWideValueValidation() { @@ -8568,8 +8616,8 @@ void TestNativeWideValueValidation() { IREmitter ir(program.blocks.front()); const auto resource = ir.Emit(ValueOpcode::GetBufferResource, {Value(0u), Value(0u), Value(0u), Value(0u)}); - ir.Emit(opcode, - {resource, Value(0u), Value(0u), Value(0u), Value(true)}, flags); + ir.Emit(opcode, {resource, Value(0u), Value(0u), Value(0u), Value(true)}, + flags); }; const auto check_rejected = [](const ValueProgram &program, const char *expected) { @@ -8585,8 +8633,7 @@ void TestNativeWideValueValidation() { memory.kind = ResourceKind::Buffer; memory.data_dwords = 3; program.memory_info.push_back(memory); - append_load(program, ValueOpcode::LoadBufferU32x3, - MemoryFlags{.index = 1}); + append_load(program, ValueOpcode::LoadBufferU32x3, MemoryFlags{.index = 1}); check_rejected(program, "invalid memory-info index"); } { @@ -8624,6 +8671,43 @@ void TestNativeWideValueValidation() { ir.Emit(ValueOpcode::CompositeExtractU32x3, {vector, Value(3u)}); check_rejected(program, "invalid component index"); } + const auto append_shared = [](ValueProgram &program, ValueOpcode opcode, + MemoryFlags flags) { + IREmitter ir(program.blocks.front()); + ir.Emit(opcode, {Value(0u), Value(true)}, flags); + }; + { + auto program = make_program(); + append_shared(program, ValueOpcode::LoadSharedU32, MemoryFlags{.index = 1}); + check_rejected(program, "invalid memory-info index"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::Buffer; + program.memory_info.push_back(memory); + append_shared(program, ValueOpcode::LoadSharedU32, {}); + check_rejected(program, "invalid shared-memory metadata"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::Gds; + memory.resource = 1; + program.memory_info.push_back(memory); + append_shared(program, ValueOpcode::LoadSharedU32, {}); + check_rejected(program, "invalid shared-memory metadata"); + } + { + auto program = make_program(); + MemoryInfo memory; + memory.kind = ResourceKind::Lds; + memory.data_dwords = 2; + memory.component_count = 2; + program.memory_info.push_back(memory); + append_shared(program, ValueOpcode::LoadSharedU32x3, {}); + check_rejected(program, "inconsistent shared-memory width"); + } } void TestNewShaderRecompilerZeroInitialRegisterState() { @@ -8800,7 +8884,7 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { const uint32_t native_wqm_shader[] = { EncodeSop1(0x0a, 2, 0), // s_wqm_b64 s[2:3], s[0:1] EncodeVop1(0x01, 0, 2), // v_mov_b32 v0, s2 - EncodeExp0(0x0c, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01), + EncodeExp0(0x0c, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01), }; auto native_options = options; native_options.lane_mask_mode = ShaderLaneMaskMode::NativeWave; @@ -8814,7 +8898,8 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { "native scalar WQM retained frontend expansion or subgroup machinery"); Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 2u && SpirvInstructionOpcodeCount(result.spirv, 194u) == 4u, - "native scalar WQM emitter was folded away or did not use the compact path"); + "native scalar WQM emitter was folded away or did not use the compact " + "path"); const uint32_t native_special_wqm_shader[] = { EncodeSop1(0x04, 126, 0), // s_mov_b64 exec, s[0:1] @@ -8823,7 +8908,7 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { EncodeSop1(0x0a, 106, 106), // s_wqm_b64 vcc, vcc EncodeVop1(0x01, 0, 126), // v_mov_b32 v0, exec_lo EncodeVop1(0x01, 1, 106), // v_mov_b32 v1, vcc_lo - EncodeExp0(0x0c, 0x3), EncodeExp1(0, 1, 0, 0), EncodeSopp(0x01), + EncodeExp0(0x0c, 0x3), EncodeExp1(0, 1, 0, 0), EncodeSopp(0x01), }; Check(ShaderRecompiler::TryRecompile(native_special_wqm_shader, native_options, result, &error), @@ -10406,6 +10491,19 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { empty_metrics.type_pointers == 0u, "empty shader emitted unused core types or pointers"); + const uint32_t dead_gds[] = { + EncodeDs0(0x36, 0) | (1u << 17u), + EncodeDs1(0, 0, 1), + EncodeSopp(0x01), + }; + const auto dead_gds_result = + compile("dead-gds", dead_gds, + {.words = 64, .instructions = 20, .labels = 4, .branches = 3}); + Check(ShaderRecompiler::IR::FindBinding( + dead_gds_result.program.bindings, + ShaderRecompiler::IR::DescriptorBindingKind::Gds) == nullptr, + "dead GDS load retained a descriptor binding"); + const uint32_t structured_phi[] = { EncodeSMovB32(0, 128), // s0 = 0 EncodeSopc(0x0a, 0, 129), // loop: s_cmp_lt_u32 s0, 1 @@ -10494,6 +10592,83 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { scalar_stores == 0u, "wide buffer fixture retained scalar sibling memory operations"); + const uint32_t wide_lds[] = { + EncodeDs0(0xff, 0), EncodeDs1(0, 0, 4), // ds_read_b128 v[0:3], v4 + EncodeDs0(0xdf, 16), EncodeDs1(0, 0, 4), // ds_write_b128 v[0:3], v4 + EncodeSopp(0x01), + }; + const auto wide_lds_result = compile("wide-lds", wide_lds, + {.words = 548, + .instructions = 156, + .variables = 1, + .workgroup_variables = 1, + .loads = 4, + .stores = 4, + .phis = 5, + .labels = 34, + .selection_merges = 10, + .branches = 23, + .conditional_branches = 10}); + const auto wide_lds_metrics = MeasureSpirv(wide_lds_result.spirv); + Check(wide_lds_metrics.workgroup_variables == 1u && + wide_lds_metrics.runtime_arrays == 0u && + wide_lds_metrics.array_lengths == 0u, + "native wide LDS fixture emitted the wrong storage topology"); + + const uint32_t wide_gds[] = { + EncodeDs0(0xff, 0) | (1u << 17u), + EncodeDs1(0, 0, 4), + EncodeDs0(0xdf, 16) | (1u << 17u), + EncodeDs1(0, 0, 4), + EncodeSopp(0x01), + }; + const auto wide_gds_result = compile("wide-gds", wide_gds, + {.words = 577, + .instructions = 162, + .runtime_arrays = 1, + .variables = 1, + .loads = 4, + .stores = 4, + .array_lengths = 1, + .phis = 5, + .labels = 34, + .selection_merges = 10, + .branches = 23, + .conditional_branches = 10}); + const auto wide_gds_metrics = MeasureSpirv(wide_gds_result.spirv); + Check(wide_gds_metrics.runtime_arrays == 1u && + wide_gds_metrics.workgroup_variables == 0u && + wide_gds_metrics.array_lengths == 1u, + "native wide GDS fixture did not share one entry-dominating length"); + Check(ShaderRecompiler::IR::FindBinding( + wide_gds_result.program.bindings, + ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr, + "live native wide GDS access did not allocate its descriptor binding"); + const auto count_shared = [](const ShaderRecompiler::CompileResult &result, + ShaderRecompiler::IR::ValueOpcode opcode) { + uint32_t count = 0; + for (const auto *block : result.program.values->blocks) { + count += std::ranges::count(*block, opcode, + &ShaderRecompiler::IR::Inst::GetOpcode); + } + return count; + }; + for (const auto *result : {&wide_lds_result, &wide_gds_result}) { + Check(count_shared(*result, + ShaderRecompiler::IR::ValueOpcode::LoadSharedU32x4) == + 1u && + count_shared( + *result, + ShaderRecompiler::IR::ValueOpcode::WriteSharedU32x4) == 1u && + count_shared(*result, + ShaderRecompiler::IR::ValueOpcode::LoadSharedU32) == + 0u && + count_shared(*result, + ShaderRecompiler::IR::ValueOpcode::WriteSharedU32) == + 0u, + "native wide shared fixture retained scalar sibling operations"); + } + const uint32_t wqm[] = { EncodeVopc(0xc1, 5 + 256, 8), // v_cmp_lt_u32 vcc, v5, v8 EncodeSop1(0x0a, 2, 106), // s_wqm_b64 s[2:3], vcc @@ -10563,6 +10738,8 @@ int main() { TestNewShaderRecompilerNativeWideBufferIr(); TestNewShaderRecompilerMubufFormatLowering(); TestNewShaderRecompilerTypedBufferLowering(); + TestNewShaderRecompilerDsReadWrite2Lowering(); + TestNewShaderRecompilerDsWideAndAtomicLowering(); TestNewShaderRecompilerCapturedVop1SdwaByteConvert(); TestNewShaderRecompilerScalarMemoryBindingDomains(); // Opcode semantics and optimized direct SPIR-V are exercised by