diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 9577b33e1..deede10ea 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -741,30 +741,58 @@ uint32_t FormattedOutOfBoundsValue(ValueEmitContext& ctx, const IR::MemoryInfo& return ConstructU32Composite(ctx.state, components, values); } -uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { - return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - const auto info = Format::GetFormatInfo(BufferFormat(ctx, mem)); - if (info.type == Format::ComponentType::Unknown) { - return LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource); - } - const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info); - const auto value = EmitValueOrDefaultIfCondition( - ctx.state, plan.in_bounds, TypeU32(ctx.state), - FormattedOutOfBoundsValue(ctx, mem, plan, 1u), - [&]() { return LoadFormattedInBounds(ctx, mem, plan, 0u); }); - if (mem.data_bits != 16u) { - return value; +uint32_t LoadFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst, + const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, + const Format::BufferFormatInfo& info, uint32_t components) { + auto& state = ctx.state; + const auto count = mem.data_bits == 16u ? mem.component_count : components; + const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info); + const auto value = EmitValueOrDefaultIfCondition( + state, plan.in_bounds, TypeU32Composite(state, count), + FormattedOutOfBoundsValue(ctx, mem, plan, count), [&]() { + std::array values {}; + for (uint32_t component = 0; component < count; component++) { + values[component] = LoadFormattedInBounds(ctx, mem, plan, component); + } + return ConstructU32Composite(state, count, values); + }); + if (mem.data_bits != 16u) return value; + const bool integer = info.type == Format::ComponentType::Uint || + info.type == Format::ComponentType::Sint; + const auto scalar = [&](uint32_t component) { + if (component >= count) return integer ? ConstantU32(state, 0u) : ConstantF32(state, 0); + auto result = value; + if (count != 1u) { + result = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), result, value, + component); } if (info.type == Format::ComponentType::Uint) { - return EmitUMin32(ctx.state, value, ConstantU32(ctx.state, 0xffffu)); + return EmitUMin32(state, result, ConstantU32(state, 0xffffu)); } if (info.type == Format::ComponentType::Sint) { - const auto lower = EmitSMax32(ctx.state, value, ConstantU32(ctx.state, 0xffff8000u)); - return EmitSMin32(ctx.state, lower, ConstantU32(ctx.state, 0x7fffu)); + const auto lower = EmitSMax32(state, result, ConstantU32(state, 0xffff8000u)); + return EmitSMin32(state, lower, ConstantU32(state, 0x7fffu)); } - return EmitConvertF16F32(ctx.state, EmitBitCastF32U32(ctx.state, value)); - }); + return EmitBitCastF32U32(state, result); + }; + std::array packed {}; + for (uint32_t word = 0; word < components; word++) { + const auto low = scalar(word * 2u); + const auto high = scalar(word * 2u + 1u); + if (integer) { + packed[word] = Binary( + state, spv::OpBitwiseOr, TypeU32(state), + Binary(state, spv::OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, 0xffffu)), + Binary(state, spv::OpShiftLeftLogical, TypeU32(state), high, ConstantU32(state, 16u))); + } else { + const auto pair = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 2), pair, + low, high); + packed[word] = EmitPackHalf2x16(state, pair); + } + } + return ConstructU32Composite(state, components, packed); } void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst, @@ -1042,10 +1070,13 @@ uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) { uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) { auto& state = ctx.state; + const auto mem = ctx.Memory(inst); + if (mem.kind == IR::ResourceKind::IndirectBuffer && mem.formatted) { + return LoadIndirectFormattedX(ctx, inst); + } return EmitValueOrDefaultIfCondition( state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() { - const auto mem = ctx.Memory(inst); if (mem.kind == IR::ResourceKind::IndirectBuffer) { return LoadIndirectBuffer(ctx, inst, components); } @@ -1053,16 +1084,7 @@ uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compon const auto info = Format::GetFormatInfo( mem.formatted ? BufferFormat(ctx, mem) : Prospero::BufferFormat::kInvalid); if (info.type != Format::ComponentType::Unknown) { - const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info); - return EmitValueOrDefaultIfCondition( - state, plan.in_bounds, TypeU32Composite(state, components), - FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() { - std::array values {}; - for (uint32_t component = 0; component < components; component++) { - values[component] = LoadFormattedInBounds(ctx, mem, plan, component); - } - return ConstructU32Composite(state, components, values); - }); + return LoadFormattedPrepared(ctx, inst, mem, resource, info, components); } std::array values {}; for (uint32_t component = 0; component < components; component++) { @@ -1422,7 +1444,8 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) { if (mem.kind == IR::ResourceKind::FlatLocal) value = LoadLocalFlat(ctx, inst); else if (buffer_components > 1u || - (buffer_components == 1u && mem.kind == IR::ResourceKind::IndirectBuffer && !mem.formatted)) + (op == IR::ValueOpcode::LoadBufferU32 && + (mem.kind == IR::ResourceKind::IndirectBuffer || mem.formatted))) value = LoadBuffer(ctx, inst, buffer_components); else if (shared_components > 1u) value = LoadWideShared(ctx, inst, shared_components); @@ -1432,9 +1455,6 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) { mem.kind != IR::ResourceKind::Scratch) value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1), address_info.data_bits, mem.coherent); - else if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted) - value = mem.kind == IR::ResourceKind::IndirectBuffer ? LoadIndirectFormattedX(ctx, inst) - : FormattedLoad(ctx, inst, mem); else if (inst.GetType() == IR::Type::U8) value = LoadSubword(ctx, inst, mem, 8, false); else if (inst.GetType() == IR::Type::U16) diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index e38b1c3ec..4cb15443d 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -66,6 +66,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = { {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32}, {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32}, {0x80u, Opcode::BUFFER_LOAD_FORMAT_D16_X, 1, 16, false, false, true}, + {0x83u, Opcode::BUFFER_LOAD_FORMAT_D16_XYZW, 2, 16, false, false, true}, {0x84u, Opcode::BUFFER_STORE_FORMAT_D16_X, 1, 16, false, false, true}, }; @@ -304,6 +305,9 @@ void DecodeMubuf(uint32_t pc, std::span code, uint32_t word_inde } DecodeVectorGpr(vdata, inst.dst); + if (inst.opcode == Opcode::BUFFER_LOAD_FORMAT_D16_XYZW) { + inst.data_components = 4u; + } if (inst.opcode == Opcode::BUFFER_LOAD_FORMAT_D16_X || inst.opcode == Opcode::BUFFER_LOAD_SHORT_D16 || inst.opcode == Opcode::BUFFER_LOAD_SHORT_D16_HI) { diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index d3b0f776c..060eb9490 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -631,6 +631,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::BUFFER_LOAD_FORMAT_XYZ: case Opcode::BUFFER_LOAD_FORMAT_XYZW: case Opcode::BUFFER_LOAD_FORMAT_D16_X: + case Opcode::BUFFER_LOAD_FORMAT_D16_XYZW: case Opcode::BUFFER_STORE_FORMAT_X: case Opcode::BUFFER_STORE_FORMAT_XY: case Opcode::BUFFER_STORE_FORMAT_XYZ: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index da8ea5443..649d469a3 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -547,6 +547,7 @@ enum class Opcode { BUFFER_LOAD_FORMAT_XYZ, BUFFER_LOAD_FORMAT_XYZW, BUFFER_LOAD_FORMAT_D16_X, + BUFFER_LOAD_FORMAT_D16_XYZW, BUFFER_STORE_FORMAT_X, BUFFER_STORE_FORMAT_XY, BUFFER_STORE_FORMAT_XYZ, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index f53be76aa..bda91a11a 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -987,6 +987,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ: case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW: case Decoder::Opcode::BUFFER_LOAD_FORMAT_D16_X: + case Decoder::Opcode::BUFFER_LOAD_FORMAT_D16_XYZW: case Decoder::Opcode::TBUFFER_LOAD_FORMAT_X: case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XY: case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZ: diff --git a/src/graphics/shader/recompiler/ir/Program.cpp b/src/graphics/shader/recompiler/ir/Program.cpp index e7818dfba..e46c0f407 100644 --- a/src/graphics/shader/recompiler/ir/Program.cpp +++ b/src/graphics/shader/recompiler/ir/Program.cpp @@ -473,8 +473,11 @@ void ValidateProgram(const Program& program, bool require_ssa) { !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) { return Fail("indirect buffer requires a scalar, raw DWORD x1/x2/x3/x4, or formatted X load"); } + const bool packed_formatted_load = + BufferAccessOf(inst.GetOpcode()) == BufferAccess::Read && memory.formatted && + memory.data_bits == 16u && memory.component_count == 4u && buffer_components == 2u; if (buffer_components > 1u && - (!vector_buffer || memory.data_bits != 32u || + (!vector_buffer || (memory.data_bits != 32u && !packed_formatted_load) || memory.data_dwords != buffer_components || memory.component_index != 0u)) { return Fail(fmt::format("{} has inconsistent native-wide metadata", ValueOpcodeName(inst.GetOpcode()))); diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 58d298185..2271586d1 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -1534,14 +1534,23 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& if (memory.formatted && !memory.typed) { const auto& buffer = buffers[memory.resource]; const auto format = Format::GetFormatInfo(buffer.descriptor_format); - for (uint32_t component = 0; component < count; component++) { + const bool d16 = memory.data_bits == 16u; + const auto components = d16 ? memory.component_count : count; + std::array constants {}; + for (uint32_t component = 0; component < components; component++) { if (format.type == Format::ComponentType::Unknown || GetDstSel(buffer.descriptor_swizzle, component) != 1u) continue; - const auto one = Format::FormattedConstantBits( + auto one = Format::FormattedConstantBits( format, Format::FormattedSourceKind::One); + if (d16 && one == 0x3f800000u) one = 0x3c00u; + constants[d16 ? component / 2u : component] |= + one << (d16 ? (component % 2u) * 16u : 0u); + } + for (uint32_t component = 0; component < count; component++) { + if (constants[component] == 0u) continue; values[component] = Value(&*block->PrependNewInst( it, ValueOpcode::SelectU32, - {inst.Arg(inst.NumArgs() - 1), Value(one), Value(0u)})); + {inst.Arg(inst.NumArgs() - 1), Value(constants[component]), Value(0u)})); } } Value result = values[0]; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index c166bc347..b8e1edac2 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -30951,6 +30951,7 @@ std::vector BufferLoadFormatD16XCases() { u32 index = 0; bool active = true; bool overlap = false; + u32 stride = 4; }; constexpr LoadCase inputs[] = { {"BufferLoadFormatD16XFloat16", F::k16Float, 0x7e63c000u, 0xabcdc000u}, @@ -30971,6 +30972,8 @@ std::vector BufferLoadFormatD16XCases() { {"BufferLoadFormatD16XOobFloatOne", F::k32Float, 0u, 0xabcd3c00u, 1, 4}, {"BufferLoadFormatD16XOobUintOne", F::k32UInt, 0u, 0xabcd0001u, 1, 4}, {"BufferLoadFormatD16XOobZero", F::k32Float, 0u, 0xabcd0000u, 4, 4}, + {"BufferLoadFormatD16XZeroStrideFloatOne", F::k32Float, 0u, 0xabcd3c00u, 1, 0, true, false, 0}, + {"BufferLoadFormatD16XZeroStrideUintOne", F::k32UInt, 0u, 0xabcd0001u, 1, 0, true, false, 0}, {"BufferLoadFormatD16XInactive", F::k32Float, 0u, 0xabcd1234u, 4, 0, false}, {"BufferLoadFormatD16XOverlappingAddress", F::k16UInt, 7u, 7u, 4, 0, true, true}, }; @@ -30993,8 +30996,8 @@ std::vector BufferLoadFormatD16XCases() { AppendEnd(&test.code); test.initial = {input.input, 0xdeadbeefu, 0u, 0xcafef00du}; test.expected = {input.input, 0xdeadbeefu, input.expected, 0xcafef00du}; - test.user_data = MakeStructuredStorageBufferData(4, 4, false, BufferFormat(input.format)); - test.user_data[3] = (test.user_data[3] & ~7u) | input.selector; + test.user_data = MakeStructuredStorageBufferData(input.stride, 4, false, BufferFormat(input.format)); + test.user_data[3] = (test.user_data[3] & ~0x30000007u) | input.selector; test.has_user_data = true; test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_D16_X, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -31003,6 +31006,88 @@ std::vector BufferLoadFormatD16XCases() { return cases; } +std::vector BufferLoadFormatD16XyzwCases() { + using F = Prospero::BufferFormat; + using O = ShaderOpcode; + struct LoadCase { + const char *name; + F format; + std::array input; + std::array expected; + u32 swizzle = DstSel(4, 5, 6, 7); + u32 stride = 16; + u32 index = 0; + bool active = true; + bool overlap = false; + }; + constexpr LoadCase inputs[] = { + {"BufferLoadFormatD16XyzwFloat32", F::k32_32_32_32Float, + {0x3f800000u, 0x3f000000u, 0u, 0xc0000000u}, {0x38003c00u, 0xc0000000u}}, + {"BufferLoadFormatD16XyzwFloat16", F::k16_16_16_16Float, + {0x38003c00u, 0xc0000000u}, {0x38003c00u, 0xc0000000u}}, + {"BufferLoadFormatD16XyzwUintSaturates", F::k32_32_32_32UInt, + {0u, 0x10000u, 0xffffu, 7u}, {0xffff0000u, 0x0007ffffu}}, + {"BufferLoadFormatD16XyzwSintSaturates", F::k32_32_32_32SInt, + {0xffff7fffu, 7u, 0x8000u, 0xffff8000u}, {0x00078000u, 0x80007fffu}}, + {"BufferLoadFormatD16XyzwUnorm", F::k8_8_8_8UNorm, + {0xffff0000u}, {0u, 0x3c003c00u}}, + {"BufferLoadFormatD16XyzwSwizzle", F::k32_32_32_32Float, + {0u, 0x3f800000u, 0xc0000000u, 0x3f000000u}, + {0xc0003800u, 0x3c003c00u}, DstSel(7, 6, 5, 1)}, + {"BufferLoadFormatD16XyzwPartialRecord", F::k32_32_32_32Float, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u}, + {0u, 0u}, DstSel(4, 5, 6, 7), 12}, + {"BufferLoadFormatD16XyzwOobOne", F::k32_32_32_32Float, + {}, {0x3c00u, 0x3c00u}, DstSel(1, 4, 1, 4), 16, 1}, + {"BufferLoadFormatD16XyzwZeroStrideFloatOne", F::k32_32_32_32Float, + {}, {0x3c003c00u, 0x3c000000u}, DstSel(1, 1, 4, 1), 0}, + {"BufferLoadFormatD16XyzwZeroStrideUintOne", F::k32_32_32_32UInt, + {}, {0x10000u, 1u}, DstSel(4, 1, 1, 4), 0}, + {"BufferLoadFormatD16XyzwInactive", F::k32_32_32_32Float, + {}, {0xabcd1234u, 0xbeef5678u}, DstSel(4, 5, 6, 7), 16, 0, false}, + {"BufferLoadFormatD16XyzwOverlappingAddress", F::k32_32_32_32Float, + {0x3f800000u, 0x3f000000u, 0u, 0xc0000000u}, + {0x38003c00u, 0xc0000000u}, DstSel(4, 5, 6, 7), 16, 0, true, true}, + }; + std::vector cases; + for (const auto &input : inputs) { + TestCase test; + test.name = input.name; + AppendVMovLiteral(&test.code, 16, input.overlap ? input.index : 0xabcd1234u); + AppendVMovLiteral(&test.code, 17, 0xbeef5678u); + AppendVMovLiteral(&test.code, 18, 0xcafef00du); + AppendVMovU32(&test.code, 12, input.index); + if (!input.active) test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + // Captured b21e373811f28e31 pc 0x1914: e20c2000 8001100c. + test.code.push_back(EncodeMubuf0(0x83u, 0, true, false)); + test.code.push_back(EncodeMubuf1(16, 1, input.overlap ? 16 : 12)); + if (!input.active) { + test.code.push_back(EncodeSMovB32(126, InlineU32(1))); + test.code.push_back(EncodeSMovB32(127, InlineU32(0))); + } + for (u32 word = 0; word < 3; word++) AppendStoreVgpr(&test.code, 16 + word, 4 + word); + AppendEnd(&test.code); + test.initial.assign(input.input.begin(), input.input.end()); + test.initial.resize(8, 0xdeadbeefu); + test.expected = test.initial; + test.expected[4] = input.expected[0]; + test.expected[5] = input.expected[1]; + test.expected[6] = 0xcafef00du; + const auto descriptor = MakeStructuredStorageBufferData( + input.stride, 1, false, BufferFormat(input.format)); + std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4); + test.user_data[7] = (test.user_data[7] & ~0x30000fffu) | input.swizzle; + test.has_user_data = true; + if (input.stride != 0 && input.active) { + test.storage_buffer_ranges = {{0, input.stride}, {0, 32}}; + } + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_D16_XYZW, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + cases.push_back(std::move(test)); + } + return cases; +} + std::vector BufferStoreFormatD16XCases() { using F = Prospero::BufferFormat; using O = ShaderOpcode; @@ -39315,6 +39400,9 @@ std::vector MakeCases() { for (auto &test : BufferLoadFormatD16XCases()) { cases.push_back(std::move(test)); } + for (auto &test : BufferLoadFormatD16XyzwCases()) { + cases.push_back(std::move(test)); + } for (auto &test : BufferStoreFormatD16XCases()) { cases.push_back(std::move(test)); } @@ -44982,6 +45070,9 @@ int main(int argc, char **argv) { for (const auto &test : BufferLoadFormatD16XCases()) { RunCase(&vulkan, test); } + for (const auto &test : BufferLoadFormatD16XyzwCases()) { + RunCase(&vulkan, test); + } for (const auto &test : BufferStoreFormatD16XCases()) { RunCase(&vulkan, test); }