diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 0724a4a86..f16ba9da4 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -463,6 +463,9 @@ void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind); uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source, bool max_value); +uint32_t EmitDsFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source, + bool max_value); + uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control); inline constexpr auto EmitSelectValueU32 = diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 02f7dc722..1393f7bbe 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -1227,34 +1227,10 @@ uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) { void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) { const auto& mem = ctx.Memory(inst); const bool max_value = inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32; - EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto index = DwordIndex(ctx, inst, mem); - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - EmitIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { - ctx.state.builder.AddFunction(spv::OpStore, ctx.scratch_u32_variable, - ctx.Arg(inst, 1)); - const auto data = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), data, - ctx.scratch_u32_variable); - AtomicUpdate( - ctx.state, EmitMemoryElementPointer(ctx.state, resource, index), - mem.kind, [&](uint32_t old) { - const auto old_f = - Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old); - const auto compare_f = - Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2)); - const auto data_f = - Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), data); - const auto compare = Binary( - ctx.state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan, - TypeBool(ctx.state), max_value ? old_f : compare_f, - max_value ? compare_f : old_f); - return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), - Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f)); - }); - }); - }); + EmitAtomicUpdate(ctx, inst, mem, + [max_value](EmitterState& state, uint32_t old, uint32_t value) { + return EmitDsFloatAtomicReplacement(state, old, value, max_value); + }); } uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index bb449a934..468c8c2c9 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -438,6 +438,20 @@ void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) { ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory)); } +namespace { + +uint32_t FloatOrderKey(EmitterState& state, uint32_t bits) { + const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual, + EmitAndConstant(state, bits, 0x80000000u), 0u); + const auto negative_key = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpNot, TypeU32(state), negative_key, bits); + const auto positive_key = + EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u)); + return EmitSelectValueU32(state, negative, negative_key, positive_key); +} + +} // namespace + uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source, bool max_value) { struct OrderedBits { @@ -447,15 +461,7 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t }; const auto classify = [&](uint32_t bits) { const auto cls = EmitClassifyF32Bits(state, bits); - const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual, - EmitAndConstant(state, bits, 0x80000000u), 0u); - const auto negative_key = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpNot, TypeU32(state), negative_key, bits); - const auto positive_key = - EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u)); - return OrderedBits { - cls.nan, cls.zero, - EmitSelectValueU32(state, negative, negative_key, positive_key)}; + return OrderedBits {cls.nan, cls.zero, FloatOrderKey(state, bits)}; }; const auto source_class = classify(source); const auto old_class = classify(old); @@ -470,6 +476,45 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t old); } +uint32_t EmitDsFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source, + bool max_value) { + struct OrderedBits { + uint32_t nan; + uint32_t signaling_nan; + uint32_t key; + }; + const auto classify = [&](uint32_t bits) { + const auto abs = EmitAndConstant(state, bits, 0x7fffffffu); + const auto nan = EmitCompareU32Constant(state, spv::OpUGreaterThan, abs, 0x7f800000u); + const auto signaling_nan = EmitLogicalAndBool( + state, nan, EmitCompareU32Constant(state, spv::OpIEqual, + EmitAndConstant(state, bits, 0x00400000u), 0u)); + const auto small = EmitCompareU32Constant(state, spv::OpULessThan, abs, 0x00800000u); + const auto sign = EmitAndConstant(state, bits, 0x80000000u); + const auto positive_denorm = EmitLogicalAndBool( + state, EmitCompareU32Constant(state, spv::OpIEqual, sign, 0u), + EmitCompareU32Constant(state, spv::OpINotEqual, abs, 0u)); + // Negative denorms tie with -0; positive denorms tie only with each other. + const auto small_bits = EmitSelectValueU32( + state, positive_denorm, ConstantU32(state, 1u), sign); + const auto key_bits = EmitSelectValueU32(state, small, small_bits, bits); + return OrderedBits {nan, signaling_nan, FloatOrderKey(state, key_bits)}; + }; + const auto lhs = classify(old); + const auto rhs = classify(source); + const auto choose_source = state.builder.AllocateId(); + state.builder.AddFunction(max_value ? spv::OpUGreaterThan : spv::OpULessThanEqual, + TypeBool(state), choose_source, rhs.key, lhs.key); + // Keep original bits: ordering denorms does not flush the selected operand. + auto result = EmitSelectValueU32(state, choose_source, source, old); + result = EmitSelectValueU32(state, lhs.nan, source, result); + result = EmitSelectValueU32(state, rhs.nan, old, result); + result = EmitSelectValueU32(state, rhs.signaling_nan, + EmitOrU32(state, source, ConstantU32(state, 0x00400000u)), result); + return EmitSelectValueU32(state, lhs.signaling_nan, + EmitOrU32(state, old, ConstantU32(state, 0x00400000u)), result); +} + uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) { if ((control & 0xc000u) == 0xc000u) { const uint32_t mask = control & 0x1fu; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 03ba346cb..cf7c86fd6 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -689,9 +689,7 @@ void EmitProgram(EmitterState& state) { DefineBvhIntersect(state); for (const auto* block: program.blocks) { if (std::ranges::any_of(*block, [](const IR::Inst& inst) { - return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 || - inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 || - inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32; + return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32; })) { ctx.scratch_u32_variable = state.builder.AllocateId(); if (state.lane_count == 2) { diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index d19f0e6fb..33988749e 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -205,9 +205,9 @@ uint32_t DsSourceCount(Opcode opcode) { case Opcode::DS_WRITE2ST64_B32: case Opcode::DS_WRITE2_B64: case Opcode::DS_WRITE2ST64_B64: - case Opcode::DS_MSKOR_B32: + case Opcode::DS_MSKOR_B32: return 3u; case Opcode::DS_MIN_F32: - case Opcode::DS_MAX_F32: return 3u; + case Opcode::DS_MAX_F32: case Opcode::DS_PERMUTE_B32: case Opcode::DS_BPERMUTE_B32: return 2u; case Opcode::DS_READ_ADDTID_B32: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index e4d2d0cfd..b73b15632 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -280,8 +280,8 @@ VALUE_OPCODE(WriteSharedU32, Void, U32, U32, U1) VALUE_OPCODE(WriteSharedU32x2, Void, U32, U32, U32, U1) VALUE_OPCODE(WriteSharedU32x3, Void, U32, U32, U32, U32, U1) VALUE_OPCODE(WriteSharedU32x4, Void, U32, U32, U32, U32, U32, U1) -VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U32, U1) -VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U1) +VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U1) VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicIAdd64, Void, U32, U64, U1) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index f6fdc33a5..9cbfe0229 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -30338,7 +30338,7 @@ TestCase DsMiscVariants() { return test; } -TestCase DsFloatMinMaxUsesSeparateCompareOperand() { +TestCase DsFloatMinMaxIgnoresData1() { using O = ShaderOpcode; std::vector code; @@ -30366,10 +30366,10 @@ TestCase DsFloatMinMaxUsesSeparateCompareOperand() { AppendEnd(&code); TestCase test; - test.name = "DsFloatMinMaxUsesSeparateCompareOperand"; + test.name = "DsFloatMinMaxIgnoresData1"; test.code = code; test.initial = std::vector(2, 0); - test.expected = {0x41100000u, 0x3f800000u}; + test.expected = {0x40800000u, 0x40800000u}; test.opcodes = {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_MIN_F32, O::DS_MAX_F32, O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; @@ -30380,6 +30380,100 @@ TestCase DsFloatMinMaxUsesSeparateCompareOperand() { return test; } +TestCase DsFloatMinMaxClasses(bool gds) { + using O = ShaderOpcode; + // Rows are memory, columns source: -inf, -normal, -denorm, -0, +0, + // +denorm, +normal, +inf, qNaN, sNaN. O/N select old/new; Q/R quiet them. + constexpr std::string_view minimum[] = { + "OOOOOOOOOR", "NNOOOOOOOR", "NNNNOOOOOR", "NNNOOOOOOR", "NNNNOOOOOR", + "NNNNNNOOOR", "NNNNNNOOOR", "NNNNNNNOOR", "NNNNNNNNOR", "QQQQQQQQQQ"}; + constexpr std::string_view maximum[] = { + "ONNNNNNNOR", "OONNNNNNOR", "OOOONNNNOR", "OOOONNNNOR", "OOOOONNNOR", + "OOOOOONNOR", "OOOOOONNOR", "OOOOOOOOOR", "NNNNNNNNOR", "QQQQQQQQQQ"}; + constexpr u32 old_values[] = { + 0xff800000u, 0xc0000000u, 0x80000100u, 0x80000000u, 0u, + 0x00000100u, 0x40000000u, 0x7f800000u, 0x7fc12345u, 0x7f812345u}; + constexpr u32 sources[] = { + 0xff800000u, 0xc0400000u, 0x80000001u, 0x80000000u, 0u, + 0x00000001u, 0x40400000u, 0x7f800000u, 0xffc54321u, 0xff854321u}; + const auto select = [](char choice, u32 old, u32 source) { + const u32 bits = choice == 'O' || choice == 'Q' ? old : source; + return choice == 'Q' || choice == 'R' ? bits | 0x00400000u : bits; + }; + TestCase test; + test.name = gds ? "DsFloatMinMaxGdsClasses" : "DsFloatMinMaxLdsClasses"; + for (u32 row = 0; row < std::size(old_values); ++row) { + for (u32 column = 0; column < std::size(sources); ++column) { + const auto old = old_values[row]; + const auto source = sources[column]; + test.initial.insert(test.initial.end(), {old, source, 0u, 0u}); + test.expected.insert(test.expected.end(), + {old, source, select(minimum[row][column], old, source), + select(maximum[row][column], old, source)}); + } + } + // Reversing positive denorm magnitudes still selects source for min and old for max. + test.initial.insert(test.initial.end(), {1u, 0x100u, 0u, 0u}); + test.expected.insert(test.expected.end(), {1u, 0x100u, 0x100u, 1u}); + const auto count = static_cast(test.initial.size() / 4u); + auto &code = test.code; + code.push_back(EncodeVop2(0x1a, 1, InlineU32(3), 0)); + code.push_back(EncodeVop2(0x1a, 8, InlineU32(4), 0)); + code.push_back(EncodeMubuf0(0x0d)); + code.push_back(EncodeMubuf1(2, 0, 8)); + for (u32 offset : {0u, 4u}) { + code.push_back(EncodeDs0(0x0d, offset, gds)); + code.push_back(EncodeDs1(0, 2, 1)); + } + code.push_back(EncodeDs0(0x12, 0, gds)); + code.push_back(EncodeDs1Ex(0, 250, 3, 1)); // DATA1 is unused and uninitialized. + code.push_back(EncodeDs0(0x13, 4, gds)); + code.push_back(EncodeDs1Ex(0, 251, 3, 1)); + AppendVMovLiteral(&code, 6, 0xff800000u); + AppendVMovLiteral(&code, 7, 0x7f800000u); + code.push_back(EncodeSop1(0x04, 4, 126)); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + // Inactive updates must leave the table results untouched. + code.push_back(EncodeDs0(0x12, 0, gds)); + code.push_back(EncodeDs1(0, 6, 1)); + code.push_back(EncodeDs0(0x13, 4, gds)); + code.push_back(EncodeDs1(0, 7, 1)); + code.push_back(EncodeSop1(0x04, 126, 4)); + // Active accesses immediately beyond the allocation must also be discarded. + AppendVMovU32(&code, 9, count * 8u); + code.push_back(EncodeDs0(0x12, 0, gds)); + code.push_back(EncodeDs1(0, 6, 9)); + code.push_back(EncodeDs0(0x13, 0, gds)); + code.push_back(EncodeDs1(0, 7, 9)); + for (u32 component = 0; component < 2u; ++component) { + code.push_back(EncodeDs0(0x36, component * 4u, gds)); + code.push_back(EncodeDs1(4u + component, 0, 1)); + } + code.push_back(EncodeMubuf0(0x1d, 8)); + code.push_back(EncodeMubuf1(4, 12, 8)); + AppendEnd(&code); + test.compute_info.threads_num[0] = count; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.lds_size_dwords = count * 2u; + test.has_compute_info = true; + test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::S_MOV_B64, + O::BUFFER_LOAD_DWORDX2, O::DS_WRITE_B32, O::DS_MIN_F32, + O::DS_MAX_F32, O::DS_READ_B32, O::BUFFER_STORE_DWORDX2, + O::S_ENDPGM}; + test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange", "OpLoopMerge"}; + test.forbidden_spirv = {"OpFOrdLessThan", "OpFOrdGreaterThan"}; + if (gds) { + test.gds_initial.resize(count * 2u); + for (u32 i = 0; i < count; ++i) { + test.expected_gds.push_back(test.expected[i * 4u + 2u]); + test.expected_gds.push_back(test.expected[i * 4u + 3u]); + } + } + return test; +} + TestCase DsSwizzleInvalidSourceLaneZero() { using O = ShaderOpcode; @@ -34528,7 +34622,9 @@ std::vector MakeCases() { } } AddCase(DsMiscVariants); - AddCase(DsFloatMinMaxUsesSeparateCompareOperand); + AddCase(DsFloatMinMaxIgnoresData1); + AddCase([] { return DsFloatMinMaxClasses(false); }); + AddCase([] { return DsFloatMinMaxClasses(true); }); AddCase(DsSwizzleInvalidSourceLaneZero); AddCase(DsPermuteCapturedExecOffsetAndWrap); AddCase(DsPermuteWave64UsesIndependentHalves); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 46c0ffa47..6fa74d598 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -7741,40 +7741,18 @@ void TestNewShaderRecompilerDsAddtidTranslation() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerDsFloatMinMaxTranslation() { - const uint32_t shader[] = { - EncodeDs0(0x12, 4), EncodeDs1Ex(0, 9, 7, 1), // ds_min_f32 v7, v9, v1 - EncodeDs0(0x13, 8), EncodeDs1Ex(0, 10, 8, 1), // ds_max_f32 v8, v10, v1 - 0xbf810000u, - }; - - auto options = MakeCompileOptions(ShaderType::Compute); - options.dump_ir = true; - - auto result = RecompileForTest(shader, options); - Check((result.decoded_dump.find("ds_min_f32") != std::string::npos), - "new decoder did not decode DS float min"); - Check((result.decoded_dump.find("ds_max_f32") != std::string::npos), - "new decoder did not decode DS float max"); - Check((result.ir_dump.find("DsMinF32 null, v7, v1") != std::string::npos), - "DS float min did not lower to explicit IR"); - Check((result.ir_dump.find("DsMaxF32 null, v8, v1") != std::string::npos), - "DS float max did not lower to explicit IR"); - Check((result.ir_dump.find("v9") != std::string::npos), - "DS float min did not retain DATA1 compare operand"); - Check((result.ir_dump.find("v10") != std::string::npos), - "DS float max did not retain DATA1 compare operand"); - Check(SpirvContainsOpcode(result.spirv, 12), - "SPIR-V binary does not contain OpExtInst"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), - "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 65), - "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 124), - "SPIR-V binary does not contain OpBitcast"); - CheckSpirvBinaryValidates(result.spirv); +void TestDsFloatMinMaxDecoder() { + using namespace ShaderRecompiler::Decoder; + for (bool max_value : {false, true}) { + const uint32_t shader[] = { + EncodeDs0(max_value ? 0x13 : 0x12, 4), EncodeDs1Ex(0, 250, 7, 1)}; + Instruction decoded; + DecodeInstruction(shader, 0u, decoded); + Check(decoded.opcode == (max_value ? Opcode::DS_MAX_F32 : Opcode::DS_MIN_F32) && + decoded.src_count == 2u && decoded.src0.reg == 1u && + decoded.src1.reg == 7u && decoded.offset == 4u, + "DS float min/max must read only address and DATA0"); + } } void TestNewShaderRecompilerCfgStraightLine() { @@ -14805,6 +14783,7 @@ int main() { // here. TestScalarAshrI64Decoder(); TestImageAtomicWidthDecoder(); + TestDsFloatMinMaxDecoder(); TestNewShaderDecoderArchitecture(); TestImageAddressOperands(); TestSopkCompareImmediateExtension();