From d8be245c221760f7c3a1d2f177b29119bbdc772e Mon Sep 17 00:00:00 2001 From: nmzik Date: Mon, 5 Oct 2026 03:48:35 +0400 Subject: [PATCH] shader: separate legacy FP32 MAD rounding from fused FMA Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com> --- .../backend/spirv/spirvEmitterAlu.cpp | 19 ++++ .../backend/spirv/spirvEmitterInstructions.h | 4 +- .../frontend/decode/ShaderDecoder.h | 3 + .../frontend/decode/VectorAluOps.cpp | 13 ++- .../recompiler/frontend/translate/Vector.cpp | 7 +- .../recompiler/ir/opcodes/ValueOpcodes.inc | 1 + tests/ShaderRecompilerComputeTests.cpp | 95 ++++++++++++++++++- 7 files changed, 134 insertions(+), 8 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp index fbae465a8..b83e6db24 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp @@ -223,6 +223,25 @@ uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) { } } // namespace + +uint32_t EmitFPFma32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) { + const auto result = EmitGlsl(state, a, b, c); + state.builder.AddAnnotation(spv::OpDecorate, result, spv::DecorationNoContraction); + return result; +} + +uint32_t EmitFPMad32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) { + // Legacy MAD/MAC round and flush between multiply and add, irrespective of SP_DENORM. + a = EmitFlushF32DenormToSignedZero(state, a); + b = EmitFlushF32DenormToSignedZero(state, b); + c = EmitFlushF32DenormToSignedZero(state, c); + const auto product = EmitFPMul32(state, a, b); + state.builder.AddAnnotation(spv::OpDecorate, product, spv::DecorationNoContraction); + const auto sum = EmitFPAdd32(state, EmitFlushF32DenormToSignedZero(state, product), c); + state.builder.AddAnnotation(spv::OpDecorate, sum, spv::DecorationNoContraction); + return EmitFlushF32DenormToSignedZero(state, sum); +} + uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) { const auto min_ab = EmitMinMaxF32Value(state, a, b, false); const auto min3 = EmitMinMaxF32Value(state, min_ab, c, false); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index 32cca8209..b6ca411a2 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -160,8 +160,8 @@ EMIT_NATIVE(FPMul64, OpFMul, F64, uint32_t, uint32_t) inline constexpr auto EmitFPFma64 = EmitGlsl; uint32_t EmitFPRecip64(EmitterState& state, uint32_t arg0); -inline constexpr auto EmitFPFma32 = - EmitGlsl; +uint32_t EmitFPFma32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); +uint32_t EmitFPMad32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1); uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1); uint32_t EmitFPMin64(EmitterState& state, uint32_t arg0, uint32_t arg1); diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 48c926923..edabd831b 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -224,6 +224,9 @@ enum class Opcode { V_MAC_F32, V_MADMK_F32, V_MADAK_F32, + V_FMAC_F32, + V_FMAMK_F32, + V_FMAAK_F32, V_MIN_F32, V_MAX_F32, V_ADD_F16, diff --git a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp index fed2e6f17..24feeecde 100644 --- a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp @@ -65,9 +65,9 @@ constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = { {0x2au, Opcode::V_SUBREV_CO_CI_U32}, {0x26u, Opcode::V_SUB_NC_U32, Vop2SdwaProfile::IntegerPartialDestination}, {0x27u, Opcode::V_SUBREV_NC_U32, Vop2SdwaProfile::IntegerFullDestination}, - {0x2bu, Opcode::V_MAC_F32}, - {0x2cu, Opcode::V_MADMK_F32}, - {0x2du, Opcode::V_MADAK_F32}, + {0x2bu, Opcode::V_FMAC_F32}, + {0x2cu, Opcode::V_FMAMK_F32}, + {0x2du, Opcode::V_FMAAK_F32}, {0x2fu, Opcode::V_CVT_PKRTZ_F16_F32, Vop2SdwaProfile::PackedFloat16}, {0x32u, Opcode::V_ADD_F16, Vop2SdwaProfile::Float16}, {0x33u, Opcode::V_SUB_F16, Vop2SdwaProfile::Float16}, @@ -800,6 +800,9 @@ bool IsVop2FloatOpcode(Opcode opcode) { case Opcode::V_MAC_F32: case Opcode::V_MADMK_F32: case Opcode::V_MADAK_F32: + case Opcode::V_FMAC_F32: + case Opcode::V_FMAMK_F32: + case Opcode::V_FMAAK_F32: case Opcode::V_CVT_PKRTZ_F16_F32: case Opcode::V_ADD_F16: case Opcode::V_SUB_F16: @@ -1057,6 +1060,7 @@ void FinalizeVop2Instruction(std::span code, uint32_t word_index Instruction& inst) { switch (inst.opcode) { case Opcode::V_MADMK_F32: + case Opcode::V_FMAMK_F32: case Opcode::V_FMAMK_F16: inst.src2 = inst.src1; inst.src1 = {}; @@ -1064,6 +1068,7 @@ void FinalizeVop2Instruction(std::span code, uint32_t word_index inst.src_count = 3; break; case Opcode::V_MADAK_F32: + case Opcode::V_FMAAK_F32: case Opcode::V_FMAAK_F16: inst.src2 = {}; inst.src2.kind = OperandKind::LiteralConstant; @@ -1401,6 +1406,7 @@ bool SupportsNativeVop3SourceModifiers(Opcode opcode) { case Opcode::V_MIN_F32: case Opcode::V_MAX_F32: case Opcode::V_MAC_F32: + case Opcode::V_FMAC_F32: case Opcode::V_MAD_F32: case Opcode::V_ADD_F64: case Opcode::V_MUL_F64: @@ -1436,6 +1442,7 @@ bool SupportsNativeVop3ResultModifiers(Opcode opcode) { case Opcode::V_MIN_F32: case Opcode::V_MAX_F32: case Opcode::V_MAC_F32: + case Opcode::V_FMAC_F32: case Opcode::V_MAD_F32: case Opcode::V_FMA_F32: case Opcode::V_FMA_F16: diff --git a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp index 87acd4d7a..5e6568839 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp @@ -479,10 +479,13 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false); case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false); case O::V_LDEXP_F32: return FloatBinary(inst, IR::ValueOpcode::FPLdexp, false); - case O::V_MAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, true, true); + case O::V_MAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPMad32, true, true); case O::V_MADMK_F32: case O::V_MADAK_F32: - case O::V_MAD_F32: + case O::V_MAD_F32: return FloatTernary(inst, IR::ValueOpcode::FPMad32, false, true); + case O::V_FMAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, true, true); + case O::V_FMAMK_F32: + case O::V_FMAAK_F32: case O::V_FMA_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, false, true); case O::V_FMA_F64: return FloatTernary(inst, IR::ValueOpcode::FPFma64, false, false); case O::V_MIN3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMinTri32, false, false); diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index ef3cb5c00..e4d2d0cfd 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -179,6 +179,7 @@ VALUE_OPCODE(FPCmpClass32, U1, F32, U32) VALUE_OPCODE(FPAdd32, F32, F32, F32) VALUE_OPCODE(FPSub32, F32, F32, F32) VALUE_OPCODE(FPFma32, F32, F32, F32, F32) +VALUE_OPCODE(FPMad32, F32, F32, F32, F32) VALUE_OPCODE(FPMul32, F32, F32, F32) VALUE_OPCODE(FPAdd64, F64, F64, F64) VALUE_OPCODE(FPMul64, F64, F64, F64) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 34d4022b5..f6fdc33a5 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -18489,6 +18489,9 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, case Opcode::V_MAC_F32: case Opcode::V_MADMK_F32: case Opcode::V_MADAK_F32: + case Opcode::V_FMAC_F32: + case Opcode::V_FMAMK_F32: + case Opcode::V_FMAAK_F32: case Opcode::V_MIN_F32: case Opcode::V_MAX_F32: case Opcode::V_MAD_F32: @@ -23508,6 +23511,86 @@ TestCase VectorDppBoundsControlZeroPreservesDestination() { return test; } +TestCase VectorMadF32RoundsProduct() { + using O = ShaderOpcode; + + std::vector code; + for (u32 source = 0; source < 3; source++) { + AppendVMovU32(&code, 30, source * sizeof(u32)); + AppendBufferLoadDword(&code, source, 30); + } + AppendVMovLiteral(&code, 10, 0xbf800000u); + AppendVMovLiteral(&code, 11, 0xbf800000u); + AppendVMovLiteral(&code, 15, 0xbf800000u); + AppendVMovLiteral(&code, 16, 0xbf800000u); + code.push_back(EncodeVop2(0x1f, 10, Vgpr(0), 1)); + AppendVop3(&code, 0x11f, 11, Vgpr(0), Vgpr(1), 0); + AppendVop3(&code, 0x141, 12, Vgpr(0), Vgpr(1), Vgpr(2)); + code.push_back(EncodeVop2(0x20, 13, Vgpr(0), 2)); + code.push_back(0x3f7ffffeu); + code.push_back(EncodeVop2(0x21, 14, Vgpr(0), 1)); + code.push_back(0xbf800000u); + code.push_back(EncodeVop2(0x2b, 15, Vgpr(0), 1)); + AppendVop3(&code, 0x12b, 16, Vgpr(0), Vgpr(1), 0); + AppendVop3(&code, 0x14b, 17, Vgpr(0), Vgpr(1), Vgpr(2)); + code.push_back(EncodeVop2(0x2c, 18, Vgpr(0), 2)); + code.push_back(0x3f7ffffeu); + code.push_back(EncodeVop2(0x2d, 19, Vgpr(0), 1)); + code.push_back(0xbf800000u); + for (u32 i = 0; i < 10; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); + + TestCase test; + test.name = "VectorMadF32RoundsProduct"; + test.code = std::move(code); + test.initial = {0x3f800001u, 0x3f7ffffeu, 0xbf800000u}; + // (1+2^-23)*(1-2^-23)-1 rounds to zero for MAD, but -2^-46 for FMA. + test.expected = {0u, 0u, 0u, 0u, 0u, 0xa8800000u, 0xa8800000u, + 0xa8800000u, 0xa8800000u, 0xa8800000u}; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAC_F32, + O::V_MAD_F32, O::V_MADMK_F32, O::V_MADAK_F32, + O::V_FMAC_F32, O::V_FMA_F32, O::V_FMAMK_F32, O::V_FMAAK_F32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpFMul", "OpFAdd", "NoContraction", "Fma"}; + return test; +} + +TestCase VectorMadF32FlushesDenorms() { + using O = ShaderOpcode; + + std::vector code; + for (u32 sample = 0; sample < 5; sample++) { + for (u32 source = 0; source < 3; source++) { + AppendVMovU32(&code, 30, (sample * 3 + source) * sizeof(u32)); + AppendBufferLoadDword(&code, source, 30); + } + AppendVop3(&code, 0x141, 10 + sample, Vgpr(0), Vgpr(1), Vgpr(2)); + } + for (u32 sample = 0; sample < 5; sample++) { + AppendStoreVgpr(&code, 10 + sample, sample); + } + AppendEnd(&code); + + TestCase test; + test.name = "VectorMadF32FlushesDenorms"; + test.code = std::move(code); + // Input denorms in each operand, an intermediate denorm, and a negative result denorm. + test.initial = {0x00400000u, 0x40000000u, 0u, + 0x40000000u, 0x00400000u, 0u, + 0u, 0x3f800000u, 0x00400000u, + 0x00800000u, 0x3f000000u, 0x00800000u, + 0x00800000u, 0x3f800000u, 0x80800001u}; + test.expected = {0u, 0u, 0u, 0x00800000u, 0x80000000u}; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_F32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + // Legacy MAD always flushes, even when the shader requests denorm preservation. + test.compute_info.float_mode = 0xf0; + test.has_compute_info = true; + return test; +} + TestCase Vop3FmacF32NegatedSourceAccumulates() { using O = ShaderOpcode; @@ -23531,7 +23614,7 @@ TestCase Vop3FmacF32NegatedSourceAccumulates() { 0x3f800001u, 0x3f7ffffeu, 0x3f800000u}; // 10 - 2*3 = 4; 1 - (1+2^-23)*(1-2^-23) = 2^-46, not rounded zero. test.expected = {0x40800000u, 0x28800000u}; - test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAC_F32, + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FMAC_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.required_spirv = {"OpFNegate", "Fma"}; return test; @@ -34214,6 +34297,8 @@ std::vector MakeCases() { AddCase(VectorDppBankMaskPreservesDestination); AddCase(VectorDppBoundsControlZeroPreservesDestination); AddCase(Vop3FmacF32NegatedSourceAccumulates); + AddCase(VectorMadF32RoundsProduct); + AddCase(VectorMadF32FlushesDenorms); AddCase(Vop3LdexpSourceModifier); AddCase(Vop1MoveRelSource); AddCase(Vop1MoveRelDestination); @@ -39884,6 +39969,14 @@ int main(int argc, char **argv) { RunCase(&vulkan, VectorVop3IntegerOps()); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--mad-f32-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, VectorMadF32RoundsProduct()); + RunCase(&vulkan, VectorMadF32FlushesDenorms()); + RunCase(&vulkan, Vop3FmacF32NegatedSourceAccumulates()); + RunCase(&vulkan, VectorFloatArithmeticOps()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--zero-shift-only") == 0) { VulkanHarness vulkan; RunCase(&vulkan, ScalarZeroShiftWithRuntimeCount());