diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp index 273f224b4..1ccb434b1 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp @@ -203,9 +203,7 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) { const auto bits = state.builder.AllocateId(); const auto abs_bits = state.builder.AllocateId(); const auto sign_bits = state.builder.AllocateId(); - const auto non_zero = state.builder.AllocateId(); const auto subnormal = state.builder.AllocateId(); - const auto flush = state.builder.AllocateId(); const auto selected = state.builder.AllocateId(); const auto ret = state.builder.AllocateId(); state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value); @@ -213,12 +211,9 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) { ConstantU32(state, 0x7fffffffu)); state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)); - state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), non_zero, abs_bits, - ConstantU32(state, 0)); state.builder.AddFunction(spv::OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)); - state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal); - state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, flush, sign_bits, bits); + state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, subnormal, sign_bits, bits); state.builder.AddFunction(spv::OpBitcast, TypeF32(state), ret, selected); return ret; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index 1610f54f6..26d1f9bdc 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -2,6 +2,8 @@ #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" +#include + namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { // Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values // (IR::Value), and instructions whose metadata or operand loads must stay lazy. @@ -112,18 +114,34 @@ EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalNot, OpLogicalNot, U1, uint32_t) -EMIT_NATIVE(FPOrdEqual32, OpFOrdEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordEqual32, OpFUnordEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPOrdNotEqual32, OpFOrdNotEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordNotEqual32, OpFUnordNotEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPOrdLessThan32, OpFOrdLessThan, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordLessThan32, OpFUnordLessThan, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPOrdGreaterThan32, OpFOrdGreaterThan, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordGreaterThan32, OpFUnordGreaterThan, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t) -EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t) +template +uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) { + // SPIR-V's DenormFlushToZero mode does not require comparison operands to flush. + const bool flush = inst.Flags().flush_input_denorms; + const auto operand = [&](size_t index) { + const auto value = inst.Arg(index); + if (flush && value.IsImmediate()) { + const auto bits = std::bit_cast(value.F32Value()); + return ConstantF32(ctx.state, (bits & 0x7fffffffu) < 0x00800000u + ? bits & 0x80000000u : bits); + } + const auto id = ctx.Arg(inst, index); + return flush ? EmitFlushF32DenormToSignedZero(ctx.state, id) : id; + }; + return EmitNative(ctx.state, operand(0), operand(1)); +} +inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPOrdLessThan32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordLessThan32 = EmitFloatCompare32; +inline constexpr auto EmitFPOrdGreaterThan32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordGreaterThan32 = EmitFloatCompare32; +inline constexpr auto EmitFPOrdLessThanEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordLessThanEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPOrdGreaterThanEqual32 = EmitFloatCompare32; +inline constexpr auto EmitFPUnordGreaterThanEqual32 = EmitFloatCompare32; uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0); inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32; EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t) diff --git a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp index bc0ce8f17..e864e9c23 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp @@ -44,7 +44,8 @@ void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpc half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32); const auto rhs = half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32); - EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx); + const IR::FPCompareFlags flags{.flush_input_denorms = !half && flush_f32_inputs}; + EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx); } void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 521d0e667..a50e2cdda 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -1278,9 +1278,11 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& builtin(IR::StageInputKind::InstanceIndex)); } } + const bool flush_f32_inputs = options.stage == ShaderType::Compute && + (options.input_info.compute->float_mode & 0x10u) == 0; for (const auto& cfg_block: cfg.blocks) { const auto typed_index = block_indices.at(cfg_block.id); - Translator translator(result, result.blocks[typed_index], vector_limit); + Translator translator(result, result.blocks[typed_index], vector_limit, flush_f32_inputs); for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) { const auto& instruction = decoded.instructions[index]; if (IsCodeTableLoad(cfg, instruction.pc)) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 687b2aeec..76a1ddc80 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -9,8 +9,10 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend { class Translator { public: - Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit) - : program(program), ir(block), current_vector_limit(vector_limit) {} + Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit, + bool flush_f32_inputs) + : program(program), ir(block), current_vector_limit(vector_limit), + flush_f32_inputs(flush_f32_inputs) {} void TranslateInstruction(const Decoder::Instruction& inst); void TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute, @@ -261,6 +263,7 @@ private: Decoder::Opcode current_opcode = Decoder::Opcode::UNKNOWN; uint32_t current_pc = 0; uint32_t current_vector_limit = 1; + bool flush_f32_inputs; }; } // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h index c5da6bef9..b853d99ba 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h @@ -52,6 +52,10 @@ struct PermlaneFlags { static_assert(sizeof(PermlaneFlags) <= sizeof(uint64_t)); static_assert(std::is_trivially_copyable_v); +struct FPCompareFlags { + bool flush_input_denorms = false; +}; + struct MemoryFlags { uint32_t index = 0; uint32_t pc = 0; diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 4d02d4f83..7cd170896 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -781,14 +781,18 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { } return false; case ValueOpcode::FPOrdLessThanEqual32: - if (binary()) { - result = Float32(a) <= Float32(b); - return true; - } - return false; case ValueOpcode::FPOrdGreaterThanEqual32: if (binary()) { - result = Float32(a) >= Float32(b); + const auto operand = [&](uint64_t bits) { + if (inst.Flags().flush_input_denorms && + (bits & 0x7fffffffu) < 0x00800000u) { + bits &= 0x80000000u; + } + return Float32(bits); + }; + result = inst.GetOpcode() == ValueOpcode::FPOrdLessThanEqual32 + ? operand(a) <= operand(b) + : operand(a) >= operand(b); return true; } return false; diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index 3dc1e3bf4..a52fb7d1f 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -396,6 +396,49 @@ void TestUniformFirstLaneSamplerLod() { "uniform sampler LOD clamp evaluated incorrectly"); } +void TestFloatComparisonDescriptorInputs() { + struct Input { + uint32_t bits; + uint32_t less_equal; + uint32_t greater_equal; + }; + constexpr std::array inputs = { + Input{0x00000001u, 0, 1}, Input{0x80000001u, 1, 0}, + Input{0x007fffffu, 0, 1}, Input{0x807fffffu, 1, 0}, + Input{0x00800000u, 0, 1}, Input{0x80800000u, 1, 0}, + Input{0x00000000u, 1, 1}, Input{0x80000000u, 1, 1}, + Input{0x7fc00000u, 0, 0}}; + for (const bool flush : {false, true}) { + Fixture fixture; + const auto user = fixture.Emit( + ValueOpcode::GetUserData, {Value(static_cast(2))}); + const auto value = fixture.Emit(ValueOpcode::BitCastF32U32, {user}); + const auto less_equal = fixture.Emit(ValueOpcode::FPOrdLessThanEqual32, + {value, Value::F32(0.0f)}); + const auto greater_equal = fixture.Emit(ValueOpcode::FPOrdGreaterThanEqual32, + {value, Value::F32(0.0f)}); + less_equal.Instruction()->SetFlags(FPCompareFlags{flush}); + greater_equal.Instruction()->SetFlags(FPCompareFlags{flush}); + const auto low = fixture.Emit(ValueOpcode::SelectU32, + {less_equal, Value(1u), Value(0u)}); + const auto high = fixture.Emit(ValueOpcode::SelectU32, + {greater_equal, Value(1u), Value(0u)}); + fixture.program.descriptor_sources.push_back( + {.dwords = {low, high}, .dword_count = 2}); + fixture.Plan(); + for (size_t index = 0; index < inputs.size(); index++) { + const auto &input = inputs[index]; + const std::array user_data{input.bits}; + DescriptorValue result; + Check(SrtWalker(fixture.program, {.user_data = user_data}) + .EvaluateDescriptor(0, result) && + result.dwords[0] == (flush && index < 4 ? 1 : input.less_equal) && + result.dwords[1] == (flush && index < 4 ? 1 : input.greater_equal), + "descriptor comparison disagrees with native FP32 input mode"); + } + } +} + void TestSharedIntegerRuntimeDependencies() { Fixture fixture; const auto lane = fixture.Emit(ValueOpcode::LaneId); @@ -591,6 +634,7 @@ int main() { TestControlDependentStandaloneLoadStaysTyped(); TestRuntime64BitDescriptorOps(); TestUniformFirstLaneSamplerLod(); + TestFloatComparisonDescriptorInputs(); TestSharedIntegerRuntimeDependencies(); TestConstantBufferBounds(); TestReadLaneElimination(); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 17fa2e78b..b83fcbb45 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -22182,6 +22182,87 @@ TestCase VectorSinCosMaxFiniteSpecialCases() { O::S_ENDPGM}}; } +TestCase VectorCompareF32ExposureGuard() { + using O = ShaderOpcode; + TestCase test; + test.name = "VectorCompareF32ExposureGuard"; + test.initial = {0x00000001u, 0x80000001u, 0x007fffffu, 0x807fffffu, + 0x00000000u, 0x80000000u, 0x3f800000u, 0x40000000u}; + test.expected = test.initial; + AppendVMovLiteral(&test.code, 1, 0x3f800000u); + for (u32 i = 0; i < test.initial.size(); i++) { + AppendVMovU32(&test.code, 31, i * sizeof(u32)); + AppendBufferLoadDword(&test.code, 0, 31); + // Captured exposure guard: replace zero with 1.0 before taking its reciprocal. + test.code.push_back(EncodeVopc(0x02, InlineU32(0), 0)); + test.code.push_back(EncodeVop2(0x01, 2, Vgpr(0), 1)); + test.code.push_back(EncodeVop1(0x2a, 3, Vgpr(2))); + AppendStoreVgpr(&test.code, 2, static_cast(test.expected.size())); + AppendStoreVgpr(&test.code, 3, static_cast(test.expected.size() + 1)); + test.expected.push_back(i == 7 ? 0x40000000u : 0x3f800000u); + test.expected.push_back(i == 7 ? 0x3f000000u : 0x3f800000u); + } + test.initial.resize(test.expected.size()); + AppendEnd(&test.code); + test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F32, O::V_CNDMASK_B32, + O::V_RCP_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + +TestCase VectorCompareF32DenormalModes(u32 mode) { + using O = ShaderOpcode; + enum Relation : u32 { Less, Equal, Greater, Unordered }; + struct Pair { + u32 lhs; + u32 rhs; + Relation relation; + }; + constexpr Pair pairs[] = { + {0x00000001u, 0x00000000u, Equal}, + {0x80000001u, 0x80000000u, Equal}, + {0x00000000u, 0x00000001u, Equal}, + {0x80000000u, 0x80000001u, Equal}, + {0x007fffffu, 0x80000001u, Equal}, + {0x00800000u, 0x007fffffu, Greater}, + {0xbf800000u, 0x80000001u, Less}, + {0xff800000u, 0xbf800000u, Less}, + {0x7f800000u, 0x7f800000u, Equal}, + {0x7fc12345u, 0x00000000u, Unordered}, + {0x00000000u, 0x7fa12345u, Unordered}, + {0x00000000u, 0x80000000u, Equal}, + }; + TestCase test; + constexpr const char* names[] = {"VectorCompareF32DenormalModeC0", + "VectorCompareF32DenormalModeE0"}; + test.name = names[(mode >> 5u) & 1u]; + test.compute_info.float_mode = static_cast(mode); + test.has_compute_info = true; + for (const auto& pair : pairs) { + test.initial.insert(test.initial.end(), {pair.lhs, pair.rhs}); + } + test.expected = test.initial; + AppendVMovU32(&test.code, 1, 1); + for (u32 i = 0; i < std::size(pairs); i++) { + AppendVMovU32(&test.code, 31, i * 2u * sizeof(u32)); + AppendBufferLoadDword(&test.code, 2, 31); + AppendVMovU32(&test.code, 31, (i * 2u + 1u) * sizeof(u32)); + AppendBufferLoadDword(&test.code, 3, 31); + // The sixteen architectural predicates select subsets of LT/EQ/GT/unordered. + for (u32 predicate = 0; predicate < 16; predicate++) { + test.code.push_back(EncodeVopc(predicate, Vgpr(2), 3)); + test.code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 1)); + AppendStoreVgpr(&test.code, 4, static_cast(test.expected.size())); + test.expected.push_back((predicate >> pairs[i].relation) & 1u); + } + } + test.initial.resize(test.expected.size()); + AppendEnd(&test.code); + test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F32, O::V_CMP_LE_F32, + O::V_CMP_GE_F32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; + return test; +} + TestCase VectorCompareOps() { using O = ShaderOpcode; @@ -22532,6 +22613,8 @@ TestCase VectorCompareClassF32() { append_class(0x40000000u, 1u << 8u, true); append_class(0x7f800000u, 1u << 9u, true); append_class(0x40000000u, 1u << 3u, false); + append_class(0x00000001u, 1u << 7u, true); + append_class(0x80000001u, 1u << 4u, true); AppendVMovLiteral(&code, 0, 0x3f800000u); AppendVMovU32(&code, 2, 1u << 3u); @@ -29686,6 +29769,10 @@ std::vector MakeCases() { AddCase(VectorF64CapturedScreenSpaceShadows); AddCase(VectorF64ModesModifiersAndExec); AddCase(VectorSinCosMaxFiniteSpecialCases); + AddCase(VectorCompareF32ExposureGuard); + for (const auto mode : {0xc0u, 0xe0u}) { + cases.push_back(VectorCompareF32DenormalModes(mode)); + } AddCase(VectorCompareOps); AddCase(VectorVop3CompareEqI64OnGpu); AddCase(VectorVop3CompareEqU64OnGpu); @@ -34554,6 +34641,16 @@ int main(int argc, char **argv) { RunCase(&vulkan, VectorVopcCmpxOrderedCapturedExecMask()); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--fp32-compare-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, VectorCompareF32ExposureGuard()); + for (const auto mode : {0xc0u, 0xe0u}) { + RunCase(&vulkan, VectorCompareF32DenormalModes(mode)); + } + RunCase(&vulkan, VectorCompareClassF32()); + RunCase(&vulkan, VectorCompareF16Ops()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--buffer-format-store-only") == 0) { VulkanHarness vulkan; RunCase(&vulkan, BufferStoreFormatXyzwPackedUnormSkinningVectors()); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 01aeee725..1a0a93b8f 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -3923,6 +3923,35 @@ void TestNewShaderRecompilerScalarB64LaneTranslation() { CheckSpirvBinaryValidates(result.spirv); } +void TestFloatComparisonInputModes() { + using namespace ShaderRecompiler; + const uint32_t shader[] = { + EncodeVopc(0x02, 256, 1), // v_cmp_eq_f32 v0, v1 + EncodeVopc(0xca, 256, 1), // v_cmp_eq_f16 v0, v1 + EncodeSopp(0x01), + }; + Decoder::Program decoded; + Decoder::DecodeProgram(shader, decoded); + for (const uint8_t mode : {0xc0, 0xd0, 0xe0, 0xf0}) { + ShaderComputeInputInfo compute{}; + compute.float_mode = mode; + Frontend::TranslateOptions options{.stage = ShaderType::Compute}; + options.input_info.compute = &compute; + const auto program = Frontend::TranslateProgram(decoded, CFG::BuildGraph(decoded), options); + uint32_t comparisons = 0; + for (const auto* block : program.blocks) { + for (const auto& inst : *block) { + if (inst.GetOpcode() != IR::ValueOpcode::FPOrdEqual32) continue; + Check(inst.Flags().flush_input_denorms == + (comparisons == 0 && (mode & 0x10u) == 0), + "compute input mode leaked into preserved or promoted F16 comparison"); + comparisons++; + } + } + Check(comparisons == 2, "comparison input-mode fixture did not translate both precisions"); + } +} + void TestNewShaderRecompilerSignedCompareAlu() { const uint32_t shader[] = { EncodeSMovB32(0, 193), // s0 = -1 @@ -13582,6 +13611,7 @@ int main() { TestNewShaderRecompilerCapturedVopcSdwaCmpxLtU16(); TestNewShaderRecompilerIrLookupMissFailsExplicitly(); TestNewShaderRecompilerRejectsDppOn64BitCompares(); + TestFloatComparisonInputModes(); TestPsInputCountRegisterDecode(); TestPixelAncillaryLayerInput(); TestNewShaderRecompilerUnbasedFlatUsesBda();