diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index dadaa2deb..172890845 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -29,6 +29,7 @@ struct GraphicContext { bool memory_budget_ext_enabled = false; bool compute_subgroup_size_control_enabled = false; bool sample_rate_shading_enabled = false; + // bool fp64_denorm_preserve = false; // Temporarily disabled. bool attachment_feedback_loop_enabled = false; bool provoking_vertex_last_enabled = false; bool supports_block_texel_view = false; diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index 386828090..15a2e1016 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -564,8 +564,10 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics, vk::PhysicalDeviceVulkan11Properties properties11 {}; properties11.pNext = &subgroup_size_control; + vk::PhysicalDeviceFloatControlsProperties float_controls {}; + float_controls.pNext = &properties11; vk::PhysicalDeviceProperties2 properties2 {}; - properties2.pNext = &properties11; + properties2.pNext = &float_controls; if (graphics.mesh_shader_enabled) { subgroup_size_control.pNext = &graphics.mesh_shader_properties; @@ -628,6 +630,15 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics, device_features.vertexPipelineStoresAndAtomics = VK_TRUE; graphics.sample_rate_shading_enabled = true; device_features.shaderInt64 = VK_TRUE; + device_features.shaderFloat64 = + supported_features2.features.shaderFloat64 && + float_controls.shaderSignedZeroInfNanPreserveFloat64 && + float_controls.shaderRoundingModeRTEFloat32; + // if (device_features.shaderFloat64 && !float_controls.shaderDenormPreserveFloat64) { + // Log::WriteToConsoleAndLog( + // "WARNING: Vulkan device does not guarantee FP64 denormal preservation; " + // "continuing with native FP64 arithmetic. Very small values may be flushed to zero.\n"); + // } vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {}; #if defined(__APPLE__) diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 56e5c809d..95e11db9b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -198,6 +198,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program SpirvRequirements requirements {}; for (const auto* block: program.blocks) { for (const auto& inst: *block) { + requirements.float64 |= inst.GetType() == IR::Type::F64; if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic && inst.GetType() == IR::Type::U64) { requirements.buffer_int64_atomics = true; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp index e2da2ef83..fe9c62a0e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp @@ -291,9 +291,20 @@ uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0) { return EmitF32ToU32(state, arg0, false); } -uint32_t EmitConvertF32S32(EmitterState& state, uint32_t arg0) { - const auto signed_value = Unary(state, spv::OpBitcast, TypeI32(state), arg0); - return EmitNative(state, signed_value); +uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0) { + const auto converted = Unary(state, spv::OpFConvert, TypeF32(state), arg0); + const auto source = EmitNative( + state, Unary(state, spv::OpBitcast, TypeU64(state), arg0), 1u); + const auto exponent = EmitAndConstant(state, source, 0x7ff00000u); + const auto overflow = + Binary(state, spv::OpLogicalAnd, TypeBool(state), + EmitCompareU32Constant(state, spv::OpUGreaterThan, exponent, 0x47e00000u), + EmitCompareU32Constant(state, spv::OpULessThan, exponent, 0x7ff00000u)); + const auto clamped = EmitOrU32(state, EmitAndConstant(state, source, 0x80000000u), + ConstantU32(state, 0x7f7fffffu)); + return EmitFlushF32DenormToSignedZero( + state, Select(state, TypeF32(state), overflow, + Unary(state, spv::OpBitcast, TypeF32(state), clamped), converted)); } uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) { @@ -453,6 +464,11 @@ uint32_t EmitFPRecip32(EmitterState& state, uint32_t arg0) { return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), source); } +uint32_t EmitFPRecip64(EmitterState& state, uint32_t arg0) { + const auto one = state.builder.Constant(spv::OpConstant, TypeF64(state), 0u, 0x3ff00000u); + return EmitNative(state, one, arg0); +} + uint32_t EmitFPRecipIFlag32(EmitterState& state, uint32_t arg0) { // Integer-to-float inputs used by IFLAG cannot be denormal. return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), arg0); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index e87236f51..53dde198b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -13,13 +13,22 @@ inline constexpr auto EmitConvertU32U16 = EmitBitCastU16F16; inline constexpr auto EmitConvertU32U8 = EmitBitCastU16F16; inline constexpr auto EmitBitCastU32F32 = EmitBitCastU16F16; EMIT_NATIVE(BitCastF32U32, OpBitcast, F32, uint32_t) +EMIT_NATIVE(BitCastU64F64, OpBitcast, U64, uint32_t) +EMIT_NATIVE(BitCastF64U64, OpBitcast, F64, uint32_t) uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0); uint32_t EmitConvertU8U32(EmitterState& state, uint32_t arg0); uint32_t EmitConvertF16F32(EmitterState& state, uint32_t arg0); inline constexpr auto EmitConvertF32F16 = EmitF16BitsToF32; uint32_t EmitConvertS32F32(EmitterState& state, uint32_t arg0); uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0); -uint32_t EmitConvertF32S32(EmitterState& state, uint32_t arg0); +template +uint32_t EmitConvertSigned32ToFloat(EmitterState& state, uint32_t arg0) { + const auto signed_value = Unary(state, spv::OpBitcast, TypeI32(state), arg0); + return EmitNative(state, signed_value); +} +inline constexpr auto EmitConvertF32S32 = EmitConvertSigned32ToFloat; +inline constexpr auto EmitConvertF64S32 = EmitConvertSigned32ToFloat; +uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0); EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t) EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t) EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t) @@ -120,6 +129,10 @@ inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32; EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t) EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t) EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t) +EMIT_NATIVE(FPMul64, OpFMul, F64, uint32_t, uint32_t) +inline constexpr auto EmitFPFma64 = + EmitGlsl; +uint32_t EmitFPRecip64(EmitterState& state, uint32_t arg0); inline constexpr auto EmitFPFma32 = EmitGlsl; uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index f1000a0f3..541632323 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -68,6 +68,7 @@ struct SpirvRequirements { bool pixel_valid_mask = false; bool buffer_int64_atomics = false; bool coherent_buffers = false; + bool float64 = false; }; SpirvRequirements AnalyzeProgramRequirements(const IR::Program& program); @@ -139,6 +140,7 @@ uint32_t TypeU32Pair(EmitterState& state); uint32_t TypeI32(EmitterState& state); uint32_t TypeI32Pair(EmitterState& state); uint32_t TypeF32(EmitterState& state); +uint32_t TypeF64(EmitterState& state); uint32_t TypeU32Vector(EmitterState& state, uint32_t components); uint32_t TypeU32Composite(EmitterState& state, uint32_t components); @@ -167,6 +169,10 @@ template uint32_t EmitNative(EmitterState& state, Args... args) { const auto result = state.builder.AllocateId(); state.builder.AddFunction(opcode, TypeId(state, type), result, args...); + if constexpr (type == IR::Type::F64 && + (opcode == spv::OpFMul || opcode == spv::OpFDiv || opcode == spv::OpExtInst)) { + state.builder.AddAnnotation(spv::OpDecorate, result, spv::DecorationNoContraction); + } return result; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index eadb9e9bb..93d72247a 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -47,6 +47,10 @@ uint32_t TypeF32(EmitterState& state) { return state.builder.Type(spv::OpTypeFloat, 32); } +uint32_t TypeF64(EmitterState& state) { + return state.builder.Type(spv::OpTypeFloat, 64); +} + uint32_t TypeU32Vector(EmitterState& state, uint32_t components) { return state.builder.Type(spv::OpTypeVector, TypeU32(state), components); } @@ -679,6 +683,19 @@ void DefineModule(EmitterState& state) { // contract prevents host compilers from treating synthesized IEEE values as finite. state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve, 32u); + if (state.requirements.float64) { + EXIT_NOT_IMPLEMENTED(state.program.stage != ShaderType::Compute); + EXIT_NOT_IMPLEMENTED(state.input_info.compute->float_mode != 0xc0); + // MODE=0xc0 uses round-to-nearest-even and preserves FP64 input/output denormals. + state.builder.RequireCapability(spv::CapabilityFloat64); + state.builder.RequireCapability(spv::CapabilityRoundingModeRTE); + state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve, + 64u); + // FP64 denormal preservation is temporarily disabled. + // state.builder.RequireCapability(spv::CapabilityDenormPreserve); + // state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeDenormPreserve, 64u); + state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeRoundingModeRTE, 32u); + } if (const auto* cs = ShaderWorkgroupInput(state.program.stage, state.input_info)) { uint32_t local_x = state.requirements.compute_derivatives ? 2u : 1u; uint32_t local_y = state.requirements.compute_derivatives ? 2u : 1u; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 916aac673..d8b9df738 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -440,6 +440,7 @@ uint32_t TypeId(EmitterState& state, IR::Type type) { case IR::Type::U64: return TypeU64(state); case IR::Type::U32x2: return TypeU32Pair(state); case IR::Type::F32: return TypeF32(state); + case IR::Type::F64: return TypeF64(state); case IR::Type::U32x3: return TypeU32Vector(state, 3); case IR::Type::U32x4: return TypeU32Vector(state, 4); case IR::Type::F32x2: return TypeF32Vector(state, 2); diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 79e9e7e4e..e0785ebc3 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -154,6 +154,8 @@ enum class Opcode { V_CUBEMA_F32, V_CNDMASK_B32, V_DOT2C_F32_F16, + V_CVT_F64_I32, + V_CVT_F32_F64, V_CVT_F32_I32, V_CVT_F32_U32, V_CVT_U32_F32, @@ -173,6 +175,7 @@ enum class Opcode { V_CVT_F32_UBYTE1, V_CVT_F32_UBYTE2, V_CVT_F32_UBYTE3, + V_RCP_F64, V_RCP_F32, V_RCP_IFLAG_F32, V_FRACT_F32, @@ -234,6 +237,8 @@ enum class Opcode { V_MAD_I32_I24, V_MAD_U32_U24, V_MAD_U64_U32, + V_FMA_F64, + V_MUL_F64, V_FMA_F32, V_FMA_F16, V_PACK_B32_F16, diff --git a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp index 26eda2f0f..2ff04b8e6 100644 --- a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp @@ -87,6 +87,9 @@ constexpr OpcodeMap VOP1_OPCODE_LIST[] = { {0x00u, Opcode::V_NOP}, {0x01u, Opcode::V_MOV_B32}, {0x02u, Opcode::V_READFIRSTLANE_B32}, + {0x04u, Opcode::V_CVT_F64_I32}, + {0x0fu, Opcode::V_CVT_F32_F64}, + {0x2fu, Opcode::V_RCP_F64}, {0x05u, Opcode::V_CVT_F32_I32}, {0x06u, Opcode::V_CVT_F32_U32}, {0x07u, Opcode::V_CVT_U32_F32}, @@ -146,6 +149,9 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = { {0x00u, Opcode::V_NOP}, {0x01u, Opcode::V_MOV_B32}, {0x02u, Opcode::V_READFIRSTLANE_B32}, + {0x04u, Opcode::V_CVT_F64_I32}, + {0x0fu, Opcode::V_CVT_F32_F64}, + {0x2fu, Opcode::V_RCP_F64}, {0x05u, Opcode::V_CVT_F32_I32}, {0x06u, Opcode::V_CVT_F32_U32}, {0x07u, Opcode::V_CVT_U32_F32}, @@ -267,6 +273,8 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = { {0x146u, Opcode::V_CUBETC_F32}, {0x147u, Opcode::V_CUBEMA_F32}, {0x14bu, Opcode::V_FMA_F32}, + {0x14cu, Opcode::V_FMA_F64}, + {0x165u, Opcode::V_MUL_F64}, {0x148u, Opcode::V_BFE_U32}, {0x149u, Opcode::V_BFE_I32}, {0x14au, Opcode::V_BFI_B32}, @@ -452,6 +460,8 @@ bool IsVopcCompareExec(Opcode opcode); bool IsVop1FloatSourceOpcode(Opcode opcode) { switch (opcode) { + case Opcode::V_CVT_F32_F64: + case Opcode::V_RCP_F64: case Opcode::V_MOV_B32: case Opcode::V_CVT_F32_F16: case Opcode::V_CVT_U32_F32: @@ -709,6 +719,11 @@ void ApplyDppModifier(Operand& operand, uint32_t modifier, uint32_t encoding) { void DecodeVop1Dpp(uint32_t pc, std::span code, uint32_t word_index, uint32_t opcode, uint32_t vdst, Instruction& inst) { + if (inst.opcode == Opcode::V_CVT_F64_I32 || inst.opcode == Opcode::V_CVT_F32_F64 || + inst.opcode == Opcode::V_RCP_F64) { + SetUnsupported(inst, Family::VOP1, opcode, "FP64 instructions do not support DPP"); + return; + } const auto modifier = code[word_index + 1u]; const auto src0 = modifier & 0xffu; SetRawWords(inst, code, word_index, 2); @@ -758,6 +773,7 @@ bool IsVop2FloatOpcode(Opcode opcode) { bool IsVop1FloatResultOpcode(Opcode opcode) { switch (opcode) { + case Opcode::V_CVT_F32_F64: case Opcode::V_CVT_F32_I32: case Opcode::V_CVT_F32_U32: case Opcode::V_CVT_F32_F16: @@ -1178,6 +1194,7 @@ void DecodeVopcDpp(uint32_t pc, std::span code, uint32_t word_in uint32_t NativeVop3SourceCount(Opcode opcode) { switch (opcode) { + case Opcode::V_MUL_F64: case Opcode::V_MUL_LO_U32: case Opcode::V_MUL_HI_U32: case Opcode::V_MUL_LO_I32: @@ -1328,6 +1345,8 @@ bool SupportsNativeVop3SourceModifiers(Opcode opcode) { case Opcode::V_MAX_F32: case Opcode::V_MAC_F32: case Opcode::V_MAD_F32: + case Opcode::V_MUL_F64: + case Opcode::V_FMA_F64: case Opcode::V_FMA_F32: case Opcode::V_PACK_B32_F16: case Opcode::V_CUBEID_F32: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index b180094a3..3edb31f67 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -225,9 +226,28 @@ IR::Value Translator::ReadOperand(const Decoder::Operand& operand, IR::Type type } return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u))); } - if (type == IR::Type::U64) { - const auto pair = ReadU32Pair(operand); - return ir.ConstructU64(pair[0], pair[1]); + if (type == IR::Type::U64 || type == IR::Type::F64) { + auto pair = ReadU32Pair(operand); + if (type == IR::Type::F64) { + if (operand.kind == Decoder::OperandKind::LiteralConstant) { + pair = {IR::U32(IR::Value(0u)), IR::U32(IR::Value(operand.value))}; + } else if (operand.kind == Decoder::OperandKind::FloatInlineConstant) { + const auto bits = operand.value == 0x3e22f983u + ? 0x3fc45f306dc9c882ull + : std::bit_cast(static_cast( + std::bit_cast(operand.value))); + pair = {IR::U32(IR::Value(static_cast(bits))), + IR::U32(IR::Value(static_cast(bits >> 32u)))}; + } + if (operand.absolute) { + pair[1] = ir.BitwiseAnd(pair[1], IR::U32(IR::Value(0x7fffffffu))); + } + if (operand.negate) { + pair[1] = ir.BitwiseXor(pair[1], IR::U32(IR::Value(0x80000000u))); + } + } + const auto bits = ir.ConstructU64(pair[0], pair[1]); + return type == IR::Type::F64 ? ir.Emit(IR::ValueOpcode::BitCastF64U64, {bits}) : IR::Value(bits); } auto bits = ApplyBitSourceModifiers(operand, ReadRawU32(operand)); if (TypesOverlap(type, IR::Type::F32) && !TypesOverlap(type, IR::Type::U32)) { @@ -403,6 +423,10 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value) Write16Bits(operand, IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {bits}))); return; } + if (type == IR::Type::F64) { + value = ir.Emit(IR::ValueOpcode::BitCastU64F64, {value}); + type = IR::Type::U64; + } if (type == IR::Type::U64) { WriteU32Pair(operand, {ir.CompositeExtract(value, 0), ir.CompositeExtract(value, 1)}); return; @@ -842,6 +866,14 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_ include_vector(inst.src1); include_vector(inst.src2); include_vector(inst.src3); + switch (inst.opcode) { + case Decoder::Opcode::V_CVT_F64_I32: include_vector(inst.dst, 2u); break; + case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]]; + case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]]; + case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]]; + case Decoder::Opcode::V_CVT_F32_F64: include_vector(inst.src0, 2u); break; + default: break; + } if (inst.family == Decoder::Family::DS) { switch (inst.opcode) { case Decoder::Opcode::DS_WRITE_B64: diff --git a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp index 765266f2f..a2b7a8db3 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp @@ -297,6 +297,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_CVT_F32_UBYTE3: V_CVT_F32_UBYTE(inst, 3); return; case O::V_CVT_F32_U32: V_CVT_F32_U32(inst); return; case O::V_CVT_F32_I32: V_CVT_F32_I32(inst); return; + case O::V_CVT_F64_I32: return FloatUnary(inst, IR::ValueOpcode::ConvertF64S32); + case O::V_CVT_F32_F64: return FloatUnary(inst, IR::ValueOpcode::ConvertF32F64); case O::V_CVT_U32_F32: V_CVT_U32_F32(inst); return; case O::V_CVT_I32_F32: V_CVT_I32_F32(inst); return; case O::V_CVT_F16_F32: V_CVT_F16_F32(inst); return; @@ -394,6 +396,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_FREXP_MANT_F32: return V_FREXP_MANT_F32(inst); case O::V_RCP_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecip32); + case O::V_RCP_F64: return FloatUnary(inst, IR::ValueOpcode::FPRecip64); case O::V_RCP_IFLAG_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipIFlag32); case O::V_FRACT_F32: return FloatUnary(inst, IR::ValueOpcode::FPFract32); case O::V_TRUNC_F32: return FloatUnary(inst, IR::ValueOpcode::FPTrunc32); @@ -410,6 +413,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_SUB_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, false); case O::V_SUBREV_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, true); case O::V_MUL_F32: return FloatBinary(inst, IR::ValueOpcode::FPMul32, false); + case O::V_MUL_F64: return FloatBinary(inst, IR::ValueOpcode::FPMul64, false); case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false); case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false); case O::V_LDEXP_F32: return FloatBinary(inst, IR::ValueOpcode::FPLdexp, false); @@ -418,6 +422,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_MADAK_F32: case O::V_MAD_F32: case O::V_FMA_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, false, true); + case O::V_FMA_F64: return FloatTernary(inst, IR::ValueOpcode::FPFma64, false, false); case O::V_MIN3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMinTri32, false, false); case O::V_MAX3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMaxTri32, false, false); case O::V_MED3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMedTri32, false, false); diff --git a/src/graphics/shader/recompiler/ir/Type.cpp b/src/graphics/shader/recompiler/ir/Type.cpp index c0653b725..db8de73eb 100644 --- a/src/graphics/shader/recompiler/ir/Type.cpp +++ b/src/graphics/shader/recompiler/ir/Type.cpp @@ -30,6 +30,7 @@ std::string TypeName(Type type) { "ImageResource", "SamplerResource", "ImageAddress", + "F64", }; const auto bits = static_cast(type); if (bits == 0) { diff --git a/src/graphics/shader/recompiler/ir/Type.h b/src/graphics/shader/recompiler/ir/Type.h index e3a150330..dd97d53ae 100644 --- a/src/graphics/shader/recompiler/ir/Type.h +++ b/src/graphics/shader/recompiler/ir/Type.h @@ -27,6 +27,7 @@ enum class Type : uint32_t { ImageResource = 1u << 17u, SamplerResource = 1u << 18u, ImageAddress = 1u << 19u, + F64 = 1u << 20u, }; constexpr Type operator|(Type lhs, Type rhs) { diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index 26665b6f7..d18873871 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -33,6 +33,7 @@ constexpr Type U32 = Type::U32; constexpr Type U64 = Type::U64; constexpr Type F16 = Type::F16; constexpr Type F32 = Type::F32; +constexpr Type F64 = Type::F64; constexpr Type U32x2 = Type::U32x2; constexpr Type U32x3 = Type::U32x3; constexpr Type U32x4 = Type::U32x4; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index e3306a207..7a903b492 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -52,6 +52,8 @@ VALUE_OPCODE(BitCastU16F16, U16, F16) VALUE_OPCODE(BitCastF16U16, F16, U16) VALUE_OPCODE(BitCastU32F32, U32, F32) VALUE_OPCODE(BitCastF32U32, F32, U32) +VALUE_OPCODE(BitCastU64F64, U64, F64) +VALUE_OPCODE(BitCastF64U64, F64, U64) VALUE_OPCODE(ConvertU16U32, U16, U32) VALUE_OPCODE(ConvertU32U16, U32, U16) VALUE_OPCODE(ConvertU8U32, U8, U32) @@ -61,6 +63,8 @@ VALUE_OPCODE(ConvertF16F32, F16, F32) VALUE_OPCODE(ConvertS32F32, U32, F32) VALUE_OPCODE(ConvertU32F32, U32, F32) VALUE_OPCODE(ConvertF32S32, F32, U32) +VALUE_OPCODE(ConvertF64S32, F64, U32) +VALUE_OPCODE(ConvertF32F64, F32, F64) VALUE_OPCODE(ConvertF32U32, F32, U32) VALUE_OPCODE(CompositeConstructU64, U64, U32, U32) VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32) @@ -166,6 +170,9 @@ VALUE_OPCODE(FPAdd32, F32, F32, F32) VALUE_OPCODE(FPSub32, F32, F32, F32) VALUE_OPCODE(FPFma32, F32, F32, F32, F32) VALUE_OPCODE(FPMul32, F32, F32, F32) +VALUE_OPCODE(FPMul64, F64, F64, F64) +VALUE_OPCODE(FPFma64, F64, F64, F64, F64) +VALUE_OPCODE(FPRecip64, F64, F64) VALUE_OPCODE(FPMin32, F32, F32, F32) VALUE_OPCODE(FPMax32, F32, F32, F32) VALUE_OPCODE(FPMinTri32, F32, F32, F32, F32) diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index 0e91fd17e..7bde696dd 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -622,6 +622,7 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs, info.threads_num[2] = regs.cs_regs.num_thread_z; info.lds_size_dwords = static_cast(regs.cs_regs.lds_size) * 128u; info.scratch_size_dwords = data.scratch_size_dwords; + info.float_mode = regs.cs_regs.float_mode; info.group_id[0] = regs.cs_regs.tgid_x_en != 0; info.group_id[1] = regs.cs_regs.tgid_y_en != 0; info.group_id[2] = regs.cs_regs.tgid_z_en != 0; @@ -727,7 +728,7 @@ void BuildStageStaticKey(const ShaderPixelInputInfo& info, std::vector void BuildStageStaticKey(const ShaderComputeInputInfo& info, std::vector& key) { key.clear(); key.push_back(info.workgroup_register); - key.push_back(info.wave_size); + key.push_back(info.wave_size | (static_cast(info.float_mode) << 8u)); key.push_back(info.host_subgroup_size); key.push_back(info.thread_ids_num); key.push_back(info.lds_size_dwords); diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 21114b757..1679ff32b 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -145,6 +145,7 @@ struct ShaderVertexInputInfo { }; struct ShaderComputeInputInfo: ShaderWorkgroupInputInfo { + uint8_t float_mode = 0xc0; uint32_t dispatch_threads_num[3] = {0, 0, 0}; bool group_id[3] = {false, false, false}; bool dispatch_thread_dimensions = false; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 7a6c4b6da..e97879caf 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -15837,6 +15837,7 @@ private: device_features.shaderImageGatherExtended = true; device_features.sampleRateShading = true; device_features.shaderInt64 = true; + device_features.shaderFloat64 = available_features.shaderFloat64; device_features.fillModeNonSolid = true; device_features.tessellationShader = true; device_info.pEnabledFeatures = &device_features; @@ -16400,6 +16401,11 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, case Opcode::V_MAX3_F32: case Opcode::V_MED3_F32: case Opcode::V_DOT2C_F32_F16: + case Opcode::V_CVT_F64_I32: + case Opcode::V_CVT_F32_F64: + case Opcode::V_RCP_F64: + case Opcode::V_MUL_F64: + case Opcode::V_FMA_F64: case Opcode::V_CVT_F32_I32: case Opcode::V_CVT_F32_U32: case Opcode::V_CVT_U32_F32: @@ -21689,6 +21695,157 @@ TestCase VectorSpecialF32FlushesDenormalInputs() { O::V_SQRT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase VectorF64CapturedScreenSpaceShadows() { + using O = ShaderOpcode; + TestCase test; + test.name = "VectorF64CapturedScreenSpaceShadows"; + const auto store_pair = [&](u32 reg, double value) { + const auto bits = std::bit_cast(value); + AppendStoreVgpr(&test.code, reg, static_cast(test.expected.size())); + AppendStoreVgpr(&test.code, reg + 1, + static_cast(test.expected.size() + 1)); + test.expected.insert(test.expected.end(), {static_cast(bits), + static_cast(bits >> 32)}); + }; + for (const auto values : + {std::array{63, 127, 101}, + std::array{16777217, -16777217, 123456789}, + std::array{INT32_MIN, INT32_MAX, -31}}) { + AppendVMovLiteral(&test.code, 7, static_cast(values[0])); + AppendVMovLiteral(&test.code, 1, static_cast(values[1])); + AppendVMovLiteral(&test.code, 9, static_cast(values[2])); + // Exact FP64 instruction words from b90e2024732c6111, PCs 0xc8 through + // 0x130. + test.code.insert(test.code.end(), {0x7e060907u, 0x7e0a0901u, 0x7e020909u}); + store_pair(3, double(values[0])); + store_pair(5, double(values[1])); + store_pair(1, double(values[2])); + test.code.push_back(0x7e065f03u); + const double reciprocal = 1.0 / double(values[0]); + store_pair(3, reciprocal); + test.code.insert(test.code.end(), {0xd5650005u, 0x00020305u}); + const double product = double(values[1]) * double(values[2]); + store_pair(5, product); + test.code.insert(test.code.end(), {0xd54c0001u, 0x84060705u}); + const double result = std::fma(product, reciprocal, -double(values[2])); + store_pair(1, result); + test.code.push_back(0x7e041f01u); + AppendStoreVgpr(&test.code, 2, static_cast(test.expected.size())); + test.expected.push_back(std::bit_cast(static_cast(result))); + } + AppendEnd(&test.code); + test.opcodes = {O::V_MOV_B32, O::V_CVT_F64_I32, O::V_RCP_F64, + O::V_MUL_F64, O::V_FMA_F64, O::V_CVT_F32_F64, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpCapability Float64", + "OpTypeFloat 64", + "OpFDiv", + "Fma", + "SignedZeroInfNanPreserve 64", + "RoundingModeRTE 32"}; + test.ir_counts = {{"ConvertF64S32", 9}, + {"FPRecip64", 3}, + {"FPMul64", 3}, + {"FPFma64", 3}, + {"ConvertF32F64", 3}}; + + ShaderComputeInputInfo info{}; + std::vector normal_key, other_key; + BuildStageStaticKey(info, normal_key); + info.float_mode = 0xc4; + BuildStageStaticKey(info, other_key); + Require(test.name, "pipeline identity", + normal_key != other_key && normal_key.size() == other_key.size(), + "FP64 rounding modes must not share a compiled shader key"); + return test; +} + +TestCase VectorF64ModesModifiersAndExec() { + using O = ShaderOpcode; + TestCase test; + test.name = "VectorF64ModesModifiersAndExec"; + const auto load_pair = [&](u32 reg, uint64_t bits) { + AppendVMovLiteral(&test.code, reg, static_cast(bits)); + AppendVMovLiteral(&test.code, reg + 1, static_cast(bits >> 32)); + }; + const auto store_pair = [&](u32 reg, uint64_t bits) { + AppendStoreVgpr(&test.code, reg, static_cast(test.expected.size())); + AppendStoreVgpr(&test.code, reg + 1, + static_cast(test.expected.size() + 1)); + test.expected.insert(test.expected.end(), {static_cast(bits), + static_cast(bits >> 32)}); + }; + const auto store_word = [&](u32 reg, u32 bits) { + AppendStoreVgpr(&test.code, reg, static_cast(test.expected.size())); + test.expected.push_back(bits); + }; + for (const auto [source, expected] : + {std::pair{0, 0x7ff0000000000000ull}, + {0x8000000000000000ull, 0xfff0000000000000ull}, + {0x7ff0000000000000ull, 0}, + {0xfff0000000000000ull, 0x8000000000000000ull}, + {0x0008000000000000ull, 0x7fe0000000000000ull}, + {0x7fe0000000000000ull, 0x0008000000000000ull}}) { + load_pair(1, source); + test.code.push_back(EncodeVop1(0x2f, 1, Vgpr(1))); + store_pair(1, expected); + } + // A fused result retains the residual that separate multiply/add would round + // to zero. + load_pair(1, std::bit_cast(1.0 + 0x1p-27)); + load_pair(3, std::bit_cast(1.0 - 0x1p-27)); + AppendVop3(&test.code, 0x14c, 1, Vgpr(1), Vgpr(3), 243u); + store_pair(1, std::bit_cast(-0x1p-54)); + // Unaligned pairs, SGPR pairs, inline floats, literal highword expansion, ABS + // then NEG. + test.code.push_back(EncodeSop1(0x03, 8, 255)); + test.code.push_back(0u); + test.code.push_back(EncodeSop1(0x03, 9, 255)); + test.code.push_back(0xbff00000u); + AppendVop3(&test.code, 0x165, 1, 8u, 244u, 0, 1, 0, false, 0, 1); + store_pair(1, std::bit_cast(-2.0)); + AppendVop3(&test.code, 0x165, 1, 255u, 242u); + test.code.push_back(0x40080000u); + store_pair(1, std::bit_cast(3.0)); + AppendVop3(&test.code, 0x165, 1, 248u, 242u); + store_pair(1, 0x3fc45f306dc9c882ull); + AppendVop3(&test.code, 0x184, 1, InlineU32(3), 0); + store_pair(1, std::bit_cast(3.0)); + AppendVop3(&test.code, 0x1af, 1, 244u, 0); + store_pair(1, std::bit_cast(0.5)); + // Preserve both destination words when EXEC is clear, restoring the original + // live lanes. + test.code.push_back(EncodeSop1(0x04, 12, 126)); + test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + test.code.push_back(EncodeVop1(0x04, 1, InlineU32(9))); + test.code.push_back(EncodeSop1(0x04, 126, 12)); + store_pair(1, std::bit_cast(0.5)); + for (const auto [source, expected] : + {std::pair{0x47f0000000000000ull, 0x7f7fffffu}, + {0xc7f0000000000000ull, 0xff7fffffu}, + {0x7ff0000000000000ull, 0x7f800000u}, + {0xfff0000000000000ull, 0xff800000u}, + {0x3800000000000000ull, 0}, + {0xb800000000000000ull, 0x80000000u}, + {0x3810000000000000ull, 0x00800000u}, + {0x3ff0000010000000ull, 0x3f800000u}}) { + load_pair(1, source); + AppendVop3(&test.code, 0x18f, 1, Vgpr(1), 0); + store_word(1, expected); + } + load_pair(1, 0x7ff8123456789abcull); + test.code.push_back(EncodeVop1(0x2f, 1, Vgpr(1))); + AppendVMovLiteral(&test.code, 3, 0x7ff80000u); + test.code.push_back(EncodeVop2(0x1b, 2, Vgpr(3), 2)); + store_word(2, 0x7ff80000u); + AppendEnd(&test.code); + test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, + O::V_CVT_F64_I32, O::V_RCP_F64, O::V_MUL_F64, + O::V_FMA_F64, O::V_CVT_F32_F64, O::V_AND_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + TestCase VectorRcpIflagF32IntegerReciprocal() { using O = ShaderOpcode; @@ -28874,6 +29031,8 @@ std::vector MakeCases() { AddCase(CvtF32ToIntSaturatesNaNAndOutOfRange); AddCase(VectorSpecialF32FlushesDenormalInputs); AddCase(VectorRcpIflagF32IntegerReciprocal); + AddCase(VectorF64CapturedScreenSpaceShadows); + AddCase(VectorF64ModesModifiersAndExec); AddCase(VectorSinCosMaxFiniteSpecialCases); AddCase(VectorCompareOps); AddCase(VectorVop3CompareEqI64OnGpu);