diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 5910f349e..b3fc3a079 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -244,7 +244,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { Fail(program, "shared operation has invalid resource kind"); } - if (inst.GetOpcode() == IR::ValueOpcode::SharedAtomicOr64) { + if (shared_access == IR::SharedAccess::Atomic && + IR::SharedComponentCount(inst.GetOpcode()) == 2u) { if (kind != IR::ResourceKind::Lds || program.stage != ShaderType::Compute) { Fail(program, "64-bit shared atomics require compute LDS"); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp index 9480911bb..db5773b4c 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp @@ -444,6 +444,18 @@ uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) { return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan); } +uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) { + return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThanEqual); +} + +uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) { + return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThanEqual); +} + +uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) { + return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThanEqual); +} + uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) { return EmitNative(state, arg0, arg0); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index 02ac7e8fd..d385d1970 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -543,7 +543,9 @@ spv::Op ImageAtomicOpcode(IR::ValueOpcode opcode) { switch (opcode) { case IR::ValueOpcode::ImageAtomicSwap32: return spv::OpAtomicExchange; case IR::ValueOpcode::ImageAtomicIAdd32: return spv::OpAtomicIAdd; + case IR::ValueOpcode::ImageAtomicSMin32: return spv::OpAtomicSMin; case IR::ValueOpcode::ImageAtomicUMin32: return spv::OpAtomicUMin; + case IR::ValueOpcode::ImageAtomicSMax32: return spv::OpAtomicSMax; case IR::ValueOpcode::ImageAtomicUMax32: return spv::OpAtomicUMax; case IR::ValueOpcode::ImageAtomicAnd32: return spv::OpAtomicAnd; case IR::ValueOpcode::ImageAtomicOr32: return spv::OpAtomicOr; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index f18314d25..cbfb7344f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -111,6 +111,9 @@ uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1); uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1); EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t) @@ -134,6 +137,8 @@ uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) { } inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32; EMIT_NATIVE(FPOrdEqual64, OpFOrdEqual, U1, uint32_t, uint32_t) +EMIT_NATIVE(FPOrdLessThanEqual64, OpFOrdLessThanEqual, U1, uint32_t, uint32_t) +EMIT_NATIVE(FPOrdGreaterThanEqual64, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t) inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32; inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32; inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32; @@ -150,6 +155,7 @@ inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32; EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t) EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t) EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t) +EMIT_NATIVE(FPAdd64, OpFAdd, F64, uint32_t, uint32_t) EMIT_NATIVE(FPMul64, OpFMul, F64, uint32_t, uint32_t) inline constexpr auto EmitFPFma64 = EmitGlsl; @@ -223,7 +229,7 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst); -void EmitSharedAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst); +void EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst); void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst); @@ -259,6 +265,7 @@ inline constexpr auto EmitBufferAtomicUMin32 = EmitAtomic32; inline constexpr auto EmitBufferAtomicSMax32 = EmitAtomic32; inline constexpr auto EmitBufferAtomicUMax32 = EmitAtomic32; inline constexpr auto EmitBufferAtomicAnd32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicAnd64 = EmitBufferAtomic64; inline constexpr auto EmitBufferAtomicOr32 = EmitAtomic32; inline constexpr auto EmitBufferAtomicOr64 = EmitBufferAtomic64; inline constexpr auto EmitBufferAtomicXor32 = EmitAtomic32; @@ -280,6 +287,7 @@ inline constexpr auto EmitSharedAtomicFMin32 = EmitSharedFloatAtomic; inline constexpr auto EmitSharedAtomicFMax32 = EmitSharedFloatAtomic; inline constexpr auto EmitSharedAtomicSwap32 = EmitAtomic32; inline constexpr auto EmitSharedAtomicIAdd32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicIAdd64 = EmitSharedAtomic64; inline constexpr auto EmitSharedAtomicISub32 = EmitAtomic32; inline constexpr auto EmitSharedAtomicInc32 = EmitSharedIncDec; inline constexpr auto EmitSharedAtomicDec32 = EmitSharedIncDec; @@ -289,6 +297,7 @@ inline constexpr auto EmitSharedAtomicSMax32 = EmitAtomic32; inline constexpr auto EmitSharedAtomicUMax32 = EmitAtomic32; inline constexpr auto EmitSharedAtomicAnd32 = EmitAtomic32; inline constexpr auto EmitSharedAtomicOr32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicOr64 = EmitSharedAtomic64; inline constexpr auto EmitSharedAtomicXor32 = EmitAtomic32; inline constexpr auto EmitDataAppend = EmitAppendConsume; inline constexpr auto EmitDataConsume = EmitAppendConsume; @@ -303,7 +312,9 @@ inline constexpr auto EmitImageSampleRaw = EmitImage; inline constexpr auto EmitImageGatherRaw = EmitImage; inline constexpr auto EmitImageAtomicSwap32 = EmitImage; inline constexpr auto EmitImageAtomicIAdd32 = EmitImage; +inline constexpr auto EmitImageAtomicSMin32 = EmitImage; inline constexpr auto EmitImageAtomicUMin32 = EmitImage; +inline constexpr auto EmitImageAtomicSMax32 = EmitImage; inline constexpr auto EmitImageAtomicUMax32 = EmitImage; inline constexpr auto EmitImageAtomicAnd32 = EmitImage; inline constexpr auto EmitImageAtomicOr32 = EmitImage; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 91520d405..1a47bfc1c 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -562,6 +562,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) { case IR::ValueOpcode::BufferAtomicSwap64: case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange; case IR::ValueOpcode::BufferAtomicIAdd32: + case IR::ValueOpcode::SharedAtomicIAdd64: case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd; case IR::ValueOpcode::BufferAtomicISub32: case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub; @@ -574,6 +575,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) { case IR::ValueOpcode::BufferAtomicUMax32: case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax; case IR::ValueOpcode::BufferAtomicAnd32: + case IR::ValueOpcode::BufferAtomicAnd64: case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd; case IR::ValueOpcode::BufferAtomicOr32: case IR::ValueOpcode::BufferAtomicOr64: @@ -1115,7 +1117,7 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) { }); } -void EmitSharedAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst) { +void EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; const auto& mem = ctx.Memory(inst); EnsureLdsStorage(state); diff --git a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp index 972f979d0..851da347d 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp @@ -193,7 +193,9 @@ constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = { constexpr Detail::OpcodeMap MIMG_ATOMIC_OPCODE_LIST[] = { {0x0fu, Opcode::IMAGE_ATOMIC_SWAP}, {0x11u, Opcode::IMAGE_ATOMIC_ADD}, + {0x14u, Opcode::IMAGE_ATOMIC_SMIN}, {0x15u, Opcode::IMAGE_ATOMIC_UMIN}, + {0x16u, Opcode::IMAGE_ATOMIC_SMAX}, {0x17u, Opcode::IMAGE_ATOMIC_UMAX}, {0x18u, Opcode::IMAGE_ATOMIC_AND}, {0x19u, Opcode::IMAGE_ATOMIC_OR}, diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index 727646e66..af5a58b42 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -61,6 +61,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = { {0x3fu, Opcode::BUFFER_ATOMIC_FMIN, 1, 32}, {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32}, {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32}, + {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32}, {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32}, }; @@ -106,7 +107,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = { {0x39u, Opcode::DS_READ_I8, 1, 8, true}, {0x3au, Opcode::DS_READ_U8, 1, 8}, {0x3bu, Opcode::DS_READ_I16, 1, 16, true}, {0x3cu, Opcode::DS_READ_U16, 1, 16}, {0x3du, Opcode::DS_CONSUME, 1, 32}, {0x3eu, Opcode::DS_APPEND, 1, 32}, - {0x4au, Opcode::DS_OR_B64, 2, 32}, + {0x40u, Opcode::DS_ADD_U64, 2, 32}, {0x4au, Opcode::DS_OR_B64, 2, 32}, {0x4du, Opcode::DS_WRITE_B64, 2, 32}, {0x4eu, Opcode::DS_WRITE2_B64, 4, 32}, {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32}, {0x76u, Opcode::DS_READ_B64, 2, 32}, {0x77u, Opcode::DS_READ2_B64, 4, 32}, {0x78u, Opcode::DS_READ2ST64_B64, 4, 32}, @@ -168,6 +169,7 @@ bool IsDsWriteOpcode(Opcode opcode) { bool IsDsAtomicOpcode(Opcode opcode) { switch (opcode) { case Opcode::DS_ADD_U32: + case Opcode::DS_ADD_U64: case Opcode::DS_ADD_RTN_U32: case Opcode::DS_SUB_U32: case Opcode::DS_SUB_RTN_U32: diff --git a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp index ae474665d..6c8b7bb84 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp @@ -46,6 +46,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = { {0x09u, Opcode::S_WQM_B32}, {0x0au, Opcode::S_WQM_B64}, {0x0bu, Opcode::S_BREV_B32}, + {0x0cu, Opcode::S_BREV_B64}, {0x0fu, Opcode::S_BCNT1_I32_B32}, {0x10u, Opcode::S_BCNT1_I32_B64}, {0x13u, Opcode::S_FF1_I32_B32}, diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index a65bba2a8..676405bca 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -504,6 +504,7 @@ std::string InstructionToString(const Instruction& inst) { OperandToString(inst.src0).c_str())); case Opcode::S_ABS_I32: case Opcode::S_BREV_B32: + case Opcode::S_BREV_B64: case Opcode::S_BCNT1_I32_B32: case Opcode::S_FLBIT_I32_B32: case Opcode::S_FF1_I32_B32: @@ -575,7 +576,9 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::IMAGE_STORE_MIP: case Opcode::IMAGE_ATOMIC_SWAP: case Opcode::IMAGE_ATOMIC_ADD: + case Opcode::IMAGE_ATOMIC_SMIN: case Opcode::IMAGE_ATOMIC_UMIN: + case Opcode::IMAGE_ATOMIC_SMAX: case Opcode::IMAGE_ATOMIC_UMAX: case Opcode::IMAGE_ATOMIC_AND: case Opcode::IMAGE_ATOMIC_OR: @@ -642,6 +645,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::BUFFER_ATOMIC_SMAX: case Opcode::BUFFER_ATOMIC_UMAX: case Opcode::BUFFER_ATOMIC_AND: + case Opcode::BUFFER_ATOMIC_AND_X2: case Opcode::BUFFER_ATOMIC_OR: case Opcode::BUFFER_ATOMIC_OR_X2: case Opcode::BUFFER_ATOMIC_XOR: @@ -664,6 +668,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::FLAT_STORE_DWORDX3: case Opcode::FLAT_STORE_DWORDX4: case Opcode::DS_ADD_U32: + case Opcode::DS_ADD_U64: case Opcode::DS_ADD_RTN_U32: case Opcode::DS_SUB_U32: case Opcode::DS_SUB_RTN_U32: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 02c185fce..79326c115 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -47,6 +47,7 @@ enum class Opcode { S_ABS_I32, S_ABSDIFF_I32, S_BREV_B32, + S_BREV_B64, S_BCNT1_I32_B32, S_BCNT1_I32_B64, S_FF1_I32_B32, @@ -244,6 +245,7 @@ enum class Opcode { V_MAD_U32_U24, V_MAD_U64_U32, V_FMA_F64, + V_ADD_F64, V_MUL_F64, V_FMA_F32, V_FMA_F16, @@ -360,6 +362,9 @@ enum class Opcode { V_CMP_NLT_F32, V_CMP_TRU_F32, V_CMP_EQ_F64, + V_CMP_LE_F64, + V_CMPX_LE_F64, + V_CMPX_GE_F64, V_CMPX_LT_F32, V_CMPX_EQ_F32, V_CMPX_LE_F32, @@ -390,12 +395,18 @@ enum class Opcode { V_CMP_NE_I16, V_CMP_GE_I16, V_CMPX_LT_I16, + V_CMPX_EQ_I16, + V_CMPX_LE_I16, + V_CMPX_GT_I16, + V_CMPX_NE_I16, + V_CMPX_GE_I16, V_CMP_LT_F16, V_CMP_EQ_F16, V_CMP_LE_F16, V_CMP_GT_F16, V_CMP_LG_F16, V_CMP_GE_F16, + V_CMP_NGE_F16, V_CMP_NGT_F16, V_CMP_NEQ_F16, V_CMP_NLT_F16, @@ -432,11 +443,16 @@ enum class Opcode { V_CMP_GE_U32, V_CMP_T_U32, V_CMP_EQ_I64, + V_CMP_LT_I64, + V_CMP_LE_I64, V_CMP_LT_U64, V_CMP_EQ_U64, + V_CMP_LE_U64, V_CMP_GT_U64, V_CMP_NE_U64, + V_CMP_GE_U64, V_CMPX_NE_I64, + V_CMPX_LE_U64, V_CMPX_NE_U64, V_CMPX_LT_U32, V_CMPX_EQ_U32, @@ -496,6 +512,7 @@ enum class Opcode { BUFFER_ATOMIC_SMAX, BUFFER_ATOMIC_UMAX, BUFFER_ATOMIC_AND, + BUFFER_ATOMIC_AND_X2, BUFFER_ATOMIC_OR, BUFFER_ATOMIC_OR_X2, BUFFER_ATOMIC_XOR, @@ -516,6 +533,7 @@ enum class Opcode { FLAT_STORE_DWORDX3, FLAT_STORE_DWORDX4, DS_ADD_U32, + DS_ADD_U64, DS_ADD_RTN_U32, DS_SUB_U32, DS_SUB_RTN_U32, @@ -582,7 +600,9 @@ enum class Opcode { IMAGE_STORE_MIP, IMAGE_ATOMIC_SWAP, IMAGE_ATOMIC_ADD, + IMAGE_ATOMIC_SMIN, IMAGE_ATOMIC_UMIN, + IMAGE_ATOMIC_SMAX, IMAGE_ATOMIC_UMAX, IMAGE_ATOMIC_AND, IMAGE_ATOMIC_OR, diff --git a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp index 9c56d989c..9bcdf7532 100644 --- a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp @@ -228,7 +228,8 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = { {0x19u, Opcode::V_CMPX_NGE_F32}, {0x1au, Opcode::V_CMPX_NLG_F32}, {0x1bu, Opcode::V_CMPX_NGT_F32}, {0x1cu, Opcode::V_CMPX_NLE_F32}, {0x1du, Opcode::V_CMPX_NEQ_F32}, {0x1eu, Opcode::V_CMPX_NLT_F32}, - {0x22u, Opcode::V_CMP_EQ_F64, false}, + {0x22u, Opcode::V_CMP_EQ_F64, false}, {0x23u, Opcode::V_CMP_LE_F64, false}, + {0x33u, Opcode::V_CMPX_LE_F64, false}, {0x36u, Opcode::V_CMPX_GE_F64, false}, {0x80u, Opcode::V_CMP_F_I32}, {0x81u, Opcode::V_CMP_LT_I32}, {0x82u, Opcode::V_CMP_EQ_I32}, {0x83u, Opcode::V_CMP_LE_I32}, {0x84u, Opcode::V_CMP_GT_I32}, {0x85u, Opcode::V_CMP_NE_I32}, @@ -240,7 +241,9 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = { {0x92u, Opcode::V_CMPX_EQ_I32}, {0x93u, Opcode::V_CMPX_LE_I32}, {0x94u, Opcode::V_CMPX_GT_I32}, {0x95u, Opcode::V_CMPX_NE_I32}, {0x96u, Opcode::V_CMPX_GE_I32}, {0x98u, Opcode::V_CMPX_CLASS_F32}, - {0x99u, Opcode::V_CMPX_LT_I16, false}, + {0x99u, Opcode::V_CMPX_LT_I16, false}, {0x9au, Opcode::V_CMPX_EQ_I16, false}, + {0x9bu, Opcode::V_CMPX_LE_I16, false}, {0x9cu, Opcode::V_CMPX_GT_I16, false}, + {0x9du, Opcode::V_CMPX_NE_I16, false}, {0x9eu, Opcode::V_CMPX_GE_I16, false}, {0xa9u, Opcode::V_CMP_LT_U16}, {0xaau, Opcode::V_CMP_EQ_U16}, {0xabu, Opcode::V_CMP_LE_U16}, {0xacu, Opcode::V_CMP_GT_U16}, {0xadu, Opcode::V_CMP_NE_U16}, {0xaeu, Opcode::V_CMP_GE_U16}, @@ -250,16 +253,19 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = { {0xc3u, Opcode::V_CMP_LE_U32}, {0xc4u, Opcode::V_CMP_GT_U32}, {0xc5u, Opcode::V_CMP_NE_U32}, {0xc6u, Opcode::V_CMP_GE_U32}, {0xc7u, Opcode::V_CMP_T_U32}, {0xa2u, Opcode::V_CMP_EQ_I64, false}, + {0xa1u, Opcode::V_CMP_LT_I64, false}, {0xa3u, Opcode::V_CMP_LE_I64, false}, {0xb5u, Opcode::V_CMPX_NE_I64, false}, {0xd1u, Opcode::V_CMPX_LT_U32}, {0xd2u, Opcode::V_CMPX_EQ_U32}, {0xd3u, Opcode::V_CMPX_LE_U32}, {0xd4u, Opcode::V_CMPX_GT_U32}, {0xd5u, Opcode::V_CMPX_NE_U32}, {0xd6u, Opcode::V_CMPX_GE_U32}, {0xe1u, Opcode::V_CMP_LT_U64, false}, - {0xe2u, Opcode::V_CMP_EQ_U64, false}, {0xe4u, Opcode::V_CMP_GT_U64, false}, - {0xe5u, Opcode::V_CMP_NE_U64, false}, + {0xe2u, Opcode::V_CMP_EQ_U64, false}, {0xe3u, Opcode::V_CMP_LE_U64, false}, + {0xe4u, Opcode::V_CMP_GT_U64, false}, {0xe5u, Opcode::V_CMP_NE_U64, false}, + {0xe6u, Opcode::V_CMP_GE_U64, false}, {0xf3u, Opcode::V_CMPX_LE_U64, false}, {0xf5u, Opcode::V_CMPX_NE_U64, false}, {0xc9u, Opcode::V_CMP_LT_F16}, {0xcau, Opcode::V_CMP_EQ_F16}, {0xcbu, Opcode::V_CMP_LE_F16}, {0xccu, Opcode::V_CMP_GT_F16}, {0xcdu, Opcode::V_CMP_LG_F16}, - {0xceu, Opcode::V_CMP_GE_F16}, {0xebu, Opcode::V_CMP_NGT_F16}, + {0xceu, Opcode::V_CMP_GE_F16}, {0xe9u, Opcode::V_CMP_NGE_F16}, + {0xebu, Opcode::V_CMP_NGT_F16}, {0xedu, Opcode::V_CMP_NEQ_F16}, {0xeeu, Opcode::V_CMP_NLT_F16}, {0xd9u, Opcode::V_CMPX_LT_F16}, {0xdau, Opcode::V_CMPX_EQ_F16}, {0xdbu, Opcode::V_CMPX_LE_F16}, {0xdcu, Opcode::V_CMPX_GT_F16}, @@ -281,6 +287,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = { {0x147u, Opcode::V_CUBEMA_F32}, {0x14bu, Opcode::V_FMA_F32}, {0x14cu, Opcode::V_FMA_F64}, + {0x164u, Opcode::V_ADD_F64}, {0x165u, Opcode::V_MUL_F64}, {0x148u, Opcode::V_BFE_U32}, {0x149u, Opcode::V_BFE_I32}, @@ -831,6 +838,9 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) { case Opcode::V_CMP_LT_F32: case Opcode::V_CMP_EQ_F32: case Opcode::V_CMP_EQ_F64: + case Opcode::V_CMP_LE_F64: + case Opcode::V_CMPX_LE_F64: + case Opcode::V_CMPX_GE_F64: case Opcode::V_CMP_LE_F32: case Opcode::V_CMP_GT_F32: case Opcode::V_CMP_LG_F32: @@ -863,6 +873,7 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) { case Opcode::V_CMP_GT_F16: case Opcode::V_CMP_LG_F16: case Opcode::V_CMP_GE_F16: + case Opcode::V_CMP_NGE_F16: case Opcode::V_CMP_NGT_F16: case Opcode::V_CMP_NEQ_F16: case Opcode::V_CMP_NLT_F16: @@ -1141,7 +1152,9 @@ VopcSdwaFields DecodeVopcSdwaFields(uint32_t modifier) { } bool SupportsVopcSdwa(Opcode opcode) { - return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_EQ_F64; + return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_EQ_F64 && + opcode != Opcode::V_CMP_LE_F64 && opcode != Opcode::V_CMPX_LE_F64 && + opcode != Opcode::V_CMPX_GE_F64; } void DecodeVopcSdwa(uint32_t pc, std::span code, uint32_t word_index, @@ -1206,6 +1219,7 @@ void DecodeVopcDpp(uint32_t pc, std::span code, uint32_t word_in uint32_t NativeVop3SourceCount(Opcode opcode) { switch (opcode) { + case Opcode::V_ADD_F64: case Opcode::V_MUL_F64: case Opcode::V_MUL_LO_U32: case Opcode::V_MUL_HI_U32: @@ -1357,6 +1371,7 @@ bool SupportsNativeVop3SourceModifiers(Opcode opcode) { case Opcode::V_MAX_F32: case Opcode::V_MAC_F32: case Opcode::V_MAD_F32: + case Opcode::V_ADD_F64: case Opcode::V_MUL_F64: case Opcode::V_FMA_F64: case Opcode::V_FMA_F32: @@ -1462,6 +1477,8 @@ void ApplyNativeVop3SourceModifiers(Instruction& inst, uint32_t abs, uint32_t ne bool IsVopcCompareExec(Opcode opcode) { switch (opcode) { + case Opcode::V_CMPX_LE_F64: + case Opcode::V_CMPX_GE_F64: case Opcode::V_CMPX_LT_F32: case Opcode::V_CMPX_EQ_F32: case Opcode::V_CMPX_LE_F32: @@ -1489,8 +1506,14 @@ bool IsVopcCompareExec(Opcode opcode) { case Opcode::V_CMPX_NE_U32: case Opcode::V_CMPX_GE_U32: case Opcode::V_CMPX_NE_I64: + case Opcode::V_CMPX_LE_U64: case Opcode::V_CMPX_NE_U64: case Opcode::V_CMPX_LT_I16: + case Opcode::V_CMPX_EQ_I16: + case Opcode::V_CMPX_LE_I16: + case Opcode::V_CMPX_GT_I16: + case Opcode::V_CMPX_NE_I16: + case Opcode::V_CMPX_GE_I16: case Opcode::V_CMPX_LT_U16: case Opcode::V_CMPX_EQ_U16: case Opcode::V_CMPX_GT_U16: diff --git a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp index 59208f1e1..7c68f297d 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp @@ -1,5 +1,7 @@ #include "graphics/shader/recompiler/frontend/translate/Translator.h" +#include + namespace Libs::Graphics::ShaderRecompiler::Frontend { void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar, @@ -26,9 +28,20 @@ void Translator::EmitCompareConstant(const Decoder::Instruction& inst, bool valu void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type, bool scalar, bool cmpx) { - const auto lhs = ReadOperand(inst.src0, type); - const auto rhs = ReadOperand(inst.src1, type); - EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), scalar, cmpx); + const bool signed_64 = opcode == IR::ValueOpcode::SLessThan64 || + opcode == IR::ValueOpcode::SLessThanEqual64 || + inst.opcode == Decoder::Opcode::V_CMP_EQ_I64 || + inst.opcode == Decoder::Opcode::V_CMPX_NE_I64; + const auto read = [&](const Decoder::Operand& operand) { + // RDNA2 expands signed 64-bit integer literals by sign extension. + if (signed_64 && operand.kind == Decoder::OperandKind::LiteralConstant) { + return IR::Value(static_cast( + static_cast(std::bit_cast(operand.value)))); + } + return ReadOperand(operand, type); + }; + EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {read(inst.src0), read(inst.src1)})), scalar, + cmpx); } void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index a81da2ebd..9f2e2bf11 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -966,6 +966,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicUMax32); case Decoder::Opcode::BUFFER_ATOMIC_AND: return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicAnd32); + case Decoder::Opcode::BUFFER_ATOMIC_AND_X2: + return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicAnd64); case Decoder::Opcode::BUFFER_ATOMIC_OR: return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicOr32); case Decoder::Opcode::BUFFER_ATOMIC_OR_X2: @@ -979,6 +981,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::DS_ADD_U32: return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, false); + case Decoder::Opcode::DS_ADD_U64: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd64, false); case Decoder::Opcode::DS_ADD_RTN_U32: return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, true); case Decoder::Opcode::DS_SUB_U32: @@ -1030,8 +1034,12 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32); case Decoder::Opcode::IMAGE_ATOMIC_ADD: return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32); + case Decoder::Opcode::IMAGE_ATOMIC_SMIN: + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMin32); case Decoder::Opcode::IMAGE_ATOMIC_UMIN: return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32); + case Decoder::Opcode::IMAGE_ATOMIC_SMAX: + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMax32); case Decoder::Opcode::IMAGE_ATOMIC_UMAX: return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMax32); case Decoder::Opcode::IMAGE_ATOMIC_AND: diff --git a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp index 5fdf858e8..ecd383849 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp @@ -155,6 +155,12 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) { case O::S_BREV_B32: return SimpleInteger(inst, IR::ValueOpcode::BitReverse32, IR::Type::U32, false, false, false); + case O::S_BREV_B64: { + const auto value = ReadU32Pair(inst.src0); + WriteU32Pair(inst.dst, {IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {value[1]})), + IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {value[0]}))}); + return; + } case O::S_BCNT1_I32_B32: return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false, true); diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 2c918dc4f..b06c3a6f3 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -894,10 +894,26 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_ case Decoder::Opcode::V_CVT_F64_F32: case Decoder::Opcode::V_CVT_F64_U32: include_vector(inst.dst, 2u); break; case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]]; + case Decoder::Opcode::V_ADD_F64: case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]]; case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]]; case Decoder::Opcode::V_CVT_F32_F64: include_vector(inst.src0, 2u); break; case Decoder::Opcode::V_CMP_EQ_F64: + case Decoder::Opcode::V_CMP_LE_F64: + case Decoder::Opcode::V_CMPX_LE_F64: + case Decoder::Opcode::V_CMPX_GE_F64: + case Decoder::Opcode::V_CMP_EQ_I64: + case Decoder::Opcode::V_CMP_LT_I64: + case Decoder::Opcode::V_CMP_LE_I64: + case Decoder::Opcode::V_CMP_LT_U64: + case Decoder::Opcode::V_CMP_EQ_U64: + case Decoder::Opcode::V_CMP_LE_U64: + case Decoder::Opcode::V_CMP_GT_U64: + case Decoder::Opcode::V_CMP_NE_U64: + case Decoder::Opcode::V_CMP_GE_U64: + case Decoder::Opcode::V_CMPX_NE_I64: + case Decoder::Opcode::V_CMPX_LE_U64: + case Decoder::Opcode::V_CMPX_NE_U64: include_vector(inst.src0, 2u); include_vector(inst.src1, 2u); break; @@ -905,6 +921,8 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_ } if (inst.family == Decoder::Family::DS) { switch (inst.opcode) { + case Decoder::Opcode::DS_ADD_U64: + case Decoder::Opcode::DS_OR_B64: case Decoder::Opcode::DS_WRITE_B64: case Decoder::Opcode::DS_WRITE_B96: case Decoder::Opcode::DS_WRITE_B128: include_vector(inst.src1, inst.data_dwords); break; diff --git a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp index 23558d5f8..344861bf3 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp @@ -114,18 +114,40 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, true); return; + case O::V_CMP_LT_I64: + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, false); + return; + case O::V_CMP_LE_I64: + EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, false); + return; + case O::V_CMP_LE_U64: + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, false); + return; + case O::V_CMPX_LE_U64: + EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, true); + return; + case O::V_CMP_GE_U64: + EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false, false); + return; + case O::V_CMP_EQ_U16: EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false); return; case O::V_CMP_EQ_I16: EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, false); return; + case O::V_CMPX_EQ_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, true); + return; case O::V_CMP_NE_U16: EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false); return; case O::V_CMP_NE_I16: EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false); return; + case O::V_CMPX_NE_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, true); + return; case O::V_CMP_GT_U16: EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, false); return; @@ -150,9 +172,15 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_CMP_GT_I16: EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, false); return; + case O::V_CMPX_GT_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, true); + return; case O::V_CMP_GE_I16: EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, false); return; + case O::V_CMPX_GE_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, true); + return; case O::V_CMP_LT_I16: EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, false); return; @@ -162,6 +190,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_CMP_LE_I16: EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, false); return; + case O::V_CMPX_LE_I16: + EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, true); + return; case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return; case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return; @@ -171,6 +202,15 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_CMP_EQ_F64: EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false, false); return; + case O::V_CMP_LE_F64: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, false); + return; + case O::V_CMPX_LE_F64: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, true); + return; + case O::V_CMPX_GE_F64: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual64, false, true); + return; case O::V_CMPX_EQ_F32: EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true); return; @@ -273,6 +313,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_CMPX_LE_F16: EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, true); return; + case O::V_CMP_NGE_F16: + EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, true, false); + return; case O::V_CMP_NGT_F16: EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, false); return; @@ -424,6 +467,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_SUB_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, false); case O::V_SUBREV_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, true); case O::V_MUL_F32: return FloatBinary(inst, IR::ValueOpcode::FPMul32, false); + case O::V_ADD_F64: return FloatBinary(inst, IR::ValueOpcode::FPAdd64, false); case O::V_MUL_F64: return FloatBinary(inst, IR::ValueOpcode::FPMul64, false); case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false); case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false); diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index 184509ccf..2ba0dbd3a 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -123,6 +123,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) { case ValueOpcode::BufferAtomicSMax32: case ValueOpcode::BufferAtomicUMax32: case ValueOpcode::BufferAtomicAnd32: + case ValueOpcode::BufferAtomicAnd64: case ValueOpcode::BufferAtomicOr32: case ValueOpcode::BufferAtomicOr64: case ValueOpcode::BufferAtomicXor32: @@ -135,6 +136,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) { uint32_t BufferComponentCount(ValueOpcode opcode) { switch (opcode) { case ValueOpcode::BufferAtomicSwap64: + case ValueOpcode::BufferAtomicAnd64: case ValueOpcode::BufferAtomicOr64: case ValueOpcode::LoadBufferU32x2: case ValueOpcode::StoreBufferU32x2: return 2u; @@ -164,6 +166,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) { case ValueOpcode::SharedAtomicFMax32: case ValueOpcode::SharedAtomicSwap32: case ValueOpcode::SharedAtomicIAdd32: + case ValueOpcode::SharedAtomicIAdd64: case ValueOpcode::SharedAtomicISub32: case ValueOpcode::SharedAtomicInc32: case ValueOpcode::SharedAtomicDec32: @@ -183,6 +186,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) { uint32_t SharedComponentCount(ValueOpcode opcode) { switch (opcode) { + case ValueOpcode::SharedAtomicIAdd64: case ValueOpcode::SharedAtomicOr64: case ValueOpcode::LoadSharedU32x2: case ValueOpcode::WriteSharedU32x2: return 2u; @@ -218,7 +222,9 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) { return {ImageAccess::Write, ImageResourceClass::Storage, false}; case ValueOpcode::ImageAtomicSwap32: case ValueOpcode::ImageAtomicIAdd32: + case ValueOpcode::ImageAtomicSMin32: case ValueOpcode::ImageAtomicUMin32: + case ValueOpcode::ImageAtomicSMax32: case ValueOpcode::ImageAtomicUMax32: case ValueOpcode::ImageAtomicAnd32: case ValueOpcode::ImageAtomicOr32: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 6774ad8a0..3f5df76b3 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -137,6 +137,9 @@ VALUE_OPCODE(SMedTri32, U32, U32, U32, U32) VALUE_OPCODE(UMedTri32, U32, U32, U32, U32) VALUE_OPCODE(SLessThan32, U1, U32, U32) VALUE_OPCODE(SLessThan64, U1, U64, U64) +VALUE_OPCODE(SLessThanEqual64, U1, U64, U64) +VALUE_OPCODE(ULessThanEqual64, U1, U64, U64) +VALUE_OPCODE(UGreaterThanEqual64, U1, U64, U64) VALUE_OPCODE(ULessThan32, U1, U32, U32) VALUE_OPCODE(ULessThan64, U1, U64, U64) VALUE_OPCODE(IEqual32, U1, U32, U32) @@ -156,6 +159,8 @@ VALUE_OPCODE(LogicalXor, U1, U1, U1) VALUE_OPCODE(LogicalNot, U1, U1) VALUE_OPCODE(FPOrdEqual32, U1, F32, F32) VALUE_OPCODE(FPOrdEqual64, U1, F64, F64) +VALUE_OPCODE(FPOrdLessThanEqual64, U1, F64, F64) +VALUE_OPCODE(FPOrdGreaterThanEqual64, U1, F64, F64) VALUE_OPCODE(FPUnordEqual32, U1, F32, F32) VALUE_OPCODE(FPOrdNotEqual32, U1, F32, F32) VALUE_OPCODE(FPUnordNotEqual32, U1, F32, F32) @@ -174,6 +179,7 @@ VALUE_OPCODE(FPAdd32, F32, F32, F32) VALUE_OPCODE(FPSub32, F32, F32, F32) VALUE_OPCODE(FPFma32, F32, F32, F32, F32) VALUE_OPCODE(FPMul32, F32, F32, F32) +VALUE_OPCODE(FPAdd64, F64, F64, F64) VALUE_OPCODE(FPMul64, F64, F64, F64) VALUE_OPCODE(FPFma64, F64, F64, F64, F64) VALUE_OPCODE(FPRecip64, F64, F64) @@ -246,6 +252,7 @@ VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicSMax32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicUMax32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicAnd32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicAnd64, U64, BufferResource, U32, U32, U32, U64, U1) VALUE_OPCODE(BufferAtomicOr32, U32, BufferResource, U32, U32, U32, U32, U1) VALUE_OPCODE(BufferAtomicOr64, U64, BufferResource, U32, U32, U32, U64, U1) VALUE_OPCODE(BufferAtomicXor32, U32, BufferResource, U32, U32, U32, U32, U1) @@ -268,6 +275,7 @@ VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicIAdd64, Void, U32, U64, U1) VALUE_OPCODE(SharedAtomicISub32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicInc32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicDec32, U32, U32, U32, U1) @@ -292,7 +300,9 @@ VALUE_OPCODE(ImageSampleRaw, U32x4, ImageResource, SamplerResource, ImageAddress VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress) VALUE_OPCODE(ImageAtomicSwap32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicIAdd32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicSMin32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicUMin32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicSMax32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1) diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index f4952e470..1e0c33986 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -575,6 +575,17 @@ void FoldInstruction(Block& block, Block::iterator instruction, case ValueOpcode::UGreaterThan64: FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a > b; }); return; + case ValueOpcode::ULessThanEqual64: + FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a <= b; }); + return; + case ValueOpcode::UGreaterThanEqual64: + FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a >= b; }); + return; + case ValueOpcode::SLessThanEqual64: + FoldU64Compare(inst, [](uint64_t a, uint64_t b) { + return std::bit_cast(a) <= std::bit_cast(b); + }); + return; case ValueOpcode::SLessThan64: FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return std::bit_cast(a) < std::bit_cast(b); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 945fed411..2311cc744 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -17364,6 +17364,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, case Opcode::BUFFER_ATOMIC_SMAX: case Opcode::BUFFER_ATOMIC_UMAX: case Opcode::BUFFER_ATOMIC_AND: + case Opcode::BUFFER_ATOMIC_AND_X2: case Opcode::BUFFER_ATOMIC_OR: case Opcode::BUFFER_ATOMIC_XOR: case Opcode::BUFFER_ATOMIC_FMIN: @@ -17383,6 +17384,7 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, case Opcode::FLAT_STORE_DWORDX3: case Opcode::FLAT_STORE_DWORDX4: case Opcode::DS_ADD_U32: + case Opcode::DS_ADD_U64: case Opcode::DS_ADD_RTN_U32: case Opcode::DS_SUB_U32: case Opcode::DS_SUB_RTN_U32: @@ -17439,6 +17441,8 @@ CoverageClass ClassifyOpcode(ShaderOpcode opcode, case Opcode::IMAGE_STORE_MIP: case Opcode::IMAGE_ATOMIC_SWAP: case Opcode::IMAGE_ATOMIC_ADD: + case Opcode::IMAGE_ATOMIC_SMIN: + case Opcode::IMAGE_ATOMIC_SMAX: case Opcode::IMAGE_ATOMIC_UMIN: case Opcode::IMAGE_ATOMIC_AND: case Opcode::IMAGE_ATOMIC_OR: @@ -18808,6 +18812,94 @@ TestCase ScalarBrevB32PreservesScc() { O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase ScalarBrevB64OperandsAndMasks(u32 wave_size) { + using O = ShaderOpcode; + TestCase test; + test.name = wave_size == 64 ? "ScalarBrevB64OperandsAndMasksWave64" + : "ScalarBrevB64OperandsAndMasksWave32"; + constexpr uint64_t values[] = {0, 1, 0x8000000000000000ull, + 0x0123456789abcdefull}; + for (const auto value : values) { + test.initial.push_back(static_cast(value)); + test.initial.push_back(static_cast(value >> 32u)); + } + test.expected = test.initial; + auto &code = test.code; + size_t instructions = 0; + const auto check = [&](u32 dst, u32 src, uint64_t value, bool scc, + std::optional literal = {}) { + uint64_t reversed = 0; + for (u32 bit = 0; bit < 64; ++bit) { + reversed |= ((value >> bit) & 1ull) << (63u - bit); + } + AppendVMovU32(&code, 1, 1); + AppendVMovU32(&code, 2, 0); + if (dst == 106u || dst == 126u) { + code.push_back(EncodeSop1(0x04, dst, src)); + src = dst; + } + code.push_back(EncodeSopc(0x06, InlineU32(scc ? 1u : 0u), InlineU32(1))); + code.push_back(EncodeSop1(0x0c, dst, src)); + if (literal) { + code.push_back(*literal); + } + code.push_back(EncodeSMovB32(24, dst)); + code.push_back(EncodeSMovB32(25, dst + 1u)); + code.push_back(EncodeSMovB32(26, 253)); + if (dst == 106u) { + code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1)); + } else if (dst == 126u) { + code.push_back(EncodeVop1(0x01, 2, InlineU32(1))); + } + code.push_back(EncodeSop1(0x04, 126, 193)); // Restore EXEC for readback. + const u32 expected[] = {static_cast(reversed), + static_cast(reversed >> 32u), scc ? 1u : 0u}; + for (u32 i = 0; i < std::size(expected); ++i) { + AppendStoreSgprAtLaneDwordOffset( + &code, 24 + i, 0, static_cast(test.expected.size())); + test.expected.insert(test.expected.end(), wave_size, expected[i]); + } + if (dst == 106u || dst == 126u) { + AppendStoreVgprAtLaneDwordOffset( + &code, 2, 0, static_cast(test.expected.size())); + for (u32 lane = 0; lane < wave_size; ++lane) { + test.expected.push_back(static_cast((reversed >> lane) & 1u)); + } + } + ++instructions; + }; + for (const u32 dst : {20u, 106u, 126u}) { + for (u32 i = 0; i < std::size(values); ++i) { + // Runtime inputs exercise the emitter, including in-place mask writes. + for (u32 word = 0; word < 2; ++word) { + AppendVMovU32(&code, 30, (2 * i + word) * 4u); + AppendBufferLoadDword(&code, 3, 30); + code.push_back(EncodeVop1(0x02, 20 + word, Vgpr(3))); + } + check(dst, 20, values[i], values[i] == 0); + } + } + check(20, 129, 1, false); + check(20, 193, ~uint64_t{0}, false); + check(20, 125, 0, true); + check(20, 255, 0x80000001u, true, 0x80000001u); + AppendEnd(&code); + test.initial.resize(test.expected.size()); + test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_MOV_B32, + O::S_MOV_B64, O::S_BREV_B64, O::S_CMP_EQ_U32, O::V_MOV_B32, + O::V_CNDMASK_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.decoded_counts = {{"S_BREV_B64", instructions}}; + test.required_spirv = {"OpBitReverse"}; + test.forbidden_spirv = {"OpTypeInt 64"}; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.has_compute_info = true; + return test; +} + TestCase ScalarBfeI32CapturedRawSignExtends() { using O = ShaderOpcode; @@ -22567,9 +22659,169 @@ TestCase VectorSpecialF32FlushesDenormalInputs() { O::V_SQRT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase VectorCompareInteger64Edges() { + using O = ShaderOpcode; + constexpr std::array, 10> pairs{{ + {0, 0}, {0, 1}, {1, 0}, {0xffffffffull, 0x100000000ull}, + {0x100000000ull, 0xffffffffull}, {0x8000000000000000ull, 0}, + {0x7fffffffffffffffull, 0x8000000000000000ull}, + {0xffffffffffffffffull, 0xffffffffffffffffull}, + {0xffffffff80000000ull, 0xffffffff7fffffffull}, + {0x1234567880000000ull, 0x123456787fffffffull}}}; + TestCase test; + test.name = "VectorCompareInteger64Edges"; + for (const auto& pair : pairs) + for (const auto bits : pair) + test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)}); + test.expected = test.initial; + const auto store_mask = [&](u32 sgpr, bool value) { + AppendStoreSgprPair(&test.code, sgpr, static_cast(test.expected.size())); + test.expected.insert(test.expected.end(), {u32(value), 0}); + }; + for (u32 i = 0; i < pairs.size(); ++i) { + for (u32 word = 0; word < 4; ++word) { + AppendVMovU32(&test.code, 31, (i * 4 + word) * 4); + AppendBufferLoadDword(&test.code, 1 + word, 31); + } + test.code.push_back(EncodeVop1(0x02, 8, Vgpr(3))); + test.code.push_back(EncodeVop1(0x02, 9, Vgpr(4))); + const auto lhs = pairs[i][0], rhs = pairs[i][1]; + const auto signed_lhs = std::bit_cast(lhs); + const auto signed_rhs = std::bit_cast(rhs); + for (const auto [opcode, value] : std::array, 4>{{ + {0xa1, signed_lhs < signed_rhs}, {0xa3, signed_lhs <= signed_rhs}, + {0xe3, lhs <= rhs}, {0xe6, lhs >= rhs}}}) { + test.code.push_back(EncodeVopc(opcode, Vgpr(1), 3)); + store_mask(106, value); + AppendVop3(&test.code, opcode, 20, Vgpr(1), 8); + store_mask(20, value); + } + // A negative literal expands by signedness, including equality comparisons. + for (const auto [opcode, value] : std::array, 4>{{ + {0xa1, -1 < signed_rhs}, {0xa3, -1 <= signed_rhs}, + {0xa2, -1 == signed_rhs}, {0xe3, 0xffffffffull <= rhs}}}) { + test.code.push_back(EncodeVopc(opcode, 255, 3)); + test.code.push_back(0xffffffffu); + store_mask(106, value); + } + AppendVop3(&test.code, 0xa3, 20, Vgpr(1), 255); + test.code.push_back(0xffffffffu); + store_mask(20, signed_lhs <= -1); + } + AppendEnd(&test.code); + test.initial.resize(test.expected.size()); + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, + O::V_CMP_LT_I64, O::V_CMP_LE_I64, O::V_CMP_EQ_I64, + O::V_CMP_LE_U64, O::V_CMP_GE_U64, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpSLessThan", "OpULessThanEqual", "OpUGreaterThanEqual"}; + return test; +} + +TestCase VectorCompareExecWaveMasks(u32 wave_size) { + using O = ShaderOpcode; + // Rows exercise signed 16-bit truncation, full-width unsigned ordering, and + // ordered FP64 comparisons (including NaN, signed zero, and subnormals). + constexpr std::array, 8>, 3> inputs{{ + {{{0x1111ffff, 0x22220000}, {0x33338000, 0x44447fff}, + {0x55550000, 0x66660000}, {0x77777fff, 0x88888000}, + {0x99990001, 0xaaaa0002}, {0xbbbb0002, 0xcccc0001}, + {0xddddffff, 0xeeeeffff}, {0xffff8001, 0x11118000}}}, + {{{0, 0}, {0, 1}, {1, 0}, {0xffffffffull, 0x100000000ull}, + {0x100000000ull, 0xffffffffull}, {0x8000000000000000ull, 0}, + {0xffffffffffffffffull, 0xffffffffffffffffull}, + {0x1234567880000000ull, 0x123456787fffffffull}}}, + {{{0, 0x8000000000000000ull}, {1, 0}, {0x8000000000000001ull, 0}, + {0x7ff8000000000001ull, 0}, {0, 0x7ff0000000000001ull}, + {0xc000000000000000ull, 0xbff0000000000000ull}, + {0x7ff0000000000000ull, 0x7ff0000000000000ull}, + {0x3ff0000000000001ull, 0x3ff0000000000000ull}}}}}; + struct Compare { u32 encoding; O opcode; u32 group; }; + constexpr std::array comparisons{ + Compare{0x99, O::V_CMPX_LT_I16, 0}, Compare{0x9a, O::V_CMPX_EQ_I16, 0}, + Compare{0x9b, O::V_CMPX_LE_I16, 0}, Compare{0x9c, O::V_CMPX_GT_I16, 0}, + Compare{0x9d, O::V_CMPX_NE_I16, 0}, Compare{0x9e, O::V_CMPX_GE_I16, 0}, + Compare{0xf3, O::V_CMPX_LE_U64, 1}, Compare{0x33, O::V_CMPX_LE_F64, 2}, + Compare{0x36, O::V_CMPX_GE_F64, 2}}; + constexpr u32 low_exec = 0xeeeeeeeeu, high_exec = 0x77777777u; + constexpr u32 vcc_lo = 0x12345678u, vcc_hi = 0x89abcdefu; + TestCase test; + test.name = wave_size == 32 ? "VectorCompareExecWave32Masks" : "VectorCompareExecWave64Masks"; + test.initial.assign(12 * wave_size, 0); + for (u32 group = 0; group < inputs.size(); ++group) + for (u32 lane = 0; lane < wave_size; ++lane) + for (u32 word = 0; word < 4; ++word) + test.initial[(group * 4 + word) * wave_size + lane] = + u32(inputs[group][lane % 8][word / 2] >> (32 * (word % 2))); + test.expected = test.initial; + auto& code = test.code; + code.push_back(EncodeSop1(0x04, 10, 126)); + AppendSMovLiteral(&code, 106, vcc_lo); + AppendSMovLiteral(&code, 107, vcc_hi); + for (const auto compare : comparisons) { + for (u32 word = 0; word < 4; ++word) { + code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0)); + AppendVMovU32(&code, 31, (compare.group * 4 + word) * wave_size * 4); + code.push_back(EncodeVop2(0x25, 31, Vgpr(31), 30)); + AppendBufferLoadDword(&code, 1 + word, 31); + } + std::array expected_exec{}; + for (u32 lane = 0; lane < wave_size; ++lane) { + const auto pair = inputs[compare.group][lane % 8]; + const auto a = std::bit_cast(uint16_t(pair[0])); + const auto b = std::bit_cast(uint16_t(pair[1])); + bool result = false; + switch (compare.encoding) { + case 0x99: result = a < b; break; + case 0x9a: result = a == b; break; + case 0x9b: result = a <= b; break; + case 0x9c: result = a > b; break; + case 0x9d: result = a != b; break; + case 0x9e: result = a >= b; break; + case 0xf3: result = pair[0] <= pair[1]; break; + case 0x33: result = std::bit_cast(pair[0]) <= std::bit_cast(pair[1]); break; + case 0x36: result = std::bit_cast(pair[0]) >= std::bit_cast(pair[1]); break; + } + if (result) expected_exec[lane / 32] |= 1u << (lane % 32); + } + expected_exec[0] &= low_exec; + expected_exec[1] &= high_exec; + for (const bool vop3 : {false, true}) { + AppendSMovLiteral(&code, 126, low_exec); + if (wave_size == 64) AppendSMovLiteral(&code, 127, high_exec); + if (vop3) AppendVop3(&code, compare.encoding, 20, Vgpr(1), Vgpr(3)); + else code.push_back(EncodeVopc(compare.encoding, Vgpr(1), 3)); + code.push_back(EncodeSMovB32(20, 126)); + code.push_back(EncodeSMovB32(21, 127)); + code.push_back(EncodeSop1(0x04, 126, 10)); + for (const auto [reg, value] : std::array, 4>{{ + {20, expected_exec[0]}, {21, expected_exec[1]}, + {106, vcc_lo}, {107, vcc_hi}}}) { + AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, u32(test.expected.size())); + test.expected.insert(test.expected.end(), wave_size, value); + } + } + test.opcodes.push_back(compare.opcode); + } + AppendEnd(&code); + test.initial.resize(test.expected.size()); + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.has_compute_info = true; + test.opcodes.insert(test.opcodes.end(), {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, + O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}); + test.required_spirv = {"OpFOrdLessThanEqual", "OpFOrdGreaterThanEqual"}; + return test; +} + TestCase VectorCompareF64Edges() { using O = ShaderOpcode; - constexpr std::array, 12> pairs{{ + constexpr std::array, 15> pairs{{ + {0xc000000000000000ull, 0xbff0000000000000ull}, + {0x8000000000000001ull, 0}, {0x3fc45f306dc9c882ull, 0x3fc45f306dc9c882ull}, {0, 0x8000000000000000ull}, {1, 0}, {1, 1}, {1, 2}, {0x11d04c7b00ull, 0}, {0x3ff0000000000000ull, 0x3ff0000000000001ull}, {0xbff0000000000000ull, 0xbff0000000000000ull}, @@ -22596,6 +22848,15 @@ TestCase VectorCompareF64Edges() { const auto rhs = std::bit_cast(pairs[i][1]); test.code.push_back(EncodeVopc(0x22, Vgpr(1), 3)); store_mask(106, lhs == rhs); + test.code.push_back(EncodeVopc(0x23, Vgpr(1), 3)); + store_mask(106, lhs <= rhs); + AppendVop3(&test.code, 0x23, 20, Vgpr(1), Vgpr(3), 0, 1, 0, false, 0, 1); + store_mask(20, -std::abs(lhs) <= rhs); + test.code.push_back(EncodeVopc(0x23, 255, 3)); + test.code.push_back(0x3ff00000u); // FP64 literals supply the high word. + store_mask(106, 1.0 <= rhs); + AppendVop3(&test.code, 0x23, 20, 248, Vgpr(3)); + store_mask(20, std::bit_cast(uint64_t{0x3fc45f306dc9c882ull}) <= rhs); AppendVop3(&test.code, 0x22, 20, Vgpr(1), Vgpr(3), 0, 1, 0, false, 0, 1); store_mask(20, -std::abs(lhs) == rhs); test.code.push_back(EncodeVop1(0x02, 100, Vgpr(3))); @@ -22603,12 +22864,15 @@ TestCase VectorCompareF64Edges() { // Captured e6d76d24f59f8015 PC 0xe4: compare zero with s100:s101. test.code.insert(test.code.end(), {0xd422006au, 0x0000c880u}); store_mask(106, 0.0 == rhs); + AppendVop3(&test.code, 0x23, 20, Vgpr(1), 100); + store_mask(20, lhs <= rhs); } AppendEnd(&test.code); test.initial.resize(test.expected.size()); test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, - O::V_CMP_EQ_F64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "OpFOrdEqual"}; + O::V_CMP_EQ_F64, O::V_CMP_LE_F64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "OpFOrdEqual", + "OpFOrdLessThanEqual"}; return test; } @@ -22808,6 +23072,18 @@ TestCase VectorF64ModesModifiersAndExec() { using O = ShaderOpcode; TestCase test; test.name = "VectorF64ModesModifiersAndExec"; + constexpr std::array, 8> add_pairs{{ + {1, 1}, {1, 0x8000000000000001ull}, + {0x8000000000000000ull, 0x8000000000000000ull}, + {0x3ff0000000000000ull, 0x3ca0000000000000ull}, + {0x3ff0000000000001ull, 0x3ca0000000000000ull}, + {0x3ff0000000000000ull, 0xbff0000000000000ull}, + {0x7ff0000000000000ull, 0x3ff0000000000000ull}, + {0xfff0000000000000ull, 0xbff0000000000000ull}}}; + for (const auto& pair : add_pairs) + for (const auto bits : pair) + test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)}); + test.expected = test.initial; const auto load_pair = [&](u32 reg, uint64_t bits) { AppendVMovLiteral(&test.code, reg, static_cast(bits)); AppendVMovLiteral(&test.code, reg + 1, static_cast(bits >> 32)); @@ -22834,6 +23110,36 @@ TestCase VectorF64ModesModifiersAndExec() { test.code.push_back(EncodeVop1(0x2f, 1, Vgpr(1))); store_pair(1, expected); } + for (u32 i = 0; i < add_pairs.size(); ++i) { + const auto pair = add_pairs[i]; + for (u32 word = 0; word < 4; ++word) { + AppendVMovU32(&test.code, 31, (i * 4 + word) * 4); + AppendBufferLoadDword(&test.code, 1 + word, 31); + } + AppendVop3(&test.code, 0x164, 1, Vgpr(1), Vgpr(3)); + store_pair(1, std::bit_cast(std::bit_cast(pair[0]) + + std::bit_cast(pair[1]))); + } + // Scalar pair, exact inline 1/(2*pi), literal high word, and ABS before NEG. + AppendSMovLiteral(&test.code, 8, 0); + AppendSMovLiteral(&test.code, 9, 0xbff00000u); + AppendVop3(&test.code, 0x164, 1, 8, 244, 0, 1, 0, false, 0, 1); + store_pair(1, std::bit_cast(1.0)); + AppendVop3(&test.code, 0x164, 1, 255, 240); + test.code.push_back(0x40080000u); + store_pair(1, std::bit_cast(3.5)); + AppendVop3(&test.code, 0x164, 1, 248, InlineU32(0)); + store_pair(1, 0x3fc45f306dc9c882ull); + for (const auto pair : std::array, 2>{{ + {0x7ff0000000000000ull, 0xfff0000000000000ull}, + {0x7ff0000000000001ull, 0}}}) { + load_pair(1, pair[0]); + load_pair(3, pair[1]); + AppendVop3(&test.code, 0x164, 1, Vgpr(1), Vgpr(3)); + AppendVMovLiteral(&test.code, 3, 0x7ff80000u); + test.code.push_back(EncodeVop2(0x1b, 2, Vgpr(3), 2)); + store_word(2, 0x7ff80000u); + } // A fused result retains the residual that separate multiply/add would round // to zero. load_pair(1, std::bit_cast(1.0 + 0x1p-27)); @@ -22862,6 +23168,7 @@ TestCase VectorF64ModesModifiersAndExec() { test.code.push_back(EncodeSop1(0x04, 12, 126)); test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); test.code.push_back(EncodeVop1(0x04, 1, InlineU32(9))); + AppendVop3(&test.code, 0x164, 1, 242, 242); test.code.push_back(EncodeSop1(0x04, 126, 12)); store_pair(1, std::bit_cast(0.5)); for (const auto [source, expected] : @@ -22885,8 +23192,10 @@ TestCase VectorF64ModesModifiersAndExec() { AppendEnd(&test.code); test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_CVT_F64_I32, O::V_RCP_F64, O::V_MUL_F64, - O::V_FMA_F64, O::V_CVT_F32_F64, O::V_AND_B32, - O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + O::V_ADD_F64, O::V_FMA_F64, O::V_CVT_F32_F64, O::V_AND_B32, + O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.initial.resize(test.expected.size()); + test.required_spirv = {"OpFAdd"}; return test; } @@ -23497,6 +23806,11 @@ TestCase VectorCompareF16Ops() { append_compare(0xcc, Vgpr(3), 2, true); append_compare(0xcd, Vgpr(2), 3, true); append_compare(0xce, Vgpr(3), 2, true); + append_compare(0xe9, Vgpr(2), 3, true); + append_compare(0xe9, Vgpr(3), 2, false); + append_compare(0xe9, Vgpr(2), 2, false); + append_compare(0xe9, Vgpr(4), 2, true); + append_compare(0xe9, Vgpr(2), 4, true); append_compare(0xed, Vgpr(4), 2, true); auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) { @@ -23521,7 +23835,7 @@ TestCase VectorCompareF16Ops() { expected, {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_LT_F16, O::V_CMP_EQ_F16, O::V_CMP_LE_F16, O::V_CMP_GT_F16, O::V_CMP_LG_F16, O::V_CMP_GE_F16, - O::V_CMP_NEQ_F16, O::V_CMPX_LT_F16, O::V_CMPX_EQ_F16, + O::V_CMP_NGE_F16, O::V_CMP_NEQ_F16, O::V_CMPX_LT_F16, O::V_CMPX_EQ_F16, O::V_CMPX_LE_F16, O::V_CMPX_GT_F16, O::V_CMPX_GE_F16, O::V_CMPX_NEQ_F16, O::V_CMPX_NLT_F16, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; @@ -27394,33 +27708,36 @@ TestCase DsAtomicReturnVariants() { O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } -TestCase DsOrB64Bounds() { +TestCase DsAtomic64Bounds(bool add) { using O = ShaderOpcode; TestCase test; - test.name = "DsOrB64Bounds"; + test.name = add ? "DsAddU64Bounds" : "DsOrB64Bounds"; + const u32 atomic_word = add ? 0xd9000000u : 0xd9280000u; auto &code = test.code; AppendVMovU32(&code, 0, 0x1234); AppendVMovU32(&code, 7, 0); - const u32 initial[] = {0x100, 0x200, 0x400, 0x800, 0x13579bdf}; + const std::array initial = add + ? std::array{0xffffffffu, 1, 0xffffffffu, 0xfffffffeu, 0x13579bdf} + : std::array{0x100, 0x200, 0x400, 0x800, 0x13579bdf}; for (u32 i = 0; i < std::size(initial); ++i) { AppendVMovLiteral(&code, 3, initial[i]); code.push_back(EncodeDs0(0x0d, i * 4u)); code.push_back(EncodeDs1(0, 3, 7)); } - AppendVMovU32(&code, 3, 1); - AppendVMovU32(&code, 4, 2); + AppendVMovU32(&code, 63, 1); + AppendVMovU32(&code, 64, 2); AppendVMovU32(&code, 7, 3); // The 64-bit address ignores the low three bits. - code.insert(code.end(), {0xd9280000u, 0x00000307u}); + code.insert(code.end(), {atomic_word, 0x00003f07u}); AppendVMovLiteral(&code, 7, 0x10007); - code.push_back(EncodeDs0(0x4a, 2)); // Offset before 16-bit wrap/alignment -> 8. - code.push_back(EncodeDs1(0, 3, 7)); - AppendVMovLiteral(&code, 3, 0xffffffffu); - AppendVMovLiteral(&code, 4, 0xffffffffu); - AppendVMovU32(&code, 7, 16); // Only one DWORD remains: ignore the entire OR. - code.insert(code.end(), {0xd9280000u, 0x00000307u}); + code.push_back(EncodeDs0(add ? 0x40 : 0x4a, 2)); // Offset before wrap/alignment -> 8. + code.push_back(EncodeDs1(0, 63, 7)); + AppendVMovLiteral(&code, 63, 0xffffffffu); + AppendVMovLiteral(&code, 64, 0xffffffffu); + AppendVMovU32(&code, 7, 16); // Only one DWORD remains: ignore the entire atomic. + code.insert(code.end(), {atomic_word, 0x00003f07u}); AppendVMovU32(&code, 7, 0); code.push_back(EncodeSMovB32(126, InlineU32(0))); - code.insert(code.end(), {0xd9280000u, 0x00000307u}); + code.insert(code.end(), {atomic_word, 0x00003f07u}); code.push_back(EncodeSMovB32(126, InlineU32(1))); for (u32 i = 0; i < std::size(initial); ++i) { code.push_back(EncodeDs0(0x36, i * 4u)); @@ -27429,18 +27746,22 @@ TestCase DsOrB64Bounds() { } AppendStoreVgpr(&code, 0, 5); // No-return DS must not overwrite the encoded VDst. AppendEnd(&code); - test.expected = {0x101, 0x202, 0x401, 0x802, 0x13579bdf, 0x1234}; + test.expected = add ? std::vector{0, 4, 0, 1, 0x13579bdf, 0x1234} + : std::vector{0x101, 0x202, 0x401, 0x802, 0x13579bdf, 0x1234}; test.compute_info.lds_size_dwords = 5; - test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, O::DS_OR_B64, + test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, + add ? O::DS_ADD_U64 : O::DS_OR_B64, O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.required_spirv = {"OpAtomicOr %ulong", "WorkgroupMemoryExplicitLayoutKHR", "Aliased"}; + test.required_spirv = {add ? "OpAtomicIAdd %ulong" : "OpAtomicOr %ulong", + "WorkgroupMemoryExplicitLayoutKHR", "Aliased"}; return test; } -TestCase DsOrB64Contention(u32 wave_size) { +TestCase DsAtomic64Contention(bool add, u32 wave_size) { using O = ShaderOpcode; TestCase test; - test.name = wave_size == 32 ? "DsOrB64ContentionWave32" : "DsOrB64ContentionWave64"; + test.name = add ? (wave_size == 32 ? "DsAddU64ContentionWave32" : "DsAddU64ContentionWave64") + : (wave_size == 32 ? "DsOrB64ContentionWave32" : "DsOrB64ContentionWave64"); auto &code = test.code; AppendVMovU32(&code, 7, 0); code.push_back(EncodeSop1(0x04, 10, 126)); @@ -27452,13 +27773,18 @@ TestCase DsOrB64Contention(u32 wave_size) { } code.push_back(EncodeSop1(0x04, 126, 10)); code.push_back(EncodeSopp(0x0a, 0)); - AppendVMovU32(&code, 5, 1); - AppendVMovU32(&code, 6, 2); - code.push_back(EncodeVop2(0x1a, 3, Vgpr(0), 5)); - code.push_back(EncodeVop2(0x1a, 4, Vgpr(0), 6)); + if (add) { + AppendVMovLiteral(&code, 3, 0xffffffffu); + AppendVMovU32(&code, 4, 1); + } else { + AppendVMovU32(&code, 5, 1); + AppendVMovU32(&code, 6, 2); + code.push_back(EncodeVop2(0x1a, 3, Vgpr(0), 5)); + code.push_back(EncodeVop2(0x1a, 4, Vgpr(0), 6)); + } AppendSMovLiteral(&code, 126, 0x55555555u); AppendSMovLiteral(&code, 127, 0xaaaaaaaau); - code.insert(code.end(), {0xd9280000u, 0x00000307u}); + code.insert(code.end(), {add ? 0xd9000000u : 0xd9280000u, 0x00000307u}); code.push_back(EncodeSop1(0x04, 126, 10)); code.push_back(EncodeSopp(0x0a, 0)); for (u32 half = 0; half < 2; ++half) { @@ -27467,8 +27793,8 @@ TestCase DsOrB64Contention(u32 wave_size) { AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, half * 128); } AppendEnd(&code); - test.expected.assign(128, wave_size == 32 ? 0x55555555u : 0xffffffffu); - test.expected.resize(256, wave_size == 32 ? 0xaaaaaaaau : 0xfffffffeu); + test.expected.assign(128, add ? 0xffffffc0u : wave_size == 32 ? 0x55555555u : 0xffffffffu); + test.expected.resize(256, add ? 0x7fu : wave_size == 32 ? 0xaaaaaaaau : 0xfffffffeu); test.compute_info.threads_num[0] = 128; test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; test.compute_info.thread_ids_num = 1; @@ -27476,10 +27802,14 @@ TestCase DsOrB64Contention(u32 wave_size) { test.compute_info.lds_size_dwords = 2; test.has_compute_info = true; test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::S_BARRIER, - O::V_CMP_EQ_U32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, - O::DS_WRITE_B32, O::DS_OR_B64, O::DS_READ_B32, + O::V_CMP_EQ_U32, O::V_ADD_NC_U32, + O::DS_WRITE_B32, add ? O::DS_ADD_U64 : O::DS_OR_B64, O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.required_spirv = {"OpAtomicOr %ulong", "WorkgroupMemoryExplicitLayoutKHR", "Aliased"}; + if (!add) { + test.opcodes.push_back(O::V_LSHLREV_B32); + } + test.required_spirv = {add ? "OpAtomicIAdd %ulong" : "OpAtomicOr %ulong", + "WorkgroupMemoryExplicitLayoutKHR", "Aliased"}; return test; } @@ -28182,6 +28512,37 @@ TestCase BufferAtomicVariants() { O::S_ENDPGM}}; } +TestCase BufferAtomicAndX2GlcAndExec() { + using O = ShaderOpcode; + TestCase test; + test.name = "BufferAtomicAndX2GlcAndExec"; + auto &code = test.code; + code.push_back(EncodeSop1(0x04, 12, 126)); + AppendVMovU32(&code, 20, 0); + const u32 values[][2] = {{0x0ff00ff0u, 0xf00ff00fu}, + {0xff00ff00u, 0x00ff00ffu}, + {0, 0}}; + for (u32 i = 0; i < std::size(values); ++i) { + AppendVMovLiteral(&code, 2, values[i][0]); + AppendVMovLiteral(&code, 3, values[i][1]); + if (i == 2) { + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + } + AppendBufferStoreOpcode(&code, 0x59, 2, 20, i != 1); + code.push_back(EncodeSop1(0x04, 126, 12)); + AppendStoreVgpr(&code, 2, 2 + i * 2); + AppendStoreVgpr(&code, 3, 3 + i * 2); + } + AppendEnd(&code); + test.initial = {0x12345678u, 0x87654321u, 0, 0, 0, 0, 0, 0}; + test.expected = {0x02000600u, 0x00050001u, 0x12345678u, 0x87654321u, + 0xff00ff00u, 0x00ff00ffu, 0, 0}; + test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_ATOMIC_AND_X2, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpAtomicAnd %ulong", "Int64Atomics"}; + return test; +} + TestCase BufferAtomicCmpSwapExactRaw() { using O = ShaderOpcode; @@ -30133,6 +30494,53 @@ TestCase ImageAtomicVariants() { return test; } +template TestCase ImageAtomicSignedMinMax() { + using O = ShaderOpcode; + // Input, old value, signed minimum, signed maximum. + constexpr u32 values[][4] = { + {0xfffffffbu, 7, 0xfffffffbu, 7}, + {7, 0xfffffffbu, 0xfffffffbu, 7}, + {0x80000000u, 0x7fffffffu, 0x80000000u, 0x7fffffffu}, + {0x7fffffffu, 0x80000000u, 0x80000000u, 0x7fffffffu}, + {0xfffffff6u, 0xfffffffbu, 0xfffffff6u, 0xfffffffbu}, + {0xfffffffbu, 0xfffffff6u, 0xfffffff6u, 0xfffffffbu}, + {2, 8, 2, 8}, {8, 2, 2, 8}, + }; + TestCase test; + test.name = max_value ? "ImageAtomicSMaxGlcAndExec" : "ImageAtomicSMinGlcAndExec"; + test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt); + test.has_user_data = true; + test.storage_image_r32ui.assign(16, 0); + test.expected_storage_image_r32ui = test.storage_image_r32ui; + auto &code = test.code; + code.push_back(EncodeSop1(0x04, 12, 126)); + for (u32 i = 0; i < std::size(values); ++i) { + AppendVMovU32(&code, 20, i & 3u); + AppendVMovU32(&code, 21, i >> 2u); + AppendVMovLiteral(&code, 2, values[i][0]); + const bool glc = (i & 1u) == 0; + code.push_back(EncodeMimg0(max_value ? 0x16 : 0x14, 1, 0, glc)); + code.push_back(EncodeMimg1(2, 20)); + AppendStoreVgpr(&code, 2, i); + test.expected.push_back(values[i][glc ? 1 : 0]); + test.storage_image_r32ui[i] = values[i][1]; + test.expected_storage_image_r32ui[i] = values[i][max_value ? 3 : 2]; + } + AppendVMovLiteral(&code, 2, max_value ? 0x7fffffffu : 0x80000000u); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeMimg0(max_value ? 0x16 : 0x14, 1, 0, true)); + code.push_back(EncodeMimg1(2, 20)); + code.push_back(EncodeSop1(0x04, 126, 12)); + AppendStoreVgpr(&code, 2, std::size(values)); + test.expected.push_back(max_value ? 0x7fffffffu : 0x80000000u); + AppendEnd(&code); + test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, + max_value ? O::IMAGE_ATOMIC_SMAX : O::IMAGE_ATOMIC_SMIN, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {max_value ? "OpAtomicSMax" : "OpAtomicSMin", "R32ui"}; + return test; +} + template TestCase ImageAtomicFloatGlcAndExec() { using O = ShaderOpcode; std::vector code; @@ -30760,6 +31168,8 @@ std::vector MakeCases() { AddCase(ScalarBitcmpB64DynamicOperands); AddCase(ScalarBitcmpB64IntegerConstants); AddCase(ScalarBrevB32PreservesScc); + cases.push_back(ScalarBrevB64OperandsAndMasks(32)); + cases.push_back(ScalarBrevB64OperandsAndMasks(64)); AddCase(ScalarBfeI32CapturedRawSignExtends); AddCase(BitfieldExtractWidthPastEndEdges); AddCase(Scalar64BitOps); @@ -30871,6 +31281,9 @@ std::vector MakeCases() { AddCase(VectorSpecialF32FlushesDenormalInputs); AddCase(VectorRcpIflagF32IntegerReciprocal); AddCase(VectorCompareF64Edges); + AddCase(VectorCompareInteger64Edges); + cases.push_back(VectorCompareExecWaveMasks(32)); + cases.push_back(VectorCompareExecWaveMasks(64)); cases.push_back(VectorCompareF64WaveMasks(32)); cases.push_back(VectorCompareF64WaveMasks(64)); AddCase(VectorF64CapturedScreenSpaceShadows); @@ -31034,8 +31447,15 @@ std::vector MakeCases() { AddCase(DsWideGdsPartialBounds); AddCase(DsAtomicNoReturnVariants); AddCase(DsAtomicReturnVariants); - AddCase(DsOrB64Bounds); - for (u32 wave_size : {32, 64}) cases.push_back(DsOrB64Contention(wave_size)); + for (bool add : {false, true}) { + cases.push_back(DsAtomic64Bounds(add)); + for (u32 wave_size : {32, 64}) { + cases.push_back(DsAtomic64Contention(add, wave_size)); + } + } + AddCase(BufferAtomicAndX2GlcAndExec); + AddCase(ImageAtomicSignedMinMax); + AddCase(ImageAtomicSignedMinMax); for (bool decrement : {false, true}) { for (bool gds : {false, true}) { cases.push_back(DsBoundedAtomicBoundaries(decrement, gds)); @@ -35766,6 +36186,27 @@ int main(int argc, char **argv) { std::setvbuf(stdout, nullptr, _IONBF, 0); EnsureConfigInitialized(); CheckLeastRecentlyUsedCacheOrdering(); + if (argc == 2 && std::strcmp(argv[1], "--new-opcodes-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, VectorCompareInteger64Edges()); + RunCase(&vulkan, VectorCompareExecWaveMasks(32)); + RunCase(&vulkan, VectorCompareExecWaveMasks(64)); + RunCase(&vulkan, VectorCompareF64Edges()); + RunCase(&vulkan, VectorF64ModesModifiersAndExec()); + RunCase(&vulkan, VectorCompareF16Ops()); + RunCase(&vulkan, ScalarBrevB64OperandsAndMasks(32)); + RunCase(&vulkan, ScalarBrevB64OperandsAndMasks(64)); + for (bool add : {false, true}) { + RunCase(&vulkan, DsAtomic64Bounds(add)); + for (u32 wave_size : {32, 64}) { + RunCase(&vulkan, DsAtomic64Contention(add, wave_size)); + } + } + RunCase(&vulkan, BufferAtomicAndX2GlcAndExec()); + RunCase(&vulkan, ImageAtomicSignedMinMax()); + RunCase(&vulkan, ImageAtomicSignedMinMax()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--s-memrealtime-only") == 0) { VulkanHarness vulkan; RunCase(&vulkan, ScalarMemRealtimeCapturedPlaceholder());