106f32e7eSjoerg //===-- AArch64TargetTransformInfo.cpp - AArch64 specific TTI -------------===//
206f32e7eSjoerg //
306f32e7eSjoerg // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
406f32e7eSjoerg // See https://llvm.org/LICENSE.txt for license information.
506f32e7eSjoerg // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
606f32e7eSjoerg //
706f32e7eSjoerg //===----------------------------------------------------------------------===//
806f32e7eSjoerg
906f32e7eSjoerg #include "AArch64TargetTransformInfo.h"
10*da58b97aSjoerg #include "AArch64ExpandImm.h"
1106f32e7eSjoerg #include "MCTargetDesc/AArch64AddressingModes.h"
1206f32e7eSjoerg #include "llvm/Analysis/LoopInfo.h"
1306f32e7eSjoerg #include "llvm/Analysis/TargetTransformInfo.h"
1406f32e7eSjoerg #include "llvm/CodeGen/BasicTTIImpl.h"
1506f32e7eSjoerg #include "llvm/CodeGen/CostTable.h"
1606f32e7eSjoerg #include "llvm/CodeGen/TargetLowering.h"
1706f32e7eSjoerg #include "llvm/IR/IntrinsicInst.h"
18*da58b97aSjoerg #include "llvm/IR/IntrinsicsAArch64.h"
19*da58b97aSjoerg #include "llvm/IR/PatternMatch.h"
2006f32e7eSjoerg #include "llvm/Support/Debug.h"
21*da58b97aSjoerg #include "llvm/Transforms/InstCombine/InstCombiner.h"
2206f32e7eSjoerg #include <algorithm>
2306f32e7eSjoerg using namespace llvm;
24*da58b97aSjoerg using namespace llvm::PatternMatch;
2506f32e7eSjoerg
2606f32e7eSjoerg #define DEBUG_TYPE "aarch64tti"
2706f32e7eSjoerg
2806f32e7eSjoerg static cl::opt<bool> EnableFalkorHWPFUnrollFix("enable-falkor-hwpf-unroll-fix",
2906f32e7eSjoerg cl::init(true), cl::Hidden);
3006f32e7eSjoerg
areInlineCompatible(const Function * Caller,const Function * Callee) const3106f32e7eSjoerg bool AArch64TTIImpl::areInlineCompatible(const Function *Caller,
3206f32e7eSjoerg const Function *Callee) const {
3306f32e7eSjoerg const TargetMachine &TM = getTLI()->getTargetMachine();
3406f32e7eSjoerg
3506f32e7eSjoerg const FeatureBitset &CallerBits =
3606f32e7eSjoerg TM.getSubtargetImpl(*Caller)->getFeatureBits();
3706f32e7eSjoerg const FeatureBitset &CalleeBits =
3806f32e7eSjoerg TM.getSubtargetImpl(*Callee)->getFeatureBits();
3906f32e7eSjoerg
4006f32e7eSjoerg // Inline a callee if its target-features are a subset of the callers
4106f32e7eSjoerg // target-features.
4206f32e7eSjoerg return (CallerBits & CalleeBits) == CalleeBits;
4306f32e7eSjoerg }
4406f32e7eSjoerg
4506f32e7eSjoerg /// Calculate the cost of materializing a 64-bit value. This helper
4606f32e7eSjoerg /// method might only calculate a fraction of a larger immediate. Therefore it
4706f32e7eSjoerg /// is valid to return a cost of ZERO.
getIntImmCost(int64_t Val)48*da58b97aSjoerg InstructionCost AArch64TTIImpl::getIntImmCost(int64_t Val) {
4906f32e7eSjoerg // Check if the immediate can be encoded within an instruction.
5006f32e7eSjoerg if (Val == 0 || AArch64_AM::isLogicalImmediate(Val, 64))
5106f32e7eSjoerg return 0;
5206f32e7eSjoerg
5306f32e7eSjoerg if (Val < 0)
5406f32e7eSjoerg Val = ~Val;
5506f32e7eSjoerg
5606f32e7eSjoerg // Calculate how many moves we will need to materialize this constant.
5706f32e7eSjoerg SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
5806f32e7eSjoerg AArch64_IMM::expandMOVImm(Val, 64, Insn);
5906f32e7eSjoerg return Insn.size();
6006f32e7eSjoerg }
6106f32e7eSjoerg
6206f32e7eSjoerg /// Calculate the cost of materializing the given constant.
getIntImmCost(const APInt & Imm,Type * Ty,TTI::TargetCostKind CostKind)63*da58b97aSjoerg InstructionCost AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
64*da58b97aSjoerg TTI::TargetCostKind CostKind) {
6506f32e7eSjoerg assert(Ty->isIntegerTy());
6606f32e7eSjoerg
6706f32e7eSjoerg unsigned BitSize = Ty->getPrimitiveSizeInBits();
6806f32e7eSjoerg if (BitSize == 0)
6906f32e7eSjoerg return ~0U;
7006f32e7eSjoerg
7106f32e7eSjoerg // Sign-extend all constants to a multiple of 64-bit.
7206f32e7eSjoerg APInt ImmVal = Imm;
7306f32e7eSjoerg if (BitSize & 0x3f)
7406f32e7eSjoerg ImmVal = Imm.sext((BitSize + 63) & ~0x3fU);
7506f32e7eSjoerg
7606f32e7eSjoerg // Split the constant into 64-bit chunks and calculate the cost for each
7706f32e7eSjoerg // chunk.
78*da58b97aSjoerg InstructionCost Cost = 0;
7906f32e7eSjoerg for (unsigned ShiftVal = 0; ShiftVal < BitSize; ShiftVal += 64) {
8006f32e7eSjoerg APInt Tmp = ImmVal.ashr(ShiftVal).sextOrTrunc(64);
8106f32e7eSjoerg int64_t Val = Tmp.getSExtValue();
8206f32e7eSjoerg Cost += getIntImmCost(Val);
8306f32e7eSjoerg }
8406f32e7eSjoerg // We need at least one instruction to materialze the constant.
85*da58b97aSjoerg return std::max<InstructionCost>(1, Cost);
8606f32e7eSjoerg }
8706f32e7eSjoerg
getIntImmCostInst(unsigned Opcode,unsigned Idx,const APInt & Imm,Type * Ty,TTI::TargetCostKind CostKind,Instruction * Inst)88*da58b97aSjoerg InstructionCost AArch64TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
89*da58b97aSjoerg const APInt &Imm, Type *Ty,
90*da58b97aSjoerg TTI::TargetCostKind CostKind,
91*da58b97aSjoerg Instruction *Inst) {
9206f32e7eSjoerg assert(Ty->isIntegerTy());
9306f32e7eSjoerg
9406f32e7eSjoerg unsigned BitSize = Ty->getPrimitiveSizeInBits();
9506f32e7eSjoerg // There is no cost model for constants with a bit size of 0. Return TCC_Free
9606f32e7eSjoerg // here, so that constant hoisting will ignore this constant.
9706f32e7eSjoerg if (BitSize == 0)
9806f32e7eSjoerg return TTI::TCC_Free;
9906f32e7eSjoerg
10006f32e7eSjoerg unsigned ImmIdx = ~0U;
10106f32e7eSjoerg switch (Opcode) {
10206f32e7eSjoerg default:
10306f32e7eSjoerg return TTI::TCC_Free;
10406f32e7eSjoerg case Instruction::GetElementPtr:
10506f32e7eSjoerg // Always hoist the base address of a GetElementPtr.
10606f32e7eSjoerg if (Idx == 0)
10706f32e7eSjoerg return 2 * TTI::TCC_Basic;
10806f32e7eSjoerg return TTI::TCC_Free;
10906f32e7eSjoerg case Instruction::Store:
11006f32e7eSjoerg ImmIdx = 0;
11106f32e7eSjoerg break;
11206f32e7eSjoerg case Instruction::Add:
11306f32e7eSjoerg case Instruction::Sub:
11406f32e7eSjoerg case Instruction::Mul:
11506f32e7eSjoerg case Instruction::UDiv:
11606f32e7eSjoerg case Instruction::SDiv:
11706f32e7eSjoerg case Instruction::URem:
11806f32e7eSjoerg case Instruction::SRem:
11906f32e7eSjoerg case Instruction::And:
12006f32e7eSjoerg case Instruction::Or:
12106f32e7eSjoerg case Instruction::Xor:
12206f32e7eSjoerg case Instruction::ICmp:
12306f32e7eSjoerg ImmIdx = 1;
12406f32e7eSjoerg break;
12506f32e7eSjoerg // Always return TCC_Free for the shift value of a shift instruction.
12606f32e7eSjoerg case Instruction::Shl:
12706f32e7eSjoerg case Instruction::LShr:
12806f32e7eSjoerg case Instruction::AShr:
12906f32e7eSjoerg if (Idx == 1)
13006f32e7eSjoerg return TTI::TCC_Free;
13106f32e7eSjoerg break;
13206f32e7eSjoerg case Instruction::Trunc:
13306f32e7eSjoerg case Instruction::ZExt:
13406f32e7eSjoerg case Instruction::SExt:
13506f32e7eSjoerg case Instruction::IntToPtr:
13606f32e7eSjoerg case Instruction::PtrToInt:
13706f32e7eSjoerg case Instruction::BitCast:
13806f32e7eSjoerg case Instruction::PHI:
13906f32e7eSjoerg case Instruction::Call:
14006f32e7eSjoerg case Instruction::Select:
14106f32e7eSjoerg case Instruction::Ret:
14206f32e7eSjoerg case Instruction::Load:
14306f32e7eSjoerg break;
14406f32e7eSjoerg }
14506f32e7eSjoerg
14606f32e7eSjoerg if (Idx == ImmIdx) {
14706f32e7eSjoerg int NumConstants = (BitSize + 63) / 64;
148*da58b97aSjoerg InstructionCost Cost = AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
14906f32e7eSjoerg return (Cost <= NumConstants * TTI::TCC_Basic)
15006f32e7eSjoerg ? static_cast<int>(TTI::TCC_Free)
15106f32e7eSjoerg : Cost;
15206f32e7eSjoerg }
153*da58b97aSjoerg return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
15406f32e7eSjoerg }
15506f32e7eSjoerg
156*da58b97aSjoerg InstructionCost
getIntImmCostIntrin(Intrinsic::ID IID,unsigned Idx,const APInt & Imm,Type * Ty,TTI::TargetCostKind CostKind)157*da58b97aSjoerg AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
158*da58b97aSjoerg const APInt &Imm, Type *Ty,
159*da58b97aSjoerg TTI::TargetCostKind CostKind) {
16006f32e7eSjoerg assert(Ty->isIntegerTy());
16106f32e7eSjoerg
16206f32e7eSjoerg unsigned BitSize = Ty->getPrimitiveSizeInBits();
16306f32e7eSjoerg // There is no cost model for constants with a bit size of 0. Return TCC_Free
16406f32e7eSjoerg // here, so that constant hoisting will ignore this constant.
16506f32e7eSjoerg if (BitSize == 0)
16606f32e7eSjoerg return TTI::TCC_Free;
16706f32e7eSjoerg
168*da58b97aSjoerg // Most (all?) AArch64 intrinsics do not support folding immediates into the
169*da58b97aSjoerg // selected instruction, so we compute the materialization cost for the
170*da58b97aSjoerg // immediate directly.
171*da58b97aSjoerg if (IID >= Intrinsic::aarch64_addg && IID <= Intrinsic::aarch64_udiv)
172*da58b97aSjoerg return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
173*da58b97aSjoerg
17406f32e7eSjoerg switch (IID) {
17506f32e7eSjoerg default:
17606f32e7eSjoerg return TTI::TCC_Free;
17706f32e7eSjoerg case Intrinsic::sadd_with_overflow:
17806f32e7eSjoerg case Intrinsic::uadd_with_overflow:
17906f32e7eSjoerg case Intrinsic::ssub_with_overflow:
18006f32e7eSjoerg case Intrinsic::usub_with_overflow:
18106f32e7eSjoerg case Intrinsic::smul_with_overflow:
18206f32e7eSjoerg case Intrinsic::umul_with_overflow:
18306f32e7eSjoerg if (Idx == 1) {
18406f32e7eSjoerg int NumConstants = (BitSize + 63) / 64;
185*da58b97aSjoerg InstructionCost Cost = AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
18606f32e7eSjoerg return (Cost <= NumConstants * TTI::TCC_Basic)
18706f32e7eSjoerg ? static_cast<int>(TTI::TCC_Free)
18806f32e7eSjoerg : Cost;
18906f32e7eSjoerg }
19006f32e7eSjoerg break;
19106f32e7eSjoerg case Intrinsic::experimental_stackmap:
19206f32e7eSjoerg if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
19306f32e7eSjoerg return TTI::TCC_Free;
19406f32e7eSjoerg break;
19506f32e7eSjoerg case Intrinsic::experimental_patchpoint_void:
19606f32e7eSjoerg case Intrinsic::experimental_patchpoint_i64:
19706f32e7eSjoerg if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
19806f32e7eSjoerg return TTI::TCC_Free;
19906f32e7eSjoerg break;
200*da58b97aSjoerg case Intrinsic::experimental_gc_statepoint:
201*da58b97aSjoerg if ((Idx < 5) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
202*da58b97aSjoerg return TTI::TCC_Free;
203*da58b97aSjoerg break;
20406f32e7eSjoerg }
205*da58b97aSjoerg return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
20606f32e7eSjoerg }
20706f32e7eSjoerg
20806f32e7eSjoerg TargetTransformInfo::PopcntSupportKind
getPopcntSupport(unsigned TyWidth)20906f32e7eSjoerg AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) {
21006f32e7eSjoerg assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
21106f32e7eSjoerg if (TyWidth == 32 || TyWidth == 64)
21206f32e7eSjoerg return TTI::PSK_FastHardware;
21306f32e7eSjoerg // TODO: AArch64TargetLowering::LowerCTPOP() supports 128bit popcount.
21406f32e7eSjoerg return TTI::PSK_Software;
21506f32e7eSjoerg }
21606f32e7eSjoerg
217*da58b97aSjoerg InstructionCost
getIntrinsicInstrCost(const IntrinsicCostAttributes & ICA,TTI::TargetCostKind CostKind)218*da58b97aSjoerg AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
219*da58b97aSjoerg TTI::TargetCostKind CostKind) {
220*da58b97aSjoerg auto *RetTy = ICA.getReturnType();
221*da58b97aSjoerg switch (ICA.getID()) {
222*da58b97aSjoerg case Intrinsic::umin:
223*da58b97aSjoerg case Intrinsic::umax: {
224*da58b97aSjoerg auto LT = TLI->getTypeLegalizationCost(DL, RetTy);
225*da58b97aSjoerg // umin(x,y) -> sub(x,usubsat(x,y))
226*da58b97aSjoerg // umax(x,y) -> add(x,usubsat(y,x))
227*da58b97aSjoerg if (LT.second == MVT::v2i64)
228*da58b97aSjoerg return LT.first * 2;
229*da58b97aSjoerg LLVM_FALLTHROUGH;
230*da58b97aSjoerg }
231*da58b97aSjoerg case Intrinsic::smin:
232*da58b97aSjoerg case Intrinsic::smax: {
233*da58b97aSjoerg static const auto ValidMinMaxTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
234*da58b97aSjoerg MVT::v8i16, MVT::v2i32, MVT::v4i32};
235*da58b97aSjoerg auto LT = TLI->getTypeLegalizationCost(DL, RetTy);
236*da58b97aSjoerg if (any_of(ValidMinMaxTys, [<](MVT M) { return M == LT.second; }))
237*da58b97aSjoerg return LT.first;
238*da58b97aSjoerg break;
239*da58b97aSjoerg }
240*da58b97aSjoerg case Intrinsic::sadd_sat:
241*da58b97aSjoerg case Intrinsic::ssub_sat:
242*da58b97aSjoerg case Intrinsic::uadd_sat:
243*da58b97aSjoerg case Intrinsic::usub_sat: {
244*da58b97aSjoerg static const auto ValidSatTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
245*da58b97aSjoerg MVT::v8i16, MVT::v2i32, MVT::v4i32,
246*da58b97aSjoerg MVT::v2i64};
247*da58b97aSjoerg auto LT = TLI->getTypeLegalizationCost(DL, RetTy);
248*da58b97aSjoerg // This is a base cost of 1 for the vadd, plus 3 extract shifts if we
249*da58b97aSjoerg // need to extend the type, as it uses shr(qadd(shl, shl)).
250*da58b97aSjoerg unsigned Instrs =
251*da58b97aSjoerg LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits() ? 1 : 4;
252*da58b97aSjoerg if (any_of(ValidSatTys, [<](MVT M) { return M == LT.second; }))
253*da58b97aSjoerg return LT.first * Instrs;
254*da58b97aSjoerg break;
255*da58b97aSjoerg }
256*da58b97aSjoerg case Intrinsic::abs: {
257*da58b97aSjoerg static const auto ValidAbsTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
258*da58b97aSjoerg MVT::v8i16, MVT::v2i32, MVT::v4i32,
259*da58b97aSjoerg MVT::v2i64};
260*da58b97aSjoerg auto LT = TLI->getTypeLegalizationCost(DL, RetTy);
261*da58b97aSjoerg if (any_of(ValidAbsTys, [<](MVT M) { return M == LT.second; }))
262*da58b97aSjoerg return LT.first;
263*da58b97aSjoerg break;
264*da58b97aSjoerg }
265*da58b97aSjoerg case Intrinsic::experimental_stepvector: {
266*da58b97aSjoerg InstructionCost Cost = 1; // Cost of the `index' instruction
267*da58b97aSjoerg auto LT = TLI->getTypeLegalizationCost(DL, RetTy);
268*da58b97aSjoerg // Legalisation of illegal vectors involves an `index' instruction plus
269*da58b97aSjoerg // (LT.first - 1) vector adds.
270*da58b97aSjoerg if (LT.first > 1) {
271*da58b97aSjoerg Type *LegalVTy = EVT(LT.second).getTypeForEVT(RetTy->getContext());
272*da58b97aSjoerg InstructionCost AddCost =
273*da58b97aSjoerg getArithmeticInstrCost(Instruction::Add, LegalVTy, CostKind);
274*da58b97aSjoerg Cost += AddCost * (LT.first - 1);
275*da58b97aSjoerg }
276*da58b97aSjoerg return Cost;
277*da58b97aSjoerg }
278*da58b97aSjoerg default:
279*da58b97aSjoerg break;
280*da58b97aSjoerg }
281*da58b97aSjoerg return BaseT::getIntrinsicInstrCost(ICA, CostKind);
282*da58b97aSjoerg }
283*da58b97aSjoerg
284*da58b97aSjoerg /// The function will remove redundant reinterprets casting in the presence
285*da58b97aSjoerg /// of the control flow
processPhiNode(InstCombiner & IC,IntrinsicInst & II)286*da58b97aSjoerg static Optional<Instruction *> processPhiNode(InstCombiner &IC,
287*da58b97aSjoerg IntrinsicInst &II) {
288*da58b97aSjoerg SmallVector<Instruction *, 32> Worklist;
289*da58b97aSjoerg auto RequiredType = II.getType();
290*da58b97aSjoerg
291*da58b97aSjoerg auto *PN = dyn_cast<PHINode>(II.getArgOperand(0));
292*da58b97aSjoerg assert(PN && "Expected Phi Node!");
293*da58b97aSjoerg
294*da58b97aSjoerg // Don't create a new Phi unless we can remove the old one.
295*da58b97aSjoerg if (!PN->hasOneUse())
296*da58b97aSjoerg return None;
297*da58b97aSjoerg
298*da58b97aSjoerg for (Value *IncValPhi : PN->incoming_values()) {
299*da58b97aSjoerg auto *Reinterpret = dyn_cast<IntrinsicInst>(IncValPhi);
300*da58b97aSjoerg if (!Reinterpret ||
301*da58b97aSjoerg Reinterpret->getIntrinsicID() !=
302*da58b97aSjoerg Intrinsic::aarch64_sve_convert_to_svbool ||
303*da58b97aSjoerg RequiredType != Reinterpret->getArgOperand(0)->getType())
304*da58b97aSjoerg return None;
305*da58b97aSjoerg }
306*da58b97aSjoerg
307*da58b97aSjoerg // Create the new Phi
308*da58b97aSjoerg LLVMContext &Ctx = PN->getContext();
309*da58b97aSjoerg IRBuilder<> Builder(Ctx);
310*da58b97aSjoerg Builder.SetInsertPoint(PN);
311*da58b97aSjoerg PHINode *NPN = Builder.CreatePHI(RequiredType, PN->getNumIncomingValues());
312*da58b97aSjoerg Worklist.push_back(PN);
313*da58b97aSjoerg
314*da58b97aSjoerg for (unsigned I = 0; I < PN->getNumIncomingValues(); I++) {
315*da58b97aSjoerg auto *Reinterpret = cast<Instruction>(PN->getIncomingValue(I));
316*da58b97aSjoerg NPN->addIncoming(Reinterpret->getOperand(0), PN->getIncomingBlock(I));
317*da58b97aSjoerg Worklist.push_back(Reinterpret);
318*da58b97aSjoerg }
319*da58b97aSjoerg
320*da58b97aSjoerg // Cleanup Phi Node and reinterprets
321*da58b97aSjoerg return IC.replaceInstUsesWith(II, NPN);
322*da58b97aSjoerg }
323*da58b97aSjoerg
instCombineConvertFromSVBool(InstCombiner & IC,IntrinsicInst & II)324*da58b97aSjoerg static Optional<Instruction *> instCombineConvertFromSVBool(InstCombiner &IC,
325*da58b97aSjoerg IntrinsicInst &II) {
326*da58b97aSjoerg // If the reinterpret instruction operand is a PHI Node
327*da58b97aSjoerg if (isa<PHINode>(II.getArgOperand(0)))
328*da58b97aSjoerg return processPhiNode(IC, II);
329*da58b97aSjoerg
330*da58b97aSjoerg SmallVector<Instruction *, 32> CandidatesForRemoval;
331*da58b97aSjoerg Value *Cursor = II.getOperand(0), *EarliestReplacement = nullptr;
332*da58b97aSjoerg
333*da58b97aSjoerg const auto *IVTy = cast<VectorType>(II.getType());
334*da58b97aSjoerg
335*da58b97aSjoerg // Walk the chain of conversions.
336*da58b97aSjoerg while (Cursor) {
337*da58b97aSjoerg // If the type of the cursor has fewer lanes than the final result, zeroing
338*da58b97aSjoerg // must take place, which breaks the equivalence chain.
339*da58b97aSjoerg const auto *CursorVTy = cast<VectorType>(Cursor->getType());
340*da58b97aSjoerg if (CursorVTy->getElementCount().getKnownMinValue() <
341*da58b97aSjoerg IVTy->getElementCount().getKnownMinValue())
342*da58b97aSjoerg break;
343*da58b97aSjoerg
344*da58b97aSjoerg // If the cursor has the same type as I, it is a viable replacement.
345*da58b97aSjoerg if (Cursor->getType() == IVTy)
346*da58b97aSjoerg EarliestReplacement = Cursor;
347*da58b97aSjoerg
348*da58b97aSjoerg auto *IntrinsicCursor = dyn_cast<IntrinsicInst>(Cursor);
349*da58b97aSjoerg
350*da58b97aSjoerg // If this is not an SVE conversion intrinsic, this is the end of the chain.
351*da58b97aSjoerg if (!IntrinsicCursor || !(IntrinsicCursor->getIntrinsicID() ==
352*da58b97aSjoerg Intrinsic::aarch64_sve_convert_to_svbool ||
353*da58b97aSjoerg IntrinsicCursor->getIntrinsicID() ==
354*da58b97aSjoerg Intrinsic::aarch64_sve_convert_from_svbool))
355*da58b97aSjoerg break;
356*da58b97aSjoerg
357*da58b97aSjoerg CandidatesForRemoval.insert(CandidatesForRemoval.begin(), IntrinsicCursor);
358*da58b97aSjoerg Cursor = IntrinsicCursor->getOperand(0);
359*da58b97aSjoerg }
360*da58b97aSjoerg
361*da58b97aSjoerg // If no viable replacement in the conversion chain was found, there is
362*da58b97aSjoerg // nothing to do.
363*da58b97aSjoerg if (!EarliestReplacement)
364*da58b97aSjoerg return None;
365*da58b97aSjoerg
366*da58b97aSjoerg return IC.replaceInstUsesWith(II, EarliestReplacement);
367*da58b97aSjoerg }
368*da58b97aSjoerg
instCombineSVEDup(InstCombiner & IC,IntrinsicInst & II)369*da58b97aSjoerg static Optional<Instruction *> instCombineSVEDup(InstCombiner &IC,
370*da58b97aSjoerg IntrinsicInst &II) {
371*da58b97aSjoerg IntrinsicInst *Pg = dyn_cast<IntrinsicInst>(II.getArgOperand(1));
372*da58b97aSjoerg if (!Pg)
373*da58b97aSjoerg return None;
374*da58b97aSjoerg
375*da58b97aSjoerg if (Pg->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue)
376*da58b97aSjoerg return None;
377*da58b97aSjoerg
378*da58b97aSjoerg const auto PTruePattern =
379*da58b97aSjoerg cast<ConstantInt>(Pg->getOperand(0))->getZExtValue();
380*da58b97aSjoerg if (PTruePattern != AArch64SVEPredPattern::vl1)
381*da58b97aSjoerg return None;
382*da58b97aSjoerg
383*da58b97aSjoerg // The intrinsic is inserting into lane zero so use an insert instead.
384*da58b97aSjoerg auto *IdxTy = Type::getInt64Ty(II.getContext());
385*da58b97aSjoerg auto *Insert = InsertElementInst::Create(
386*da58b97aSjoerg II.getArgOperand(0), II.getArgOperand(2), ConstantInt::get(IdxTy, 0));
387*da58b97aSjoerg Insert->insertBefore(&II);
388*da58b97aSjoerg Insert->takeName(&II);
389*da58b97aSjoerg
390*da58b97aSjoerg return IC.replaceInstUsesWith(II, Insert);
391*da58b97aSjoerg }
392*da58b97aSjoerg
instCombineSVELast(InstCombiner & IC,IntrinsicInst & II)393*da58b97aSjoerg static Optional<Instruction *> instCombineSVELast(InstCombiner &IC,
394*da58b97aSjoerg IntrinsicInst &II) {
395*da58b97aSjoerg Value *Pg = II.getArgOperand(0);
396*da58b97aSjoerg Value *Vec = II.getArgOperand(1);
397*da58b97aSjoerg bool IsAfter = II.getIntrinsicID() == Intrinsic::aarch64_sve_lasta;
398*da58b97aSjoerg
399*da58b97aSjoerg auto *C = dyn_cast<Constant>(Pg);
400*da58b97aSjoerg if (IsAfter && C && C->isNullValue()) {
401*da58b97aSjoerg // The intrinsic is extracting lane 0 so use an extract instead.
402*da58b97aSjoerg auto *IdxTy = Type::getInt64Ty(II.getContext());
403*da58b97aSjoerg auto *Extract = ExtractElementInst::Create(Vec, ConstantInt::get(IdxTy, 0));
404*da58b97aSjoerg Extract->insertBefore(&II);
405*da58b97aSjoerg Extract->takeName(&II);
406*da58b97aSjoerg return IC.replaceInstUsesWith(II, Extract);
407*da58b97aSjoerg }
408*da58b97aSjoerg
409*da58b97aSjoerg auto *IntrPG = dyn_cast<IntrinsicInst>(Pg);
410*da58b97aSjoerg if (!IntrPG)
411*da58b97aSjoerg return None;
412*da58b97aSjoerg
413*da58b97aSjoerg if (IntrPG->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue)
414*da58b97aSjoerg return None;
415*da58b97aSjoerg
416*da58b97aSjoerg const auto PTruePattern =
417*da58b97aSjoerg cast<ConstantInt>(IntrPG->getOperand(0))->getZExtValue();
418*da58b97aSjoerg
419*da58b97aSjoerg // Can the intrinsic's predicate be converted to a known constant index?
420*da58b97aSjoerg unsigned Idx;
421*da58b97aSjoerg switch (PTruePattern) {
422*da58b97aSjoerg default:
423*da58b97aSjoerg return None;
424*da58b97aSjoerg case AArch64SVEPredPattern::vl1:
425*da58b97aSjoerg Idx = 0;
426*da58b97aSjoerg break;
427*da58b97aSjoerg case AArch64SVEPredPattern::vl2:
428*da58b97aSjoerg Idx = 1;
429*da58b97aSjoerg break;
430*da58b97aSjoerg case AArch64SVEPredPattern::vl3:
431*da58b97aSjoerg Idx = 2;
432*da58b97aSjoerg break;
433*da58b97aSjoerg case AArch64SVEPredPattern::vl4:
434*da58b97aSjoerg Idx = 3;
435*da58b97aSjoerg break;
436*da58b97aSjoerg case AArch64SVEPredPattern::vl5:
437*da58b97aSjoerg Idx = 4;
438*da58b97aSjoerg break;
439*da58b97aSjoerg case AArch64SVEPredPattern::vl6:
440*da58b97aSjoerg Idx = 5;
441*da58b97aSjoerg break;
442*da58b97aSjoerg case AArch64SVEPredPattern::vl7:
443*da58b97aSjoerg Idx = 6;
444*da58b97aSjoerg break;
445*da58b97aSjoerg case AArch64SVEPredPattern::vl8:
446*da58b97aSjoerg Idx = 7;
447*da58b97aSjoerg break;
448*da58b97aSjoerg case AArch64SVEPredPattern::vl16:
449*da58b97aSjoerg Idx = 15;
450*da58b97aSjoerg break;
451*da58b97aSjoerg }
452*da58b97aSjoerg
453*da58b97aSjoerg // Increment the index if extracting the element after the last active
454*da58b97aSjoerg // predicate element.
455*da58b97aSjoerg if (IsAfter)
456*da58b97aSjoerg ++Idx;
457*da58b97aSjoerg
458*da58b97aSjoerg // Ignore extracts whose index is larger than the known minimum vector
459*da58b97aSjoerg // length. NOTE: This is an artificial constraint where we prefer to
460*da58b97aSjoerg // maintain what the user asked for until an alternative is proven faster.
461*da58b97aSjoerg auto *PgVTy = cast<ScalableVectorType>(Pg->getType());
462*da58b97aSjoerg if (Idx >= PgVTy->getMinNumElements())
463*da58b97aSjoerg return None;
464*da58b97aSjoerg
465*da58b97aSjoerg // The intrinsic is extracting a fixed lane so use an extract instead.
466*da58b97aSjoerg auto *IdxTy = Type::getInt64Ty(II.getContext());
467*da58b97aSjoerg auto *Extract = ExtractElementInst::Create(Vec, ConstantInt::get(IdxTy, Idx));
468*da58b97aSjoerg Extract->insertBefore(&II);
469*da58b97aSjoerg Extract->takeName(&II);
470*da58b97aSjoerg return IC.replaceInstUsesWith(II, Extract);
471*da58b97aSjoerg }
472*da58b97aSjoerg
instCombineRDFFR(InstCombiner & IC,IntrinsicInst & II)473*da58b97aSjoerg static Optional<Instruction *> instCombineRDFFR(InstCombiner &IC,
474*da58b97aSjoerg IntrinsicInst &II) {
475*da58b97aSjoerg LLVMContext &Ctx = II.getContext();
476*da58b97aSjoerg IRBuilder<> Builder(Ctx);
477*da58b97aSjoerg Builder.SetInsertPoint(&II);
478*da58b97aSjoerg // Replace rdffr with predicated rdffr.z intrinsic, so that optimizePTestInstr
479*da58b97aSjoerg // can work with RDFFR_PP for ptest elimination.
480*da58b97aSjoerg auto *AllPat =
481*da58b97aSjoerg ConstantInt::get(Type::getInt32Ty(Ctx), AArch64SVEPredPattern::all);
482*da58b97aSjoerg auto *PTrue = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue,
483*da58b97aSjoerg {II.getType()}, {AllPat});
484*da58b97aSjoerg auto *RDFFR =
485*da58b97aSjoerg Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z, {}, {PTrue});
486*da58b97aSjoerg RDFFR->takeName(&II);
487*da58b97aSjoerg return IC.replaceInstUsesWith(II, RDFFR);
488*da58b97aSjoerg }
489*da58b97aSjoerg
490*da58b97aSjoerg Optional<Instruction *>
instCombineIntrinsic(InstCombiner & IC,IntrinsicInst & II) const491*da58b97aSjoerg AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
492*da58b97aSjoerg IntrinsicInst &II) const {
493*da58b97aSjoerg Intrinsic::ID IID = II.getIntrinsicID();
494*da58b97aSjoerg switch (IID) {
495*da58b97aSjoerg default:
496*da58b97aSjoerg break;
497*da58b97aSjoerg case Intrinsic::aarch64_sve_convert_from_svbool:
498*da58b97aSjoerg return instCombineConvertFromSVBool(IC, II);
499*da58b97aSjoerg case Intrinsic::aarch64_sve_dup:
500*da58b97aSjoerg return instCombineSVEDup(IC, II);
501*da58b97aSjoerg case Intrinsic::aarch64_sve_rdffr:
502*da58b97aSjoerg return instCombineRDFFR(IC, II);
503*da58b97aSjoerg case Intrinsic::aarch64_sve_lasta:
504*da58b97aSjoerg case Intrinsic::aarch64_sve_lastb:
505*da58b97aSjoerg return instCombineSVELast(IC, II);
506*da58b97aSjoerg }
507*da58b97aSjoerg
508*da58b97aSjoerg return None;
509*da58b97aSjoerg }
510*da58b97aSjoerg
isWideningInstruction(Type * DstTy,unsigned Opcode,ArrayRef<const Value * > Args)51106f32e7eSjoerg bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode,
51206f32e7eSjoerg ArrayRef<const Value *> Args) {
51306f32e7eSjoerg
51406f32e7eSjoerg // A helper that returns a vector type from the given type. The number of
51506f32e7eSjoerg // elements in type Ty determine the vector width.
51606f32e7eSjoerg auto toVectorTy = [&](Type *ArgTy) {
51706f32e7eSjoerg return VectorType::get(ArgTy->getScalarType(),
518*da58b97aSjoerg cast<VectorType>(DstTy)->getElementCount());
51906f32e7eSjoerg };
52006f32e7eSjoerg
52106f32e7eSjoerg // Exit early if DstTy is not a vector type whose elements are at least
52206f32e7eSjoerg // 16-bits wide.
52306f32e7eSjoerg if (!DstTy->isVectorTy() || DstTy->getScalarSizeInBits() < 16)
52406f32e7eSjoerg return false;
52506f32e7eSjoerg
52606f32e7eSjoerg // Determine if the operation has a widening variant. We consider both the
52706f32e7eSjoerg // "long" (e.g., usubl) and "wide" (e.g., usubw) versions of the
52806f32e7eSjoerg // instructions.
52906f32e7eSjoerg //
53006f32e7eSjoerg // TODO: Add additional widening operations (e.g., mul, shl, etc.) once we
53106f32e7eSjoerg // verify that their extending operands are eliminated during code
53206f32e7eSjoerg // generation.
53306f32e7eSjoerg switch (Opcode) {
53406f32e7eSjoerg case Instruction::Add: // UADDL(2), SADDL(2), UADDW(2), SADDW(2).
53506f32e7eSjoerg case Instruction::Sub: // USUBL(2), SSUBL(2), USUBW(2), SSUBW(2).
53606f32e7eSjoerg break;
53706f32e7eSjoerg default:
53806f32e7eSjoerg return false;
53906f32e7eSjoerg }
54006f32e7eSjoerg
54106f32e7eSjoerg // To be a widening instruction (either the "wide" or "long" versions), the
54206f32e7eSjoerg // second operand must be a sign- or zero extend having a single user. We
54306f32e7eSjoerg // only consider extends having a single user because they may otherwise not
54406f32e7eSjoerg // be eliminated.
54506f32e7eSjoerg if (Args.size() != 2 ||
54606f32e7eSjoerg (!isa<SExtInst>(Args[1]) && !isa<ZExtInst>(Args[1])) ||
54706f32e7eSjoerg !Args[1]->hasOneUse())
54806f32e7eSjoerg return false;
54906f32e7eSjoerg auto *Extend = cast<CastInst>(Args[1]);
55006f32e7eSjoerg
55106f32e7eSjoerg // Legalize the destination type and ensure it can be used in a widening
55206f32e7eSjoerg // operation.
55306f32e7eSjoerg auto DstTyL = TLI->getTypeLegalizationCost(DL, DstTy);
55406f32e7eSjoerg unsigned DstElTySize = DstTyL.second.getScalarSizeInBits();
55506f32e7eSjoerg if (!DstTyL.second.isVector() || DstElTySize != DstTy->getScalarSizeInBits())
55606f32e7eSjoerg return false;
55706f32e7eSjoerg
55806f32e7eSjoerg // Legalize the source type and ensure it can be used in a widening
55906f32e7eSjoerg // operation.
560*da58b97aSjoerg auto *SrcTy = toVectorTy(Extend->getSrcTy());
56106f32e7eSjoerg auto SrcTyL = TLI->getTypeLegalizationCost(DL, SrcTy);
56206f32e7eSjoerg unsigned SrcElTySize = SrcTyL.second.getScalarSizeInBits();
56306f32e7eSjoerg if (!SrcTyL.second.isVector() || SrcElTySize != SrcTy->getScalarSizeInBits())
56406f32e7eSjoerg return false;
56506f32e7eSjoerg
56606f32e7eSjoerg // Get the total number of vector elements in the legalized types.
567*da58b97aSjoerg InstructionCost NumDstEls =
568*da58b97aSjoerg DstTyL.first * DstTyL.second.getVectorMinNumElements();
569*da58b97aSjoerg InstructionCost NumSrcEls =
570*da58b97aSjoerg SrcTyL.first * SrcTyL.second.getVectorMinNumElements();
57106f32e7eSjoerg
57206f32e7eSjoerg // Return true if the legalized types have the same number of vector elements
57306f32e7eSjoerg // and the destination element type size is twice that of the source type.
57406f32e7eSjoerg return NumDstEls == NumSrcEls && 2 * SrcElTySize == DstElTySize;
57506f32e7eSjoerg }
57606f32e7eSjoerg
getCastInstrCost(unsigned Opcode,Type * Dst,Type * Src,TTI::CastContextHint CCH,TTI::TargetCostKind CostKind,const Instruction * I)577*da58b97aSjoerg InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
578*da58b97aSjoerg Type *Src,
579*da58b97aSjoerg TTI::CastContextHint CCH,
580*da58b97aSjoerg TTI::TargetCostKind CostKind,
58106f32e7eSjoerg const Instruction *I) {
58206f32e7eSjoerg int ISD = TLI->InstructionOpcodeToISD(Opcode);
58306f32e7eSjoerg assert(ISD && "Invalid opcode");
58406f32e7eSjoerg
58506f32e7eSjoerg // If the cast is observable, and it is used by a widening instruction (e.g.,
58606f32e7eSjoerg // uaddl, saddw, etc.), it may be free.
58706f32e7eSjoerg if (I && I->hasOneUse()) {
58806f32e7eSjoerg auto *SingleUser = cast<Instruction>(*I->user_begin());
58906f32e7eSjoerg SmallVector<const Value *, 4> Operands(SingleUser->operand_values());
59006f32e7eSjoerg if (isWideningInstruction(Dst, SingleUser->getOpcode(), Operands)) {
59106f32e7eSjoerg // If the cast is the second operand, it is free. We will generate either
59206f32e7eSjoerg // a "wide" or "long" version of the widening instruction.
59306f32e7eSjoerg if (I == SingleUser->getOperand(1))
59406f32e7eSjoerg return 0;
59506f32e7eSjoerg // If the cast is not the second operand, it will be free if it looks the
59606f32e7eSjoerg // same as the second operand. In this case, we will generate a "long"
59706f32e7eSjoerg // version of the widening instruction.
59806f32e7eSjoerg if (auto *Cast = dyn_cast<CastInst>(SingleUser->getOperand(1)))
59906f32e7eSjoerg if (I->getOpcode() == unsigned(Cast->getOpcode()) &&
60006f32e7eSjoerg cast<CastInst>(I)->getSrcTy() == Cast->getSrcTy())
60106f32e7eSjoerg return 0;
60206f32e7eSjoerg }
60306f32e7eSjoerg }
60406f32e7eSjoerg
605*da58b97aSjoerg // TODO: Allow non-throughput costs that aren't binary.
606*da58b97aSjoerg auto AdjustCost = [&CostKind](InstructionCost Cost) -> InstructionCost {
607*da58b97aSjoerg if (CostKind != TTI::TCK_RecipThroughput)
608*da58b97aSjoerg return Cost == 0 ? 0 : 1;
609*da58b97aSjoerg return Cost;
610*da58b97aSjoerg };
611*da58b97aSjoerg
61206f32e7eSjoerg EVT SrcTy = TLI->getValueType(DL, Src);
61306f32e7eSjoerg EVT DstTy = TLI->getValueType(DL, Dst);
61406f32e7eSjoerg
61506f32e7eSjoerg if (!SrcTy.isSimple() || !DstTy.isSimple())
616*da58b97aSjoerg return AdjustCost(
617*da58b97aSjoerg BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
61806f32e7eSjoerg
61906f32e7eSjoerg static const TypeConversionCostTblEntry
62006f32e7eSjoerg ConversionTbl[] = {
62106f32e7eSjoerg { ISD::TRUNCATE, MVT::v4i16, MVT::v4i32, 1 },
62206f32e7eSjoerg { ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, 0 },
62306f32e7eSjoerg { ISD::TRUNCATE, MVT::v8i8, MVT::v8i32, 3 },
62406f32e7eSjoerg { ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, 6 },
62506f32e7eSjoerg
626*da58b97aSjoerg // Truncations on nxvmiN
627*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i16, 1 },
628*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i32, 1 },
629*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i64, 1 },
630*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i16, 1 },
631*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i32, 1 },
632*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i64, 2 },
633*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i16, 1 },
634*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i32, 3 },
635*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i64, 5 },
636*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv2i16, MVT::nxv2i32, 1 },
637*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv2i32, MVT::nxv2i64, 1 },
638*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv4i16, MVT::nxv4i32, 1 },
639*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv4i32, MVT::nxv4i64, 2 },
640*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv8i16, MVT::nxv8i32, 3 },
641*da58b97aSjoerg { ISD::TRUNCATE, MVT::nxv8i32, MVT::nxv8i64, 6 },
642*da58b97aSjoerg
64306f32e7eSjoerg // The number of shll instructions for the extension.
64406f32e7eSjoerg { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i16, 3 },
64506f32e7eSjoerg { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i16, 3 },
64606f32e7eSjoerg { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, 2 },
64706f32e7eSjoerg { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, 2 },
64806f32e7eSjoerg { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i8, 3 },
64906f32e7eSjoerg { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i8, 3 },
65006f32e7eSjoerg { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 2 },
65106f32e7eSjoerg { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 2 },
65206f32e7eSjoerg { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i8, 7 },
65306f32e7eSjoerg { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i8, 7 },
65406f32e7eSjoerg { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i16, 6 },
65506f32e7eSjoerg { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i16, 6 },
65606f32e7eSjoerg { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 2 },
65706f32e7eSjoerg { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 2 },
65806f32e7eSjoerg { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, 6 },
65906f32e7eSjoerg { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, 6 },
66006f32e7eSjoerg
66106f32e7eSjoerg // LowerVectorINT_TO_FP:
66206f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i32, 1 },
66306f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 1 },
66406f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i64, 1 },
66506f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, 1 },
66606f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 1 },
66706f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, 1 },
66806f32e7eSjoerg
66906f32e7eSjoerg // Complex: to v2f32
67006f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i8, 3 },
67106f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i16, 3 },
67206f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i64, 2 },
67306f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i8, 3 },
67406f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i16, 3 },
67506f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i64, 2 },
67606f32e7eSjoerg
67706f32e7eSjoerg // Complex: to v4f32
67806f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i8, 4 },
67906f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i16, 2 },
68006f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i8, 3 },
68106f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i16, 2 },
68206f32e7eSjoerg
68306f32e7eSjoerg // Complex: to v8f32
68406f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i8, 10 },
68506f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, 4 },
68606f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i8, 10 },
68706f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, 4 },
68806f32e7eSjoerg
68906f32e7eSjoerg // Complex: to v16f32
69006f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i8, 21 },
69106f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i8, 21 },
69206f32e7eSjoerg
69306f32e7eSjoerg // Complex: to v2f64
69406f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i8, 4 },
69506f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i16, 4 },
69606f32e7eSjoerg { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 },
69706f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i8, 4 },
69806f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i16, 4 },
69906f32e7eSjoerg { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 },
70006f32e7eSjoerg
70106f32e7eSjoerg
70206f32e7eSjoerg // LowerVectorFP_TO_INT
70306f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f32, 1 },
70406f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f32, 1 },
70506f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f64, 1 },
70606f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f32, 1 },
70706f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, 1 },
70806f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f64, 1 },
70906f32e7eSjoerg
71006f32e7eSjoerg // Complex, from v2f32: legal type is v2i32 (no cost) or v2i64 (1 ext).
71106f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f32, 2 },
71206f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f32, 1 },
71306f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v2i8, MVT::v2f32, 1 },
71406f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f32, 2 },
71506f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f32, 1 },
71606f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v2i8, MVT::v2f32, 1 },
71706f32e7eSjoerg
71806f32e7eSjoerg // Complex, from v4f32: legal type is v4i16, 1 narrowing => ~2
71906f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v4i16, MVT::v4f32, 2 },
72006f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v4i8, MVT::v4f32, 2 },
72106f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v4i16, MVT::v4f32, 2 },
72206f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v4i8, MVT::v4f32, 2 },
72306f32e7eSjoerg
724*da58b97aSjoerg // Complex, from nxv2f32.
725*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f32, 1 },
726*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f32, 1 },
727*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f32, 1 },
728*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f32, 1 },
729*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f32, 1 },
730*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f32, 1 },
731*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f32, 1 },
732*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f32, 1 },
733*da58b97aSjoerg
73406f32e7eSjoerg // Complex, from v2f64: legal type is v2i32, 1 narrowing => ~2.
73506f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f64, 2 },
73606f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f64, 2 },
73706f32e7eSjoerg { ISD::FP_TO_SINT, MVT::v2i8, MVT::v2f64, 2 },
73806f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f64, 2 },
73906f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f64, 2 },
74006f32e7eSjoerg { ISD::FP_TO_UINT, MVT::v2i8, MVT::v2f64, 2 },
741*da58b97aSjoerg
742*da58b97aSjoerg // Complex, from nxv2f64.
743*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f64, 1 },
744*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f64, 1 },
745*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f64, 1 },
746*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f64, 1 },
747*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f64, 1 },
748*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f64, 1 },
749*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f64, 1 },
750*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f64, 1 },
751*da58b97aSjoerg
752*da58b97aSjoerg // Complex, from nxv4f32.
753*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f32, 4 },
754*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f32, 1 },
755*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f32, 1 },
756*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f32, 1 },
757*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f32, 4 },
758*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f32, 1 },
759*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f32, 1 },
760*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f32, 1 },
761*da58b97aSjoerg
762*da58b97aSjoerg // Complex, from nxv8f64. Illegal -> illegal conversions not required.
763*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f64, 7 },
764*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f64, 7 },
765*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f64, 7 },
766*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f64, 7 },
767*da58b97aSjoerg
768*da58b97aSjoerg // Complex, from nxv4f64. Illegal -> illegal conversions not required.
769*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f64, 3 },
770*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f64, 3 },
771*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f64, 3 },
772*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f64, 3 },
773*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f64, 3 },
774*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f64, 3 },
775*da58b97aSjoerg
776*da58b97aSjoerg // Complex, from nxv8f32. Illegal -> illegal conversions not required.
777*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f32, 3 },
778*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f32, 3 },
779*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f32, 3 },
780*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f32, 3 },
781*da58b97aSjoerg
782*da58b97aSjoerg // Complex, from nxv8f16.
783*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv8i64, MVT::nxv8f16, 10 },
784*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv8i32, MVT::nxv8f16, 4 },
785*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f16, 1 },
786*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f16, 1 },
787*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv8i64, MVT::nxv8f16, 10 },
788*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv8i32, MVT::nxv8f16, 4 },
789*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f16, 1 },
790*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f16, 1 },
791*da58b97aSjoerg
792*da58b97aSjoerg // Complex, from nxv4f16.
793*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f16, 4 },
794*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f16, 1 },
795*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f16, 1 },
796*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f16, 1 },
797*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f16, 4 },
798*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f16, 1 },
799*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f16, 1 },
800*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f16, 1 },
801*da58b97aSjoerg
802*da58b97aSjoerg // Complex, from nxv2f16.
803*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f16, 1 },
804*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f16, 1 },
805*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f16, 1 },
806*da58b97aSjoerg { ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f16, 1 },
807*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f16, 1 },
808*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f16, 1 },
809*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f16, 1 },
810*da58b97aSjoerg { ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f16, 1 },
811*da58b97aSjoerg
812*da58b97aSjoerg // Truncate from nxvmf32 to nxvmf16.
813*da58b97aSjoerg { ISD::FP_ROUND, MVT::nxv2f16, MVT::nxv2f32, 1 },
814*da58b97aSjoerg { ISD::FP_ROUND, MVT::nxv4f16, MVT::nxv4f32, 1 },
815*da58b97aSjoerg { ISD::FP_ROUND, MVT::nxv8f16, MVT::nxv8f32, 3 },
816*da58b97aSjoerg
817*da58b97aSjoerg // Truncate from nxvmf64 to nxvmf16.
818*da58b97aSjoerg { ISD::FP_ROUND, MVT::nxv2f16, MVT::nxv2f64, 1 },
819*da58b97aSjoerg { ISD::FP_ROUND, MVT::nxv4f16, MVT::nxv4f64, 3 },
820*da58b97aSjoerg { ISD::FP_ROUND, MVT::nxv8f16, MVT::nxv8f64, 7 },
821*da58b97aSjoerg
822*da58b97aSjoerg // Truncate from nxvmf64 to nxvmf32.
823*da58b97aSjoerg { ISD::FP_ROUND, MVT::nxv2f32, MVT::nxv2f64, 1 },
824*da58b97aSjoerg { ISD::FP_ROUND, MVT::nxv4f32, MVT::nxv4f64, 3 },
825*da58b97aSjoerg { ISD::FP_ROUND, MVT::nxv8f32, MVT::nxv8f64, 6 },
826*da58b97aSjoerg
827*da58b97aSjoerg // Extend from nxvmf16 to nxvmf32.
828*da58b97aSjoerg { ISD::FP_EXTEND, MVT::nxv2f32, MVT::nxv2f16, 1},
829*da58b97aSjoerg { ISD::FP_EXTEND, MVT::nxv4f32, MVT::nxv4f16, 1},
830*da58b97aSjoerg { ISD::FP_EXTEND, MVT::nxv8f32, MVT::nxv8f16, 2},
831*da58b97aSjoerg
832*da58b97aSjoerg // Extend from nxvmf16 to nxvmf64.
833*da58b97aSjoerg { ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2f16, 1},
834*da58b97aSjoerg { ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f16, 2},
835*da58b97aSjoerg { ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f16, 4},
836*da58b97aSjoerg
837*da58b97aSjoerg // Extend from nxvmf32 to nxvmf64.
838*da58b97aSjoerg { ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2f32, 1},
839*da58b97aSjoerg { ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f32, 2},
840*da58b97aSjoerg { ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f32, 6},
841*da58b97aSjoerg
84206f32e7eSjoerg };
84306f32e7eSjoerg
84406f32e7eSjoerg if (const auto *Entry = ConvertCostTableLookup(ConversionTbl, ISD,
84506f32e7eSjoerg DstTy.getSimpleVT(),
84606f32e7eSjoerg SrcTy.getSimpleVT()))
847*da58b97aSjoerg return AdjustCost(Entry->Cost);
84806f32e7eSjoerg
849*da58b97aSjoerg return AdjustCost(
850*da58b97aSjoerg BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
85106f32e7eSjoerg }
85206f32e7eSjoerg
getExtractWithExtendCost(unsigned Opcode,Type * Dst,VectorType * VecTy,unsigned Index)853*da58b97aSjoerg InstructionCost AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode,
854*da58b97aSjoerg Type *Dst,
85506f32e7eSjoerg VectorType *VecTy,
85606f32e7eSjoerg unsigned Index) {
85706f32e7eSjoerg
85806f32e7eSjoerg // Make sure we were given a valid extend opcode.
85906f32e7eSjoerg assert((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) &&
86006f32e7eSjoerg "Invalid opcode");
86106f32e7eSjoerg
86206f32e7eSjoerg // We are extending an element we extract from a vector, so the source type
86306f32e7eSjoerg // of the extend is the element type of the vector.
86406f32e7eSjoerg auto *Src = VecTy->getElementType();
86506f32e7eSjoerg
86606f32e7eSjoerg // Sign- and zero-extends are for integer types only.
86706f32e7eSjoerg assert(isa<IntegerType>(Dst) && isa<IntegerType>(Src) && "Invalid type");
86806f32e7eSjoerg
86906f32e7eSjoerg // Get the cost for the extract. We compute the cost (if any) for the extend
87006f32e7eSjoerg // below.
871*da58b97aSjoerg InstructionCost Cost =
872*da58b97aSjoerg getVectorInstrCost(Instruction::ExtractElement, VecTy, Index);
87306f32e7eSjoerg
87406f32e7eSjoerg // Legalize the types.
87506f32e7eSjoerg auto VecLT = TLI->getTypeLegalizationCost(DL, VecTy);
87606f32e7eSjoerg auto DstVT = TLI->getValueType(DL, Dst);
87706f32e7eSjoerg auto SrcVT = TLI->getValueType(DL, Src);
878*da58b97aSjoerg TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
87906f32e7eSjoerg
88006f32e7eSjoerg // If the resulting type is still a vector and the destination type is legal,
88106f32e7eSjoerg // we may get the extension for free. If not, get the default cost for the
88206f32e7eSjoerg // extend.
88306f32e7eSjoerg if (!VecLT.second.isVector() || !TLI->isTypeLegal(DstVT))
884*da58b97aSjoerg return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
885*da58b97aSjoerg CostKind);
88606f32e7eSjoerg
88706f32e7eSjoerg // The destination type should be larger than the element type. If not, get
88806f32e7eSjoerg // the default cost for the extend.
889*da58b97aSjoerg if (DstVT.getFixedSizeInBits() < SrcVT.getFixedSizeInBits())
890*da58b97aSjoerg return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
891*da58b97aSjoerg CostKind);
89206f32e7eSjoerg
89306f32e7eSjoerg switch (Opcode) {
89406f32e7eSjoerg default:
89506f32e7eSjoerg llvm_unreachable("Opcode should be either SExt or ZExt");
89606f32e7eSjoerg
89706f32e7eSjoerg // For sign-extends, we only need a smov, which performs the extension
89806f32e7eSjoerg // automatically.
89906f32e7eSjoerg case Instruction::SExt:
90006f32e7eSjoerg return Cost;
90106f32e7eSjoerg
90206f32e7eSjoerg // For zero-extends, the extend is performed automatically by a umov unless
90306f32e7eSjoerg // the destination type is i64 and the element type is i8 or i16.
90406f32e7eSjoerg case Instruction::ZExt:
90506f32e7eSjoerg if (DstVT.getSizeInBits() != 64u || SrcVT.getSizeInBits() == 32u)
90606f32e7eSjoerg return Cost;
90706f32e7eSjoerg }
90806f32e7eSjoerg
90906f32e7eSjoerg // If we are unable to perform the extend for free, get the default cost.
910*da58b97aSjoerg return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
911*da58b97aSjoerg CostKind);
91206f32e7eSjoerg }
91306f32e7eSjoerg
getCFInstrCost(unsigned Opcode,TTI::TargetCostKind CostKind,const Instruction * I)914*da58b97aSjoerg InstructionCost AArch64TTIImpl::getCFInstrCost(unsigned Opcode,
915*da58b97aSjoerg TTI::TargetCostKind CostKind,
916*da58b97aSjoerg const Instruction *I) {
917*da58b97aSjoerg if (CostKind != TTI::TCK_RecipThroughput)
918*da58b97aSjoerg return Opcode == Instruction::PHI ? 0 : 1;
919*da58b97aSjoerg assert(CostKind == TTI::TCK_RecipThroughput && "unexpected CostKind");
920*da58b97aSjoerg // Branches are assumed to be predicted.
921*da58b97aSjoerg return 0;
922*da58b97aSjoerg }
923*da58b97aSjoerg
getVectorInstrCost(unsigned Opcode,Type * Val,unsigned Index)924*da58b97aSjoerg InstructionCost AArch64TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
92506f32e7eSjoerg unsigned Index) {
92606f32e7eSjoerg assert(Val->isVectorTy() && "This must be a vector type");
92706f32e7eSjoerg
92806f32e7eSjoerg if (Index != -1U) {
92906f32e7eSjoerg // Legalize the type.
930*da58b97aSjoerg std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Val);
93106f32e7eSjoerg
93206f32e7eSjoerg // This type is legalized to a scalar type.
93306f32e7eSjoerg if (!LT.second.isVector())
93406f32e7eSjoerg return 0;
93506f32e7eSjoerg
93606f32e7eSjoerg // The type may be split. Normalize the index to the new type.
93706f32e7eSjoerg unsigned Width = LT.second.getVectorNumElements();
93806f32e7eSjoerg Index = Index % Width;
93906f32e7eSjoerg
94006f32e7eSjoerg // The element at index zero is already inside the vector.
94106f32e7eSjoerg if (Index == 0)
94206f32e7eSjoerg return 0;
94306f32e7eSjoerg }
94406f32e7eSjoerg
94506f32e7eSjoerg // All other insert/extracts cost this much.
94606f32e7eSjoerg return ST->getVectorInsertExtractBaseCost();
94706f32e7eSjoerg }
94806f32e7eSjoerg
getArithmeticInstrCost(unsigned Opcode,Type * Ty,TTI::TargetCostKind CostKind,TTI::OperandValueKind Opd1Info,TTI::OperandValueKind Opd2Info,TTI::OperandValueProperties Opd1PropInfo,TTI::OperandValueProperties Opd2PropInfo,ArrayRef<const Value * > Args,const Instruction * CxtI)949*da58b97aSjoerg InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
950*da58b97aSjoerg unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
951*da58b97aSjoerg TTI::OperandValueKind Opd1Info, TTI::OperandValueKind Opd2Info,
952*da58b97aSjoerg TTI::OperandValueProperties Opd1PropInfo,
953*da58b97aSjoerg TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args,
954*da58b97aSjoerg const Instruction *CxtI) {
955*da58b97aSjoerg // TODO: Handle more cost kinds.
956*da58b97aSjoerg if (CostKind != TTI::TCK_RecipThroughput)
957*da58b97aSjoerg return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info,
958*da58b97aSjoerg Opd2Info, Opd1PropInfo,
959*da58b97aSjoerg Opd2PropInfo, Args, CxtI);
960*da58b97aSjoerg
96106f32e7eSjoerg // Legalize the type.
962*da58b97aSjoerg std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
96306f32e7eSjoerg
96406f32e7eSjoerg // If the instruction is a widening instruction (e.g., uaddl, saddw, etc.),
96506f32e7eSjoerg // add in the widening overhead specified by the sub-target. Since the
96606f32e7eSjoerg // extends feeding widening instructions are performed automatically, they
96706f32e7eSjoerg // aren't present in the generated code and have a zero cost. By adding a
96806f32e7eSjoerg // widening overhead here, we attach the total cost of the combined operation
96906f32e7eSjoerg // to the widening instruction.
970*da58b97aSjoerg InstructionCost Cost = 0;
97106f32e7eSjoerg if (isWideningInstruction(Ty, Opcode, Args))
97206f32e7eSjoerg Cost += ST->getWideningBaseCost();
97306f32e7eSjoerg
97406f32e7eSjoerg int ISD = TLI->InstructionOpcodeToISD(Opcode);
97506f32e7eSjoerg
97606f32e7eSjoerg switch (ISD) {
97706f32e7eSjoerg default:
978*da58b97aSjoerg return Cost + BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info,
979*da58b97aSjoerg Opd2Info,
98006f32e7eSjoerg Opd1PropInfo, Opd2PropInfo);
98106f32e7eSjoerg case ISD::SDIV:
98206f32e7eSjoerg if (Opd2Info == TargetTransformInfo::OK_UniformConstantValue &&
98306f32e7eSjoerg Opd2PropInfo == TargetTransformInfo::OP_PowerOf2) {
98406f32e7eSjoerg // On AArch64, scalar signed division by constants power-of-two are
98506f32e7eSjoerg // normally expanded to the sequence ADD + CMP + SELECT + SRA.
98606f32e7eSjoerg // The OperandValue properties many not be same as that of previous
98706f32e7eSjoerg // operation; conservatively assume OP_None.
988*da58b97aSjoerg Cost += getArithmeticInstrCost(Instruction::Add, Ty, CostKind,
989*da58b97aSjoerg Opd1Info, Opd2Info,
99006f32e7eSjoerg TargetTransformInfo::OP_None,
99106f32e7eSjoerg TargetTransformInfo::OP_None);
992*da58b97aSjoerg Cost += getArithmeticInstrCost(Instruction::Sub, Ty, CostKind,
993*da58b97aSjoerg Opd1Info, Opd2Info,
99406f32e7eSjoerg TargetTransformInfo::OP_None,
99506f32e7eSjoerg TargetTransformInfo::OP_None);
996*da58b97aSjoerg Cost += getArithmeticInstrCost(Instruction::Select, Ty, CostKind,
997*da58b97aSjoerg Opd1Info, Opd2Info,
99806f32e7eSjoerg TargetTransformInfo::OP_None,
99906f32e7eSjoerg TargetTransformInfo::OP_None);
1000*da58b97aSjoerg Cost += getArithmeticInstrCost(Instruction::AShr, Ty, CostKind,
1001*da58b97aSjoerg Opd1Info, Opd2Info,
100206f32e7eSjoerg TargetTransformInfo::OP_None,
100306f32e7eSjoerg TargetTransformInfo::OP_None);
100406f32e7eSjoerg return Cost;
100506f32e7eSjoerg }
100606f32e7eSjoerg LLVM_FALLTHROUGH;
100706f32e7eSjoerg case ISD::UDIV:
100806f32e7eSjoerg if (Opd2Info == TargetTransformInfo::OK_UniformConstantValue) {
100906f32e7eSjoerg auto VT = TLI->getValueType(DL, Ty);
101006f32e7eSjoerg if (TLI->isOperationLegalOrCustom(ISD::MULHU, VT)) {
101106f32e7eSjoerg // Vector signed division by constant are expanded to the
101206f32e7eSjoerg // sequence MULHS + ADD/SUB + SRA + SRL + ADD, and unsigned division
101306f32e7eSjoerg // to MULHS + SUB + SRL + ADD + SRL.
1014*da58b97aSjoerg InstructionCost MulCost = getArithmeticInstrCost(
1015*da58b97aSjoerg Instruction::Mul, Ty, CostKind, Opd1Info, Opd2Info,
1016*da58b97aSjoerg TargetTransformInfo::OP_None, TargetTransformInfo::OP_None);
1017*da58b97aSjoerg InstructionCost AddCost = getArithmeticInstrCost(
1018*da58b97aSjoerg Instruction::Add, Ty, CostKind, Opd1Info, Opd2Info,
1019*da58b97aSjoerg TargetTransformInfo::OP_None, TargetTransformInfo::OP_None);
1020*da58b97aSjoerg InstructionCost ShrCost = getArithmeticInstrCost(
1021*da58b97aSjoerg Instruction::AShr, Ty, CostKind, Opd1Info, Opd2Info,
1022*da58b97aSjoerg TargetTransformInfo::OP_None, TargetTransformInfo::OP_None);
102306f32e7eSjoerg return MulCost * 2 + AddCost * 2 + ShrCost * 2 + 1;
102406f32e7eSjoerg }
102506f32e7eSjoerg }
102606f32e7eSjoerg
1027*da58b97aSjoerg Cost += BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info,
1028*da58b97aSjoerg Opd2Info,
102906f32e7eSjoerg Opd1PropInfo, Opd2PropInfo);
103006f32e7eSjoerg if (Ty->isVectorTy()) {
103106f32e7eSjoerg // On AArch64, vector divisions are not supported natively and are
103206f32e7eSjoerg // expanded into scalar divisions of each pair of elements.
1033*da58b97aSjoerg Cost += getArithmeticInstrCost(Instruction::ExtractElement, Ty, CostKind,
1034*da58b97aSjoerg Opd1Info, Opd2Info, Opd1PropInfo,
1035*da58b97aSjoerg Opd2PropInfo);
1036*da58b97aSjoerg Cost += getArithmeticInstrCost(Instruction::InsertElement, Ty, CostKind,
1037*da58b97aSjoerg Opd1Info, Opd2Info, Opd1PropInfo,
1038*da58b97aSjoerg Opd2PropInfo);
103906f32e7eSjoerg // TODO: if one of the arguments is scalar, then it's not necessary to
104006f32e7eSjoerg // double the cost of handling the vector elements.
104106f32e7eSjoerg Cost += Cost;
104206f32e7eSjoerg }
104306f32e7eSjoerg return Cost;
104406f32e7eSjoerg
104506f32e7eSjoerg case ISD::MUL:
1046*da58b97aSjoerg if (LT.second != MVT::v2i64)
1047*da58b97aSjoerg return (Cost + 1) * LT.first;
1048*da58b97aSjoerg // Since we do not have a MUL.2d instruction, a mul <2 x i64> is expensive
1049*da58b97aSjoerg // as elements are extracted from the vectors and the muls scalarized.
1050*da58b97aSjoerg // As getScalarizationOverhead is a bit too pessimistic, we estimate the
1051*da58b97aSjoerg // cost for a i64 vector directly here, which is:
1052*da58b97aSjoerg // - four i64 extracts,
1053*da58b97aSjoerg // - two i64 inserts, and
1054*da58b97aSjoerg // - two muls.
1055*da58b97aSjoerg // So, for a v2i64 with LT.First = 1 the cost is 8, and for a v4i64 with
1056*da58b97aSjoerg // LT.first = 2 the cost is 16.
1057*da58b97aSjoerg return LT.first * 8;
1058*da58b97aSjoerg case ISD::ADD:
105906f32e7eSjoerg case ISD::XOR:
106006f32e7eSjoerg case ISD::OR:
106106f32e7eSjoerg case ISD::AND:
106206f32e7eSjoerg // These nodes are marked as 'custom' for combining purposes only.
106306f32e7eSjoerg // We know that they are legal. See LowerAdd in ISelLowering.
106406f32e7eSjoerg return (Cost + 1) * LT.first;
1065*da58b97aSjoerg
1066*da58b97aSjoerg case ISD::FADD:
1067*da58b97aSjoerg // These nodes are marked as 'custom' just to lower them to SVE.
1068*da58b97aSjoerg // We know said lowering will incur no additional cost.
1069*da58b97aSjoerg if (isa<FixedVectorType>(Ty) && !Ty->getScalarType()->isFP128Ty())
1070*da58b97aSjoerg return (Cost + 2) * LT.first;
1071*da58b97aSjoerg
1072*da58b97aSjoerg return Cost + BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info,
1073*da58b97aSjoerg Opd2Info,
1074*da58b97aSjoerg Opd1PropInfo, Opd2PropInfo);
107506f32e7eSjoerg }
107606f32e7eSjoerg }
107706f32e7eSjoerg
getAddressComputationCost(Type * Ty,ScalarEvolution * SE,const SCEV * Ptr)1078*da58b97aSjoerg InstructionCost AArch64TTIImpl::getAddressComputationCost(Type *Ty,
1079*da58b97aSjoerg ScalarEvolution *SE,
108006f32e7eSjoerg const SCEV *Ptr) {
108106f32e7eSjoerg // Address computations in vectorized code with non-consecutive addresses will
108206f32e7eSjoerg // likely result in more instructions compared to scalar code where the
108306f32e7eSjoerg // computation can more often be merged into the index mode. The resulting
108406f32e7eSjoerg // extra micro-ops can significantly decrease throughput.
108506f32e7eSjoerg unsigned NumVectorInstToHideOverhead = 10;
108606f32e7eSjoerg int MaxMergeDistance = 64;
108706f32e7eSjoerg
108806f32e7eSjoerg if (Ty->isVectorTy() && SE &&
108906f32e7eSjoerg !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MaxMergeDistance + 1))
109006f32e7eSjoerg return NumVectorInstToHideOverhead;
109106f32e7eSjoerg
109206f32e7eSjoerg // In many cases the address computation is not merged into the instruction
109306f32e7eSjoerg // addressing mode.
109406f32e7eSjoerg return 1;
109506f32e7eSjoerg }
109606f32e7eSjoerg
getCmpSelInstrCost(unsigned Opcode,Type * ValTy,Type * CondTy,CmpInst::Predicate VecPred,TTI::TargetCostKind CostKind,const Instruction * I)1097*da58b97aSjoerg InstructionCost AArch64TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
1098*da58b97aSjoerg Type *CondTy,
1099*da58b97aSjoerg CmpInst::Predicate VecPred,
1100*da58b97aSjoerg TTI::TargetCostKind CostKind,
1101*da58b97aSjoerg const Instruction *I) {
1102*da58b97aSjoerg // TODO: Handle other cost kinds.
1103*da58b97aSjoerg if (CostKind != TTI::TCK_RecipThroughput)
1104*da58b97aSjoerg return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
1105*da58b97aSjoerg I);
110606f32e7eSjoerg
110706f32e7eSjoerg int ISD = TLI->InstructionOpcodeToISD(Opcode);
110806f32e7eSjoerg // We don't lower some vector selects well that are wider than the register
110906f32e7eSjoerg // width.
1110*da58b97aSjoerg if (isa<FixedVectorType>(ValTy) && ISD == ISD::SELECT) {
111106f32e7eSjoerg // We would need this many instructions to hide the scalarization happening.
111206f32e7eSjoerg const int AmortizationCost = 20;
1113*da58b97aSjoerg
1114*da58b97aSjoerg // If VecPred is not set, check if we can get a predicate from the context
1115*da58b97aSjoerg // instruction, if its type matches the requested ValTy.
1116*da58b97aSjoerg if (VecPred == CmpInst::BAD_ICMP_PREDICATE && I && I->getType() == ValTy) {
1117*da58b97aSjoerg CmpInst::Predicate CurrentPred;
1118*da58b97aSjoerg if (match(I, m_Select(m_Cmp(CurrentPred, m_Value(), m_Value()), m_Value(),
1119*da58b97aSjoerg m_Value())))
1120*da58b97aSjoerg VecPred = CurrentPred;
1121*da58b97aSjoerg }
1122*da58b97aSjoerg // Check if we have a compare/select chain that can be lowered using CMxx &
1123*da58b97aSjoerg // BFI pair.
1124*da58b97aSjoerg if (CmpInst::isIntPredicate(VecPred)) {
1125*da58b97aSjoerg static const auto ValidMinMaxTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
1126*da58b97aSjoerg MVT::v8i16, MVT::v2i32, MVT::v4i32,
1127*da58b97aSjoerg MVT::v2i64};
1128*da58b97aSjoerg auto LT = TLI->getTypeLegalizationCost(DL, ValTy);
1129*da58b97aSjoerg if (any_of(ValidMinMaxTys, [<](MVT M) { return M == LT.second; }))
1130*da58b97aSjoerg return LT.first;
1131*da58b97aSjoerg }
1132*da58b97aSjoerg
113306f32e7eSjoerg static const TypeConversionCostTblEntry
113406f32e7eSjoerg VectorSelectTbl[] = {
113506f32e7eSjoerg { ISD::SELECT, MVT::v16i1, MVT::v16i16, 16 },
113606f32e7eSjoerg { ISD::SELECT, MVT::v8i1, MVT::v8i32, 8 },
113706f32e7eSjoerg { ISD::SELECT, MVT::v16i1, MVT::v16i32, 16 },
113806f32e7eSjoerg { ISD::SELECT, MVT::v4i1, MVT::v4i64, 4 * AmortizationCost },
113906f32e7eSjoerg { ISD::SELECT, MVT::v8i1, MVT::v8i64, 8 * AmortizationCost },
114006f32e7eSjoerg { ISD::SELECT, MVT::v16i1, MVT::v16i64, 16 * AmortizationCost }
114106f32e7eSjoerg };
114206f32e7eSjoerg
114306f32e7eSjoerg EVT SelCondTy = TLI->getValueType(DL, CondTy);
114406f32e7eSjoerg EVT SelValTy = TLI->getValueType(DL, ValTy);
114506f32e7eSjoerg if (SelCondTy.isSimple() && SelValTy.isSimple()) {
114606f32e7eSjoerg if (const auto *Entry = ConvertCostTableLookup(VectorSelectTbl, ISD,
114706f32e7eSjoerg SelCondTy.getSimpleVT(),
114806f32e7eSjoerg SelValTy.getSimpleVT()))
114906f32e7eSjoerg return Entry->Cost;
115006f32e7eSjoerg }
115106f32e7eSjoerg }
1152*da58b97aSjoerg // The base case handles scalable vectors fine for now, since it treats the
1153*da58b97aSjoerg // cost as 1 * legalization cost.
1154*da58b97aSjoerg return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
115506f32e7eSjoerg }
115606f32e7eSjoerg
115706f32e7eSjoerg AArch64TTIImpl::TTI::MemCmpExpansionOptions
enableMemCmpExpansion(bool OptSize,bool IsZeroCmp) const115806f32e7eSjoerg AArch64TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
115906f32e7eSjoerg TTI::MemCmpExpansionOptions Options;
1160*da58b97aSjoerg if (ST->requiresStrictAlign()) {
1161*da58b97aSjoerg // TODO: Add cost modeling for strict align. Misaligned loads expand to
1162*da58b97aSjoerg // a bunch of instructions when strict align is enabled.
1163*da58b97aSjoerg return Options;
1164*da58b97aSjoerg }
1165*da58b97aSjoerg Options.AllowOverlappingLoads = true;
116606f32e7eSjoerg Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
116706f32e7eSjoerg Options.NumLoadsPerBlock = Options.MaxNumLoads;
116806f32e7eSjoerg // TODO: Though vector loads usually perform well on AArch64, in some targets
116906f32e7eSjoerg // they may wake up the FP unit, which raises the power consumption. Perhaps
117006f32e7eSjoerg // they could be used with no holds barred (-O3).
117106f32e7eSjoerg Options.LoadSizes = {8, 4, 2, 1};
117206f32e7eSjoerg return Options;
117306f32e7eSjoerg }
117406f32e7eSjoerg
1175*da58b97aSjoerg InstructionCost
getMaskedMemoryOpCost(unsigned Opcode,Type * Src,Align Alignment,unsigned AddressSpace,TTI::TargetCostKind CostKind)1176*da58b97aSjoerg AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
1177*da58b97aSjoerg Align Alignment, unsigned AddressSpace,
1178*da58b97aSjoerg TTI::TargetCostKind CostKind) {
1179*da58b97aSjoerg if (!isa<ScalableVectorType>(Src))
1180*da58b97aSjoerg return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1181*da58b97aSjoerg CostKind);
1182*da58b97aSjoerg auto LT = TLI->getTypeLegalizationCost(DL, Src);
1183*da58b97aSjoerg return LT.first * 2;
1184*da58b97aSjoerg }
1185*da58b97aSjoerg
getGatherScatterOpCost(unsigned Opcode,Type * DataTy,const Value * Ptr,bool VariableMask,Align Alignment,TTI::TargetCostKind CostKind,const Instruction * I)1186*da58b97aSjoerg InstructionCost AArch64TTIImpl::getGatherScatterOpCost(
1187*da58b97aSjoerg unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1188*da58b97aSjoerg Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
1189*da58b97aSjoerg
1190*da58b97aSjoerg if (!isa<ScalableVectorType>(DataTy))
1191*da58b97aSjoerg return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
1192*da58b97aSjoerg Alignment, CostKind, I);
1193*da58b97aSjoerg auto *VT = cast<VectorType>(DataTy);
1194*da58b97aSjoerg auto LT = TLI->getTypeLegalizationCost(DL, DataTy);
1195*da58b97aSjoerg ElementCount LegalVF = LT.second.getVectorElementCount();
1196*da58b97aSjoerg Optional<unsigned> MaxNumVScale = getMaxVScale();
1197*da58b97aSjoerg assert(MaxNumVScale && "Expected valid max vscale value");
1198*da58b97aSjoerg
1199*da58b97aSjoerg InstructionCost MemOpCost =
1200*da58b97aSjoerg getMemoryOpCost(Opcode, VT->getElementType(), Alignment, 0, CostKind, I);
1201*da58b97aSjoerg unsigned MaxNumElementsPerGather =
1202*da58b97aSjoerg MaxNumVScale.getValue() * LegalVF.getKnownMinValue();
1203*da58b97aSjoerg return LT.first * MaxNumElementsPerGather * MemOpCost;
1204*da58b97aSjoerg }
1205*da58b97aSjoerg
useNeonVector(const Type * Ty) const1206*da58b97aSjoerg bool AArch64TTIImpl::useNeonVector(const Type *Ty) const {
1207*da58b97aSjoerg return isa<FixedVectorType>(Ty) && !ST->useSVEForFixedLengthVectors();
1208*da58b97aSjoerg }
1209*da58b97aSjoerg
getMemoryOpCost(unsigned Opcode,Type * Ty,MaybeAlign Alignment,unsigned AddressSpace,TTI::TargetCostKind CostKind,const Instruction * I)1210*da58b97aSjoerg InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
1211*da58b97aSjoerg MaybeAlign Alignment,
1212*da58b97aSjoerg unsigned AddressSpace,
1213*da58b97aSjoerg TTI::TargetCostKind CostKind,
121406f32e7eSjoerg const Instruction *I) {
1215*da58b97aSjoerg // Type legalization can't handle structs
1216*da58b97aSjoerg if (TLI->getValueType(DL, Ty, true) == MVT::Other)
1217*da58b97aSjoerg return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace,
1218*da58b97aSjoerg CostKind);
1219*da58b97aSjoerg
122006f32e7eSjoerg auto LT = TLI->getTypeLegalizationCost(DL, Ty);
122106f32e7eSjoerg
1222*da58b97aSjoerg // TODO: consider latency as well for TCK_SizeAndLatency.
1223*da58b97aSjoerg if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency)
1224*da58b97aSjoerg return LT.first;
1225*da58b97aSjoerg
1226*da58b97aSjoerg if (CostKind != TTI::TCK_RecipThroughput)
1227*da58b97aSjoerg return 1;
1228*da58b97aSjoerg
122906f32e7eSjoerg if (ST->isMisaligned128StoreSlow() && Opcode == Instruction::Store &&
123006f32e7eSjoerg LT.second.is128BitVector() && (!Alignment || *Alignment < Align(16))) {
123106f32e7eSjoerg // Unaligned stores are extremely inefficient. We don't split all
123206f32e7eSjoerg // unaligned 128-bit stores because the negative impact that has shown in
123306f32e7eSjoerg // practice on inlined block copy code.
123406f32e7eSjoerg // We make such stores expensive so that we will only vectorize if there
123506f32e7eSjoerg // are 6 other instructions getting vectorized.
123606f32e7eSjoerg const int AmortizationCost = 6;
123706f32e7eSjoerg
123806f32e7eSjoerg return LT.first * 2 * AmortizationCost;
123906f32e7eSjoerg }
124006f32e7eSjoerg
1241*da58b97aSjoerg if (useNeonVector(Ty) &&
1242*da58b97aSjoerg cast<VectorType>(Ty)->getElementType()->isIntegerTy(8)) {
124306f32e7eSjoerg unsigned ProfitableNumElements;
124406f32e7eSjoerg if (Opcode == Instruction::Store)
124506f32e7eSjoerg // We use a custom trunc store lowering so v.4b should be profitable.
124606f32e7eSjoerg ProfitableNumElements = 4;
124706f32e7eSjoerg else
124806f32e7eSjoerg // We scalarize the loads because there is not v.4b register and we
124906f32e7eSjoerg // have to promote the elements to v.2.
125006f32e7eSjoerg ProfitableNumElements = 8;
125106f32e7eSjoerg
1252*da58b97aSjoerg if (cast<FixedVectorType>(Ty)->getNumElements() < ProfitableNumElements) {
1253*da58b97aSjoerg unsigned NumVecElts = cast<FixedVectorType>(Ty)->getNumElements();
125406f32e7eSjoerg unsigned NumVectorizableInstsToAmortize = NumVecElts * 2;
125506f32e7eSjoerg // We generate 2 instructions per vector element.
125606f32e7eSjoerg return NumVectorizableInstsToAmortize * NumVecElts * 2;
125706f32e7eSjoerg }
125806f32e7eSjoerg }
125906f32e7eSjoerg
126006f32e7eSjoerg return LT.first;
126106f32e7eSjoerg }
126206f32e7eSjoerg
getInterleavedMemoryOpCost(unsigned Opcode,Type * VecTy,unsigned Factor,ArrayRef<unsigned> Indices,Align Alignment,unsigned AddressSpace,TTI::TargetCostKind CostKind,bool UseMaskForCond,bool UseMaskForGaps)1263*da58b97aSjoerg InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
1264*da58b97aSjoerg unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1265*da58b97aSjoerg Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1266*da58b97aSjoerg bool UseMaskForCond, bool UseMaskForGaps) {
126706f32e7eSjoerg assert(Factor >= 2 && "Invalid interleave factor");
1268*da58b97aSjoerg auto *VecVTy = cast<FixedVectorType>(VecTy);
126906f32e7eSjoerg
127006f32e7eSjoerg if (!UseMaskForCond && !UseMaskForGaps &&
127106f32e7eSjoerg Factor <= TLI->getMaxSupportedInterleaveFactor()) {
1272*da58b97aSjoerg unsigned NumElts = VecVTy->getNumElements();
1273*da58b97aSjoerg auto *SubVecTy =
1274*da58b97aSjoerg FixedVectorType::get(VecTy->getScalarType(), NumElts / Factor);
127506f32e7eSjoerg
127606f32e7eSjoerg // ldN/stN only support legal vector types of size 64 or 128 in bits.
127706f32e7eSjoerg // Accesses having vector types that are a multiple of 128 bits can be
127806f32e7eSjoerg // matched to more than one ldN/stN instruction.
127906f32e7eSjoerg if (NumElts % Factor == 0 &&
128006f32e7eSjoerg TLI->isLegalInterleavedAccessType(SubVecTy, DL))
128106f32e7eSjoerg return Factor * TLI->getNumInterleavedAccesses(SubVecTy, DL);
128206f32e7eSjoerg }
128306f32e7eSjoerg
128406f32e7eSjoerg return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1285*da58b97aSjoerg Alignment, AddressSpace, CostKind,
128606f32e7eSjoerg UseMaskForCond, UseMaskForGaps);
128706f32e7eSjoerg }
128806f32e7eSjoerg
1289*da58b97aSjoerg InstructionCost
getCostOfKeepingLiveOverCall(ArrayRef<Type * > Tys)1290*da58b97aSjoerg AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) {
1291*da58b97aSjoerg InstructionCost Cost = 0;
1292*da58b97aSjoerg TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
129306f32e7eSjoerg for (auto *I : Tys) {
129406f32e7eSjoerg if (!I->isVectorTy())
129506f32e7eSjoerg continue;
1296*da58b97aSjoerg if (I->getScalarSizeInBits() * cast<FixedVectorType>(I)->getNumElements() ==
1297*da58b97aSjoerg 128)
1298*da58b97aSjoerg Cost += getMemoryOpCost(Instruction::Store, I, Align(128), 0, CostKind) +
1299*da58b97aSjoerg getMemoryOpCost(Instruction::Load, I, Align(128), 0, CostKind);
130006f32e7eSjoerg }
130106f32e7eSjoerg return Cost;
130206f32e7eSjoerg }
130306f32e7eSjoerg
getMaxInterleaveFactor(unsigned VF)130406f32e7eSjoerg unsigned AArch64TTIImpl::getMaxInterleaveFactor(unsigned VF) {
130506f32e7eSjoerg return ST->getMaxInterleaveFactor();
130606f32e7eSjoerg }
130706f32e7eSjoerg
130806f32e7eSjoerg // For Falkor, we want to avoid having too many strided loads in a loop since
130906f32e7eSjoerg // that can exhaust the HW prefetcher resources. We adjust the unroller
131006f32e7eSjoerg // MaxCount preference below to attempt to ensure unrolling doesn't create too
131106f32e7eSjoerg // many strided loads.
131206f32e7eSjoerg static void
getFalkorUnrollingPreferences(Loop * L,ScalarEvolution & SE,TargetTransformInfo::UnrollingPreferences & UP)131306f32e7eSjoerg getFalkorUnrollingPreferences(Loop *L, ScalarEvolution &SE,
131406f32e7eSjoerg TargetTransformInfo::UnrollingPreferences &UP) {
131506f32e7eSjoerg enum { MaxStridedLoads = 7 };
131606f32e7eSjoerg auto countStridedLoads = [](Loop *L, ScalarEvolution &SE) {
131706f32e7eSjoerg int StridedLoads = 0;
131806f32e7eSjoerg // FIXME? We could make this more precise by looking at the CFG and
131906f32e7eSjoerg // e.g. not counting loads in each side of an if-then-else diamond.
132006f32e7eSjoerg for (const auto BB : L->blocks()) {
132106f32e7eSjoerg for (auto &I : *BB) {
132206f32e7eSjoerg LoadInst *LMemI = dyn_cast<LoadInst>(&I);
132306f32e7eSjoerg if (!LMemI)
132406f32e7eSjoerg continue;
132506f32e7eSjoerg
132606f32e7eSjoerg Value *PtrValue = LMemI->getPointerOperand();
132706f32e7eSjoerg if (L->isLoopInvariant(PtrValue))
132806f32e7eSjoerg continue;
132906f32e7eSjoerg
133006f32e7eSjoerg const SCEV *LSCEV = SE.getSCEV(PtrValue);
133106f32e7eSjoerg const SCEVAddRecExpr *LSCEVAddRec = dyn_cast<SCEVAddRecExpr>(LSCEV);
133206f32e7eSjoerg if (!LSCEVAddRec || !LSCEVAddRec->isAffine())
133306f32e7eSjoerg continue;
133406f32e7eSjoerg
133506f32e7eSjoerg // FIXME? We could take pairing of unrolled load copies into account
133606f32e7eSjoerg // by looking at the AddRec, but we would probably have to limit this
133706f32e7eSjoerg // to loops with no stores or other memory optimization barriers.
133806f32e7eSjoerg ++StridedLoads;
133906f32e7eSjoerg // We've seen enough strided loads that seeing more won't make a
134006f32e7eSjoerg // difference.
134106f32e7eSjoerg if (StridedLoads > MaxStridedLoads / 2)
134206f32e7eSjoerg return StridedLoads;
134306f32e7eSjoerg }
134406f32e7eSjoerg }
134506f32e7eSjoerg return StridedLoads;
134606f32e7eSjoerg };
134706f32e7eSjoerg
134806f32e7eSjoerg int StridedLoads = countStridedLoads(L, SE);
134906f32e7eSjoerg LLVM_DEBUG(dbgs() << "falkor-hwpf: detected " << StridedLoads
135006f32e7eSjoerg << " strided loads\n");
135106f32e7eSjoerg // Pick the largest power of 2 unroll count that won't result in too many
135206f32e7eSjoerg // strided loads.
135306f32e7eSjoerg if (StridedLoads) {
135406f32e7eSjoerg UP.MaxCount = 1 << Log2_32(MaxStridedLoads / StridedLoads);
135506f32e7eSjoerg LLVM_DEBUG(dbgs() << "falkor-hwpf: setting unroll MaxCount to "
135606f32e7eSjoerg << UP.MaxCount << '\n');
135706f32e7eSjoerg }
135806f32e7eSjoerg }
135906f32e7eSjoerg
getUnrollingPreferences(Loop * L,ScalarEvolution & SE,TTI::UnrollingPreferences & UP)136006f32e7eSjoerg void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
136106f32e7eSjoerg TTI::UnrollingPreferences &UP) {
136206f32e7eSjoerg // Enable partial unrolling and runtime unrolling.
136306f32e7eSjoerg BaseT::getUnrollingPreferences(L, SE, UP);
136406f32e7eSjoerg
136506f32e7eSjoerg // For inner loop, it is more likely to be a hot one, and the runtime check
136606f32e7eSjoerg // can be promoted out from LICM pass, so the overhead is less, let's try
136706f32e7eSjoerg // a larger threshold to unroll more loops.
136806f32e7eSjoerg if (L->getLoopDepth() > 1)
136906f32e7eSjoerg UP.PartialThreshold *= 2;
137006f32e7eSjoerg
137106f32e7eSjoerg // Disable partial & runtime unrolling on -Os.
137206f32e7eSjoerg UP.PartialOptSizeThreshold = 0;
137306f32e7eSjoerg
137406f32e7eSjoerg if (ST->getProcFamily() == AArch64Subtarget::Falkor &&
137506f32e7eSjoerg EnableFalkorHWPFUnrollFix)
137606f32e7eSjoerg getFalkorUnrollingPreferences(L, SE, UP);
1377*da58b97aSjoerg
1378*da58b97aSjoerg // Scan the loop: don't unroll loops with calls as this could prevent
1379*da58b97aSjoerg // inlining. Don't unroll vector loops either, as they don't benefit much from
1380*da58b97aSjoerg // unrolling.
1381*da58b97aSjoerg for (auto *BB : L->getBlocks()) {
1382*da58b97aSjoerg for (auto &I : *BB) {
1383*da58b97aSjoerg // Don't unroll vectorised loop.
1384*da58b97aSjoerg if (I.getType()->isVectorTy())
1385*da58b97aSjoerg return;
1386*da58b97aSjoerg
1387*da58b97aSjoerg if (isa<CallInst>(I) || isa<InvokeInst>(I)) {
1388*da58b97aSjoerg if (const Function *F = cast<CallBase>(I).getCalledFunction()) {
1389*da58b97aSjoerg if (!isLoweredToCall(F))
1390*da58b97aSjoerg continue;
1391*da58b97aSjoerg }
1392*da58b97aSjoerg return;
1393*da58b97aSjoerg }
1394*da58b97aSjoerg }
1395*da58b97aSjoerg }
1396*da58b97aSjoerg
1397*da58b97aSjoerg // Enable runtime unrolling for in-order models
1398*da58b97aSjoerg // If mcpu is omitted, getProcFamily() returns AArch64Subtarget::Others, so by
1399*da58b97aSjoerg // checking for that case, we can ensure that the default behaviour is
1400*da58b97aSjoerg // unchanged
1401*da58b97aSjoerg if (ST->getProcFamily() != AArch64Subtarget::Others &&
1402*da58b97aSjoerg !ST->getSchedModel().isOutOfOrder()) {
1403*da58b97aSjoerg UP.Runtime = true;
1404*da58b97aSjoerg UP.Partial = true;
1405*da58b97aSjoerg UP.UpperBound = true;
1406*da58b97aSjoerg UP.UnrollRemainder = true;
1407*da58b97aSjoerg UP.DefaultUnrollRuntimeCount = 4;
1408*da58b97aSjoerg }
1409*da58b97aSjoerg }
1410*da58b97aSjoerg
getPeelingPreferences(Loop * L,ScalarEvolution & SE,TTI::PeelingPreferences & PP)1411*da58b97aSjoerg void AArch64TTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
1412*da58b97aSjoerg TTI::PeelingPreferences &PP) {
1413*da58b97aSjoerg BaseT::getPeelingPreferences(L, SE, PP);
141406f32e7eSjoerg }
141506f32e7eSjoerg
getOrCreateResultFromMemIntrinsic(IntrinsicInst * Inst,Type * ExpectedType)141606f32e7eSjoerg Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
141706f32e7eSjoerg Type *ExpectedType) {
141806f32e7eSjoerg switch (Inst->getIntrinsicID()) {
141906f32e7eSjoerg default:
142006f32e7eSjoerg return nullptr;
142106f32e7eSjoerg case Intrinsic::aarch64_neon_st2:
142206f32e7eSjoerg case Intrinsic::aarch64_neon_st3:
142306f32e7eSjoerg case Intrinsic::aarch64_neon_st4: {
142406f32e7eSjoerg // Create a struct type
142506f32e7eSjoerg StructType *ST = dyn_cast<StructType>(ExpectedType);
142606f32e7eSjoerg if (!ST)
142706f32e7eSjoerg return nullptr;
142806f32e7eSjoerg unsigned NumElts = Inst->getNumArgOperands() - 1;
142906f32e7eSjoerg if (ST->getNumElements() != NumElts)
143006f32e7eSjoerg return nullptr;
143106f32e7eSjoerg for (unsigned i = 0, e = NumElts; i != e; ++i) {
143206f32e7eSjoerg if (Inst->getArgOperand(i)->getType() != ST->getElementType(i))
143306f32e7eSjoerg return nullptr;
143406f32e7eSjoerg }
143506f32e7eSjoerg Value *Res = UndefValue::get(ExpectedType);
143606f32e7eSjoerg IRBuilder<> Builder(Inst);
143706f32e7eSjoerg for (unsigned i = 0, e = NumElts; i != e; ++i) {
143806f32e7eSjoerg Value *L = Inst->getArgOperand(i);
143906f32e7eSjoerg Res = Builder.CreateInsertValue(Res, L, i);
144006f32e7eSjoerg }
144106f32e7eSjoerg return Res;
144206f32e7eSjoerg }
144306f32e7eSjoerg case Intrinsic::aarch64_neon_ld2:
144406f32e7eSjoerg case Intrinsic::aarch64_neon_ld3:
144506f32e7eSjoerg case Intrinsic::aarch64_neon_ld4:
144606f32e7eSjoerg if (Inst->getType() == ExpectedType)
144706f32e7eSjoerg return Inst;
144806f32e7eSjoerg return nullptr;
144906f32e7eSjoerg }
145006f32e7eSjoerg }
145106f32e7eSjoerg
getTgtMemIntrinsic(IntrinsicInst * Inst,MemIntrinsicInfo & Info)145206f32e7eSjoerg bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
145306f32e7eSjoerg MemIntrinsicInfo &Info) {
145406f32e7eSjoerg switch (Inst->getIntrinsicID()) {
145506f32e7eSjoerg default:
145606f32e7eSjoerg break;
145706f32e7eSjoerg case Intrinsic::aarch64_neon_ld2:
145806f32e7eSjoerg case Intrinsic::aarch64_neon_ld3:
145906f32e7eSjoerg case Intrinsic::aarch64_neon_ld4:
146006f32e7eSjoerg Info.ReadMem = true;
146106f32e7eSjoerg Info.WriteMem = false;
146206f32e7eSjoerg Info.PtrVal = Inst->getArgOperand(0);
146306f32e7eSjoerg break;
146406f32e7eSjoerg case Intrinsic::aarch64_neon_st2:
146506f32e7eSjoerg case Intrinsic::aarch64_neon_st3:
146606f32e7eSjoerg case Intrinsic::aarch64_neon_st4:
146706f32e7eSjoerg Info.ReadMem = false;
146806f32e7eSjoerg Info.WriteMem = true;
146906f32e7eSjoerg Info.PtrVal = Inst->getArgOperand(Inst->getNumArgOperands() - 1);
147006f32e7eSjoerg break;
147106f32e7eSjoerg }
147206f32e7eSjoerg
147306f32e7eSjoerg switch (Inst->getIntrinsicID()) {
147406f32e7eSjoerg default:
147506f32e7eSjoerg return false;
147606f32e7eSjoerg case Intrinsic::aarch64_neon_ld2:
147706f32e7eSjoerg case Intrinsic::aarch64_neon_st2:
147806f32e7eSjoerg Info.MatchingId = VECTOR_LDST_TWO_ELEMENTS;
147906f32e7eSjoerg break;
148006f32e7eSjoerg case Intrinsic::aarch64_neon_ld3:
148106f32e7eSjoerg case Intrinsic::aarch64_neon_st3:
148206f32e7eSjoerg Info.MatchingId = VECTOR_LDST_THREE_ELEMENTS;
148306f32e7eSjoerg break;
148406f32e7eSjoerg case Intrinsic::aarch64_neon_ld4:
148506f32e7eSjoerg case Intrinsic::aarch64_neon_st4:
148606f32e7eSjoerg Info.MatchingId = VECTOR_LDST_FOUR_ELEMENTS;
148706f32e7eSjoerg break;
148806f32e7eSjoerg }
148906f32e7eSjoerg return true;
149006f32e7eSjoerg }
149106f32e7eSjoerg
149206f32e7eSjoerg /// See if \p I should be considered for address type promotion. We check if \p
149306f32e7eSjoerg /// I is a sext with right type and used in memory accesses. If it used in a
149406f32e7eSjoerg /// "complex" getelementptr, we allow it to be promoted without finding other
149506f32e7eSjoerg /// sext instructions that sign extended the same initial value. A getelementptr
149606f32e7eSjoerg /// is considered as "complex" if it has more than 2 operands.
shouldConsiderAddressTypePromotion(const Instruction & I,bool & AllowPromotionWithoutCommonHeader)149706f32e7eSjoerg bool AArch64TTIImpl::shouldConsiderAddressTypePromotion(
149806f32e7eSjoerg const Instruction &I, bool &AllowPromotionWithoutCommonHeader) {
149906f32e7eSjoerg bool Considerable = false;
150006f32e7eSjoerg AllowPromotionWithoutCommonHeader = false;
150106f32e7eSjoerg if (!isa<SExtInst>(&I))
150206f32e7eSjoerg return false;
150306f32e7eSjoerg Type *ConsideredSExtType =
150406f32e7eSjoerg Type::getInt64Ty(I.getParent()->getParent()->getContext());
150506f32e7eSjoerg if (I.getType() != ConsideredSExtType)
150606f32e7eSjoerg return false;
150706f32e7eSjoerg // See if the sext is the one with the right type and used in at least one
150806f32e7eSjoerg // GetElementPtrInst.
150906f32e7eSjoerg for (const User *U : I.users()) {
151006f32e7eSjoerg if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(U)) {
151106f32e7eSjoerg Considerable = true;
151206f32e7eSjoerg // A getelementptr is considered as "complex" if it has more than 2
151306f32e7eSjoerg // operands. We will promote a SExt used in such complex GEP as we
151406f32e7eSjoerg // expect some computation to be merged if they are done on 64 bits.
151506f32e7eSjoerg if (GEPInst->getNumOperands() > 2) {
151606f32e7eSjoerg AllowPromotionWithoutCommonHeader = true;
151706f32e7eSjoerg break;
151806f32e7eSjoerg }
151906f32e7eSjoerg }
152006f32e7eSjoerg }
152106f32e7eSjoerg return Considerable;
152206f32e7eSjoerg }
152306f32e7eSjoerg
isLegalToVectorizeReduction(RecurrenceDescriptor RdxDesc,ElementCount VF) const1524*da58b97aSjoerg bool AArch64TTIImpl::isLegalToVectorizeReduction(RecurrenceDescriptor RdxDesc,
1525*da58b97aSjoerg ElementCount VF) const {
1526*da58b97aSjoerg if (!VF.isScalable())
1527*da58b97aSjoerg return true;
1528*da58b97aSjoerg
1529*da58b97aSjoerg Type *Ty = RdxDesc.getRecurrenceType();
1530*da58b97aSjoerg if (Ty->isBFloatTy() || !isLegalElementTypeForSVE(Ty))
153106f32e7eSjoerg return false;
1532*da58b97aSjoerg
1533*da58b97aSjoerg switch (RdxDesc.getRecurrenceKind()) {
1534*da58b97aSjoerg case RecurKind::Add:
1535*da58b97aSjoerg case RecurKind::FAdd:
1536*da58b97aSjoerg case RecurKind::And:
1537*da58b97aSjoerg case RecurKind::Or:
1538*da58b97aSjoerg case RecurKind::Xor:
1539*da58b97aSjoerg case RecurKind::SMin:
1540*da58b97aSjoerg case RecurKind::SMax:
1541*da58b97aSjoerg case RecurKind::UMin:
1542*da58b97aSjoerg case RecurKind::UMax:
1543*da58b97aSjoerg case RecurKind::FMin:
1544*da58b97aSjoerg case RecurKind::FMax:
1545*da58b97aSjoerg return true;
154606f32e7eSjoerg default:
154706f32e7eSjoerg return false;
154806f32e7eSjoerg }
1549*da58b97aSjoerg }
155006f32e7eSjoerg
1551*da58b97aSjoerg InstructionCost
getMinMaxReductionCost(VectorType * Ty,VectorType * CondTy,bool IsPairwise,bool IsUnsigned,TTI::TargetCostKind CostKind)1552*da58b97aSjoerg AArch64TTIImpl::getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy,
1553*da58b97aSjoerg bool IsPairwise, bool IsUnsigned,
1554*da58b97aSjoerg TTI::TargetCostKind CostKind) {
1555*da58b97aSjoerg if (!isa<ScalableVectorType>(Ty))
1556*da58b97aSjoerg return BaseT::getMinMaxReductionCost(Ty, CondTy, IsPairwise, IsUnsigned,
1557*da58b97aSjoerg CostKind);
1558*da58b97aSjoerg assert((isa<ScalableVectorType>(Ty) && isa<ScalableVectorType>(CondTy)) &&
1559*da58b97aSjoerg "Both vector needs to be scalable");
156006f32e7eSjoerg
1561*da58b97aSjoerg std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
1562*da58b97aSjoerg InstructionCost LegalizationCost = 0;
1563*da58b97aSjoerg if (LT.first > 1) {
1564*da58b97aSjoerg Type *LegalVTy = EVT(LT.second).getTypeForEVT(Ty->getContext());
1565*da58b97aSjoerg unsigned CmpOpcode =
1566*da58b97aSjoerg Ty->isFPOrFPVectorTy() ? Instruction::FCmp : Instruction::ICmp;
1567*da58b97aSjoerg LegalizationCost =
1568*da58b97aSjoerg getCmpSelInstrCost(CmpOpcode, LegalVTy, LegalVTy,
1569*da58b97aSjoerg CmpInst::BAD_ICMP_PREDICATE, CostKind) +
1570*da58b97aSjoerg getCmpSelInstrCost(Instruction::Select, LegalVTy, LegalVTy,
1571*da58b97aSjoerg CmpInst::BAD_ICMP_PREDICATE, CostKind);
1572*da58b97aSjoerg LegalizationCost *= LT.first - 1;
1573*da58b97aSjoerg }
1574*da58b97aSjoerg
1575*da58b97aSjoerg return LegalizationCost + /*Cost of horizontal reduction*/ 2;
1576*da58b97aSjoerg }
1577*da58b97aSjoerg
getArithmeticReductionCostSVE(unsigned Opcode,VectorType * ValTy,bool IsPairwise,TTI::TargetCostKind CostKind)1578*da58b97aSjoerg InstructionCost AArch64TTIImpl::getArithmeticReductionCostSVE(
1579*da58b97aSjoerg unsigned Opcode, VectorType *ValTy, bool IsPairwise,
1580*da58b97aSjoerg TTI::TargetCostKind CostKind) {
1581*da58b97aSjoerg assert(!IsPairwise && "Cannot be pair wise to continue");
1582*da58b97aSjoerg
1583*da58b97aSjoerg std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, ValTy);
1584*da58b97aSjoerg InstructionCost LegalizationCost = 0;
1585*da58b97aSjoerg if (LT.first > 1) {
1586*da58b97aSjoerg Type *LegalVTy = EVT(LT.second).getTypeForEVT(ValTy->getContext());
1587*da58b97aSjoerg LegalizationCost = getArithmeticInstrCost(Opcode, LegalVTy, CostKind);
1588*da58b97aSjoerg LegalizationCost *= LT.first - 1;
1589*da58b97aSjoerg }
1590*da58b97aSjoerg
1591*da58b97aSjoerg int ISD = TLI->InstructionOpcodeToISD(Opcode);
1592*da58b97aSjoerg assert(ISD && "Invalid opcode");
1593*da58b97aSjoerg // Add the final reduction cost for the legal horizontal reduction
1594*da58b97aSjoerg switch (ISD) {
1595*da58b97aSjoerg case ISD::ADD:
1596*da58b97aSjoerg case ISD::AND:
1597*da58b97aSjoerg case ISD::OR:
1598*da58b97aSjoerg case ISD::XOR:
1599*da58b97aSjoerg case ISD::FADD:
1600*da58b97aSjoerg return LegalizationCost + 2;
1601*da58b97aSjoerg default:
1602*da58b97aSjoerg return InstructionCost::getInvalid();
1603*da58b97aSjoerg }
1604*da58b97aSjoerg }
1605*da58b97aSjoerg
1606*da58b97aSjoerg InstructionCost
getArithmeticReductionCost(unsigned Opcode,VectorType * ValTy,bool IsPairwiseForm,TTI::TargetCostKind CostKind)1607*da58b97aSjoerg AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
1608*da58b97aSjoerg bool IsPairwiseForm,
1609*da58b97aSjoerg TTI::TargetCostKind CostKind) {
1610*da58b97aSjoerg
1611*da58b97aSjoerg if (isa<ScalableVectorType>(ValTy))
1612*da58b97aSjoerg return getArithmeticReductionCostSVE(Opcode, ValTy, IsPairwiseForm,
1613*da58b97aSjoerg CostKind);
161406f32e7eSjoerg if (IsPairwiseForm)
1615*da58b97aSjoerg return BaseT::getArithmeticReductionCost(Opcode, ValTy, IsPairwiseForm,
1616*da58b97aSjoerg CostKind);
161706f32e7eSjoerg
1618*da58b97aSjoerg std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, ValTy);
161906f32e7eSjoerg MVT MTy = LT.second;
162006f32e7eSjoerg int ISD = TLI->InstructionOpcodeToISD(Opcode);
162106f32e7eSjoerg assert(ISD && "Invalid opcode");
162206f32e7eSjoerg
162306f32e7eSjoerg // Horizontal adds can use the 'addv' instruction. We model the cost of these
162406f32e7eSjoerg // instructions as normal vector adds. This is the only arithmetic vector
162506f32e7eSjoerg // reduction operation for which we have an instruction.
162606f32e7eSjoerg static const CostTblEntry CostTblNoPairwise[]{
162706f32e7eSjoerg {ISD::ADD, MVT::v8i8, 1},
162806f32e7eSjoerg {ISD::ADD, MVT::v16i8, 1},
162906f32e7eSjoerg {ISD::ADD, MVT::v4i16, 1},
163006f32e7eSjoerg {ISD::ADD, MVT::v8i16, 1},
163106f32e7eSjoerg {ISD::ADD, MVT::v4i32, 1},
163206f32e7eSjoerg };
163306f32e7eSjoerg
163406f32e7eSjoerg if (const auto *Entry = CostTableLookup(CostTblNoPairwise, ISD, MTy))
163506f32e7eSjoerg return LT.first * Entry->Cost;
163606f32e7eSjoerg
1637*da58b97aSjoerg return BaseT::getArithmeticReductionCost(Opcode, ValTy, IsPairwiseForm,
1638*da58b97aSjoerg CostKind);
163906f32e7eSjoerg }
164006f32e7eSjoerg
getShuffleCost(TTI::ShuffleKind Kind,VectorType * Tp,ArrayRef<int> Mask,int Index,VectorType * SubTp)1641*da58b97aSjoerg InstructionCost AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
1642*da58b97aSjoerg VectorType *Tp,
1643*da58b97aSjoerg ArrayRef<int> Mask, int Index,
1644*da58b97aSjoerg VectorType *SubTp) {
1645*da58b97aSjoerg Kind = improveShuffleKindFromMask(Kind, Mask);
164606f32e7eSjoerg if (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Transpose ||
1647*da58b97aSjoerg Kind == TTI::SK_Select || Kind == TTI::SK_PermuteSingleSrc ||
1648*da58b97aSjoerg Kind == TTI::SK_Reverse) {
164906f32e7eSjoerg static const CostTblEntry ShuffleTbl[] = {
165006f32e7eSjoerg // Broadcast shuffle kinds can be performed with 'dup'.
165106f32e7eSjoerg { TTI::SK_Broadcast, MVT::v8i8, 1 },
165206f32e7eSjoerg { TTI::SK_Broadcast, MVT::v16i8, 1 },
165306f32e7eSjoerg { TTI::SK_Broadcast, MVT::v4i16, 1 },
165406f32e7eSjoerg { TTI::SK_Broadcast, MVT::v8i16, 1 },
165506f32e7eSjoerg { TTI::SK_Broadcast, MVT::v2i32, 1 },
165606f32e7eSjoerg { TTI::SK_Broadcast, MVT::v4i32, 1 },
165706f32e7eSjoerg { TTI::SK_Broadcast, MVT::v2i64, 1 },
165806f32e7eSjoerg { TTI::SK_Broadcast, MVT::v2f32, 1 },
165906f32e7eSjoerg { TTI::SK_Broadcast, MVT::v4f32, 1 },
166006f32e7eSjoerg { TTI::SK_Broadcast, MVT::v2f64, 1 },
166106f32e7eSjoerg // Transpose shuffle kinds can be performed with 'trn1/trn2' and
166206f32e7eSjoerg // 'zip1/zip2' instructions.
166306f32e7eSjoerg { TTI::SK_Transpose, MVT::v8i8, 1 },
166406f32e7eSjoerg { TTI::SK_Transpose, MVT::v16i8, 1 },
166506f32e7eSjoerg { TTI::SK_Transpose, MVT::v4i16, 1 },
166606f32e7eSjoerg { TTI::SK_Transpose, MVT::v8i16, 1 },
166706f32e7eSjoerg { TTI::SK_Transpose, MVT::v2i32, 1 },
166806f32e7eSjoerg { TTI::SK_Transpose, MVT::v4i32, 1 },
166906f32e7eSjoerg { TTI::SK_Transpose, MVT::v2i64, 1 },
167006f32e7eSjoerg { TTI::SK_Transpose, MVT::v2f32, 1 },
167106f32e7eSjoerg { TTI::SK_Transpose, MVT::v4f32, 1 },
167206f32e7eSjoerg { TTI::SK_Transpose, MVT::v2f64, 1 },
167306f32e7eSjoerg // Select shuffle kinds.
167406f32e7eSjoerg // TODO: handle vXi8/vXi16.
167506f32e7eSjoerg { TTI::SK_Select, MVT::v2i32, 1 }, // mov.
167606f32e7eSjoerg { TTI::SK_Select, MVT::v4i32, 2 }, // rev+trn (or similar).
167706f32e7eSjoerg { TTI::SK_Select, MVT::v2i64, 1 }, // mov.
167806f32e7eSjoerg { TTI::SK_Select, MVT::v2f32, 1 }, // mov.
167906f32e7eSjoerg { TTI::SK_Select, MVT::v4f32, 2 }, // rev+trn (or similar).
168006f32e7eSjoerg { TTI::SK_Select, MVT::v2f64, 1 }, // mov.
168106f32e7eSjoerg // PermuteSingleSrc shuffle kinds.
168206f32e7eSjoerg // TODO: handle vXi8/vXi16.
168306f32e7eSjoerg { TTI::SK_PermuteSingleSrc, MVT::v2i32, 1 }, // mov.
168406f32e7eSjoerg { TTI::SK_PermuteSingleSrc, MVT::v4i32, 3 }, // perfectshuffle worst case.
168506f32e7eSjoerg { TTI::SK_PermuteSingleSrc, MVT::v2i64, 1 }, // mov.
168606f32e7eSjoerg { TTI::SK_PermuteSingleSrc, MVT::v2f32, 1 }, // mov.
168706f32e7eSjoerg { TTI::SK_PermuteSingleSrc, MVT::v4f32, 3 }, // perfectshuffle worst case.
168806f32e7eSjoerg { TTI::SK_PermuteSingleSrc, MVT::v2f64, 1 }, // mov.
1689*da58b97aSjoerg // Reverse can be lowered with `rev`.
1690*da58b97aSjoerg { TTI::SK_Reverse, MVT::v2i32, 1 }, // mov.
1691*da58b97aSjoerg { TTI::SK_Reverse, MVT::v4i32, 2 }, // REV64; EXT
1692*da58b97aSjoerg { TTI::SK_Reverse, MVT::v2i64, 1 }, // mov.
1693*da58b97aSjoerg { TTI::SK_Reverse, MVT::v2f32, 1 }, // mov.
1694*da58b97aSjoerg { TTI::SK_Reverse, MVT::v4f32, 2 }, // REV64; EXT
1695*da58b97aSjoerg { TTI::SK_Reverse, MVT::v2f64, 1 }, // mov.
1696*da58b97aSjoerg // Broadcast shuffle kinds for scalable vectors
1697*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv16i8, 1 },
1698*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv8i16, 1 },
1699*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv4i32, 1 },
1700*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv2i64, 1 },
1701*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv2f16, 1 },
1702*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv4f16, 1 },
1703*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv8f16, 1 },
1704*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv2bf16, 1 },
1705*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv4bf16, 1 },
1706*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv8bf16, 1 },
1707*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv2f32, 1 },
1708*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv4f32, 1 },
1709*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv2f64, 1 },
1710*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv16i1, 1 },
1711*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv8i1, 1 },
1712*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv4i1, 1 },
1713*da58b97aSjoerg { TTI::SK_Broadcast, MVT::nxv2i1, 1 },
1714*da58b97aSjoerg // Handle the cases for vector.reverse with scalable vectors
1715*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv16i8, 1 },
1716*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv8i16, 1 },
1717*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv4i32, 1 },
1718*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv2i64, 1 },
1719*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv2f16, 1 },
1720*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv4f16, 1 },
1721*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv8f16, 1 },
1722*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv2bf16, 1 },
1723*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv4bf16, 1 },
1724*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv8bf16, 1 },
1725*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv2f32, 1 },
1726*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv4f32, 1 },
1727*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv2f64, 1 },
1728*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv16i1, 1 },
1729*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv8i1, 1 },
1730*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv4i1, 1 },
1731*da58b97aSjoerg { TTI::SK_Reverse, MVT::nxv2i1, 1 },
173206f32e7eSjoerg };
1733*da58b97aSjoerg std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
173406f32e7eSjoerg if (const auto *Entry = CostTableLookup(ShuffleTbl, Kind, LT.second))
173506f32e7eSjoerg return LT.first * Entry->Cost;
173606f32e7eSjoerg }
173706f32e7eSjoerg
1738*da58b97aSjoerg return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp);
173906f32e7eSjoerg }
1740