1 //===- AArch64TargetTransformInfo.h - AArch64 specific TTI ------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file a TargetTransformInfo::Concept conforming object specific to the
10 /// AArch64 target machine. It uses the target's detailed information to
11 /// provide more precise answers to certain TTI queries, while letting the
12 /// target independent and default TTI implementations handle the rest.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64TARGETTRANSFORMINFO_H
17 #define LLVM_LIB_TARGET_AARCH64_AARCH64TARGETTRANSFORMINFO_H
18 
19 #include "AArch64.h"
20 #include "AArch64Subtarget.h"
21 #include "AArch64TargetMachine.h"
22 #include "llvm/ADT/ArrayRef.h"
23 #include "llvm/Analysis/TargetTransformInfo.h"
24 #include "llvm/CodeGen/BasicTTIImpl.h"
25 #include "llvm/IR/Function.h"
26 #include "llvm/IR/Intrinsics.h"
27 #include <cstdint>
28 
29 namespace llvm {
30 
31 class APInt;
32 class Instruction;
33 class IntrinsicInst;
34 class Loop;
35 class SCEV;
36 class ScalarEvolution;
37 class Type;
38 class Value;
39 class VectorType;
40 
41 class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
42   using BaseT = BasicTTIImplBase<AArch64TTIImpl>;
43   using TTI = TargetTransformInfo;
44 
45   friend BaseT;
46 
47   const AArch64Subtarget *ST;
48   const AArch64TargetLowering *TLI;
49 
50   const AArch64Subtarget *getST() const { return ST; }
51   const AArch64TargetLowering *getTLI() const { return TLI; }
52 
53   enum MemIntrinsicType {
54     VECTOR_LDST_TWO_ELEMENTS,
55     VECTOR_LDST_THREE_ELEMENTS,
56     VECTOR_LDST_FOUR_ELEMENTS
57   };
58 
59   bool isWideningInstruction(Type *Ty, unsigned Opcode,
60                              ArrayRef<const Value *> Args);
61 
62 public:
63   explicit AArch64TTIImpl(const AArch64TargetMachine *TM, const Function &F)
64       : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
65         TLI(ST->getTargetLowering()) {}
66 
67   bool areInlineCompatible(const Function *Caller,
68                            const Function *Callee) const;
69 
70   /// \name Scalar TTI Implementations
71   /// @{
72 
73   using BaseT::getIntImmCost;
74   InstructionCost getIntImmCost(int64_t Val);
75   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
76                                 TTI::TargetCostKind CostKind);
77   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
78                                     const APInt &Imm, Type *Ty,
79                                     TTI::TargetCostKind CostKind,
80                                     Instruction *Inst = nullptr);
81   InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
82                                       const APInt &Imm, Type *Ty,
83                                       TTI::TargetCostKind CostKind);
84   TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
85 
86   /// @}
87 
88   /// \name Vector TTI Implementations
89   /// @{
90 
91   bool enableInterleavedAccessVectorization() { return true; }
92 
93   unsigned getNumberOfRegisters(unsigned ClassID) const {
94     bool Vector = (ClassID == 1);
95     if (Vector) {
96       if (ST->hasNEON())
97         return 32;
98       return 0;
99     }
100     return 31;
101   }
102 
103   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
104                                         TTI::TargetCostKind CostKind);
105 
106   Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
107                                                IntrinsicInst &II) const;
108 
109   Optional<Value *> simplifyDemandedVectorEltsIntrinsic(
110       InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
111       APInt &UndefElts2, APInt &UndefElts3,
112       std::function<void(Instruction *, unsigned, APInt, APInt &)>
113           SimplifyAndSetOp) const;
114 
115   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
116     switch (K) {
117     case TargetTransformInfo::RGK_Scalar:
118       return TypeSize::getFixed(64);
119     case TargetTransformInfo::RGK_FixedWidthVector:
120       if (ST->hasSVE())
121         return TypeSize::getFixed(
122             std::max(ST->getMinSVEVectorSizeInBits(), 128u));
123       return TypeSize::getFixed(ST->hasNEON() ? 128 : 0);
124     case TargetTransformInfo::RGK_ScalableVector:
125       return TypeSize::getScalable(ST->hasSVE() ? 128 : 0);
126     }
127     llvm_unreachable("Unsupported register kind");
128   }
129 
130   unsigned getMinVectorRegisterBitWidth() const {
131     return ST->getMinVectorRegisterBitWidth();
132   }
133 
134   Optional<unsigned> getVScaleForTuning() const {
135     return ST->getVScaleForTuning();
136   }
137 
138   /// Try to return an estimate cost factor that can be used as a multiplier
139   /// when scalarizing an operation for a vector with ElementCount \p VF.
140   /// For scalable vectors this currently takes the most pessimistic view based
141   /// upon the maximum possible value for vscale.
142   unsigned getMaxNumElements(ElementCount VF) const {
143     if (!VF.isScalable())
144       return VF.getFixedValue();
145 
146     return VF.getKnownMinValue() * ST->getVScaleForTuning();
147   }
148 
149   unsigned getMaxInterleaveFactor(unsigned VF);
150 
151   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
152                                         Align Alignment, unsigned AddressSpace,
153                                         TTI::TargetCostKind CostKind);
154 
155   InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
156                                          const Value *Ptr, bool VariableMask,
157                                          Align Alignment,
158                                          TTI::TargetCostKind CostKind,
159                                          const Instruction *I = nullptr);
160 
161   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
162                                    TTI::CastContextHint CCH,
163                                    TTI::TargetCostKind CostKind,
164                                    const Instruction *I = nullptr);
165 
166   InstructionCost getExtractWithExtendCost(unsigned Opcode, Type *Dst,
167                                            VectorType *VecTy, unsigned Index);
168 
169   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
170                                  const Instruction *I = nullptr);
171 
172   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
173                                      unsigned Index);
174 
175   InstructionCost getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy,
176                                          bool IsUnsigned,
177                                          TTI::TargetCostKind CostKind);
178 
179   InstructionCost getArithmeticReductionCostSVE(unsigned Opcode,
180                                                 VectorType *ValTy,
181                                                 TTI::TargetCostKind CostKind);
182 
183   InstructionCost getSpliceCost(VectorType *Tp, int Index);
184 
185   InstructionCost getArithmeticInstrCost(
186       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
187       TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue,
188       TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue,
189       TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None,
190       TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None,
191       ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
192       const Instruction *CxtI = nullptr);
193 
194   InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
195                                             const SCEV *Ptr);
196 
197   InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
198                                      CmpInst::Predicate VecPred,
199                                      TTI::TargetCostKind CostKind,
200                                      const Instruction *I = nullptr);
201 
202   TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize,
203                                                     bool IsZeroCmp) const;
204   bool useNeonVector(const Type *Ty) const;
205 
206   InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
207                                   MaybeAlign Alignment, unsigned AddressSpace,
208                                   TTI::TargetCostKind CostKind,
209                                   const Instruction *I = nullptr);
210 
211   InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys);
212 
213   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
214                                TTI::UnrollingPreferences &UP,
215                                OptimizationRemarkEmitter *ORE);
216 
217   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
218                              TTI::PeelingPreferences &PP);
219 
220   Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
221                                            Type *ExpectedType);
222 
223   bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
224 
225   bool isElementTypeLegalForScalableVector(Type *Ty) const {
226     if (Ty->isPointerTy())
227       return true;
228 
229     if (Ty->isBFloatTy() && ST->hasBF16())
230       return true;
231 
232     if (Ty->isHalfTy() || Ty->isFloatTy() || Ty->isDoubleTy())
233       return true;
234 
235     if (Ty->isIntegerTy(8) || Ty->isIntegerTy(16) ||
236         Ty->isIntegerTy(32) || Ty->isIntegerTy(64))
237       return true;
238 
239     return false;
240   }
241 
242   bool isLegalMaskedLoadStore(Type *DataType, Align Alignment) {
243     if (!ST->hasSVE())
244       return false;
245 
246     // For fixed vectors, avoid scalarization if using SVE for them.
247     if (isa<FixedVectorType>(DataType) && !ST->useSVEForFixedLengthVectors())
248       return false; // Fall back to scalarization of masked operations.
249 
250     return isElementTypeLegalForScalableVector(DataType->getScalarType());
251   }
252 
253   bool isLegalMaskedLoad(Type *DataType, Align Alignment) {
254     return isLegalMaskedLoadStore(DataType, Alignment);
255   }
256 
257   bool isLegalMaskedStore(Type *DataType, Align Alignment) {
258     return isLegalMaskedLoadStore(DataType, Alignment);
259   }
260 
261   bool isLegalMaskedGatherScatter(Type *DataType) const {
262     if (!ST->hasSVE())
263       return false;
264 
265     // For fixed vectors, scalarize if not using SVE for them.
266     auto *DataTypeFVTy = dyn_cast<FixedVectorType>(DataType);
267     if (DataTypeFVTy && (!ST->useSVEForFixedLengthVectors() ||
268                          DataTypeFVTy->getNumElements() < 2))
269       return false;
270 
271     return isElementTypeLegalForScalableVector(DataType->getScalarType());
272   }
273 
274   bool isLegalMaskedGather(Type *DataType, Align Alignment) const {
275     return isLegalMaskedGatherScatter(DataType);
276   }
277   bool isLegalMaskedScatter(Type *DataType, Align Alignment) const {
278     return isLegalMaskedGatherScatter(DataType);
279   }
280 
281   bool isLegalNTStore(Type *DataType, Align Alignment) {
282     // NOTE: The logic below is mostly geared towards LV, which calls it with
283     //       vectors with 2 elements. We might want to improve that, if other
284     //       users show up.
285     // Nontemporal vector stores can be directly lowered to STNP, if the vector
286     // can be halved so that each half fits into a register. That's the case if
287     // the element type fits into a register and the number of elements is a
288     // power of 2 > 1.
289     if (auto *DataTypeVTy = dyn_cast<VectorType>(DataType)) {
290       unsigned NumElements =
291           cast<FixedVectorType>(DataTypeVTy)->getNumElements();
292       unsigned EltSize = DataTypeVTy->getElementType()->getScalarSizeInBits();
293       return NumElements > 1 && isPowerOf2_64(NumElements) && EltSize >= 8 &&
294              EltSize <= 128 && isPowerOf2_64(EltSize);
295     }
296     return BaseT::isLegalNTStore(DataType, Alignment);
297   }
298 
299   bool enableOrderedReductions() const { return true; }
300 
301   InstructionCost getInterleavedMemoryOpCost(
302       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
303       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
304       bool UseMaskForCond = false, bool UseMaskForGaps = false);
305 
306   bool
307   shouldConsiderAddressTypePromotion(const Instruction &I,
308                                      bool &AllowPromotionWithoutCommonHeader);
309 
310   bool shouldExpandReduction(const IntrinsicInst *II) const { return false; }
311 
312   unsigned getGISelRematGlobalCost() const {
313     return 2;
314   }
315 
316   bool emitGetActiveLaneMask() const {
317     return ST->hasSVE();
318   }
319 
320   bool supportsScalableVectors() const { return ST->hasSVE(); }
321 
322   bool enableScalableVectorization() const { return ST->hasSVE(); }
323 
324   bool isLegalToVectorizeReduction(const RecurrenceDescriptor &RdxDesc,
325                                    ElementCount VF) const;
326 
327   InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
328                                              Optional<FastMathFlags> FMF,
329                                              TTI::TargetCostKind CostKind);
330 
331   InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
332                                  ArrayRef<int> Mask, int Index,
333                                  VectorType *SubTp);
334   /// @}
335 };
336 
337 } // end namespace llvm
338 
339 #endif // LLVM_LIB_TARGET_AARCH64_AARCH64TARGETTRANSFORMINFO_H
340