1 //===- AArch64TargetTransformInfo.h - AArch64 specific TTI ------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file a TargetTransformInfo::Concept conforming object specific to the
10 /// AArch64 target machine. It uses the target's detailed information to
11 /// provide more precise answers to certain TTI queries, while letting the
12 /// target independent and default TTI implementations handle the rest.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64TARGETTRANSFORMINFO_H
17 #define LLVM_LIB_TARGET_AARCH64_AARCH64TARGETTRANSFORMINFO_H
18 
19 #include "AArch64.h"
20 #include "AArch64Subtarget.h"
21 #include "AArch64TargetMachine.h"
22 #include "llvm/ADT/ArrayRef.h"
23 #include "llvm/Analysis/TargetTransformInfo.h"
24 #include "llvm/CodeGen/BasicTTIImpl.h"
25 #include "llvm/IR/Function.h"
26 #include "llvm/IR/Intrinsics.h"
27 #include <cstdint>
28 
29 namespace llvm {
30 
31 class APInt;
32 class Instruction;
33 class IntrinsicInst;
34 class Loop;
35 class SCEV;
36 class ScalarEvolution;
37 class Type;
38 class Value;
39 class VectorType;
40 
41 class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
42   using BaseT = BasicTTIImplBase<AArch64TTIImpl>;
43   using TTI = TargetTransformInfo;
44 
45   friend BaseT;
46 
47   const AArch64Subtarget *ST;
48   const AArch64TargetLowering *TLI;
49 
getST()50   const AArch64Subtarget *getST() const { return ST; }
getTLI()51   const AArch64TargetLowering *getTLI() const { return TLI; }
52 
53   enum MemIntrinsicType {
54     VECTOR_LDST_TWO_ELEMENTS,
55     VECTOR_LDST_THREE_ELEMENTS,
56     VECTOR_LDST_FOUR_ELEMENTS
57   };
58 
59   bool isWideningInstruction(Type *Ty, unsigned Opcode,
60                              ArrayRef<const Value *> Args);
61 
62 public:
AArch64TTIImpl(const AArch64TargetMachine * TM,const Function & F)63   explicit AArch64TTIImpl(const AArch64TargetMachine *TM, const Function &F)
64       : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
65         TLI(ST->getTargetLowering()) {}
66 
67   bool areInlineCompatible(const Function *Caller,
68                            const Function *Callee) const;
69 
70   /// \name Scalar TTI Implementations
71   /// @{
72 
73   using BaseT::getIntImmCost;
74   InstructionCost getIntImmCost(int64_t Val);
75   InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
76                                 TTI::TargetCostKind CostKind);
77   InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
78                                     const APInt &Imm, Type *Ty,
79                                     TTI::TargetCostKind CostKind,
80                                     Instruction *Inst = nullptr);
81   InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
82                                       const APInt &Imm, Type *Ty,
83                                       TTI::TargetCostKind CostKind);
84   TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
85 
86   /// @}
87 
88   /// \name Vector TTI Implementations
89   /// @{
90 
enableInterleavedAccessVectorization()91   bool enableInterleavedAccessVectorization() { return true; }
92 
getNumberOfRegisters(unsigned ClassID)93   unsigned getNumberOfRegisters(unsigned ClassID) const {
94     bool Vector = (ClassID == 1);
95     if (Vector) {
96       if (ST->hasNEON())
97         return 32;
98       return 0;
99     }
100     return 31;
101   }
102 
103   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
104                                         TTI::TargetCostKind CostKind);
105 
106   Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
107                                                IntrinsicInst &II) const;
108 
getRegisterBitWidth(TargetTransformInfo::RegisterKind K)109   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
110     switch (K) {
111     case TargetTransformInfo::RGK_Scalar:
112       return TypeSize::getFixed(64);
113     case TargetTransformInfo::RGK_FixedWidthVector:
114       if (ST->hasSVE())
115         return TypeSize::getFixed(
116             std::max(ST->getMinSVEVectorSizeInBits(), 128u));
117       return TypeSize::getFixed(ST->hasNEON() ? 128 : 0);
118     case TargetTransformInfo::RGK_ScalableVector:
119       return TypeSize::getScalable(ST->hasSVE() ? 128 : 0);
120     }
121     llvm_unreachable("Unsupported register kind");
122   }
123 
getMinVectorRegisterBitWidth()124   unsigned getMinVectorRegisterBitWidth() const {
125     return ST->getMinVectorRegisterBitWidth();
126   }
127 
128 
129   /// Try to return an estimate cost factor that can be used as a multiplier
130   /// when scalarizing an operation for a vector with ElementCount \p VF.
131   /// For scalable vectors this currently takes the most pessimistic view based
132   /// upon the maximum possible value for vscale.
133   unsigned getMaxNumElements(ElementCount VF,
134                              const Function *F = nullptr) const {
135     if (!VF.isScalable())
136       return VF.getFixedValue();
137 
138     unsigned MaxNumVScale = 16;
139     if (F && F->hasFnAttribute(Attribute::VScaleRange)) {
140       unsigned VScaleMax =
141           F->getFnAttribute(Attribute::VScaleRange).getVScaleRangeArgs().second;
142       if (VScaleMax > 0)
143         MaxNumVScale = VScaleMax;
144     }
145 
146     return MaxNumVScale * VF.getKnownMinValue();
147   }
148 
149   unsigned getMaxInterleaveFactor(unsigned VF);
150 
151   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
152                                         Align Alignment, unsigned AddressSpace,
153                                         TTI::TargetCostKind CostKind);
154 
155   InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
156                                          const Value *Ptr, bool VariableMask,
157                                          Align Alignment,
158                                          TTI::TargetCostKind CostKind,
159                                          const Instruction *I = nullptr);
160 
161   InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
162                                    TTI::CastContextHint CCH,
163                                    TTI::TargetCostKind CostKind,
164                                    const Instruction *I = nullptr);
165 
166   InstructionCost getExtractWithExtendCost(unsigned Opcode, Type *Dst,
167                                            VectorType *VecTy, unsigned Index);
168 
169   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
170                                  const Instruction *I = nullptr);
171 
172   InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
173                                      unsigned Index);
174 
175   InstructionCost getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy,
176                                          bool IsUnsigned,
177                                          TTI::TargetCostKind CostKind);
178 
179   InstructionCost getArithmeticReductionCostSVE(unsigned Opcode,
180                                                 VectorType *ValTy,
181                                                 TTI::TargetCostKind CostKind);
182 
183   InstructionCost getSpliceCost(VectorType *Tp, int Index);
184 
185   InstructionCost getArithmeticInstrCost(
186       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
187       TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue,
188       TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue,
189       TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None,
190       TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None,
191       ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
192       const Instruction *CxtI = nullptr);
193 
194   InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
195                                             const SCEV *Ptr);
196 
197   InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
198                                      CmpInst::Predicate VecPred,
199                                      TTI::TargetCostKind CostKind,
200                                      const Instruction *I = nullptr);
201 
202   TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize,
203                                                     bool IsZeroCmp) const;
204   bool useNeonVector(const Type *Ty) const;
205 
206   InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
207                                   MaybeAlign Alignment, unsigned AddressSpace,
208                                   TTI::TargetCostKind CostKind,
209                                   const Instruction *I = nullptr);
210 
211   InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys);
212 
213   void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
214                                TTI::UnrollingPreferences &UP,
215                                OptimizationRemarkEmitter *ORE);
216 
217   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
218                              TTI::PeelingPreferences &PP);
219 
220   Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
221                                            Type *ExpectedType);
222 
223   bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
224 
isElementTypeLegalForScalableVector(Type * Ty)225   bool isElementTypeLegalForScalableVector(Type *Ty) const {
226     if (Ty->isPointerTy())
227       return true;
228 
229     if (Ty->isBFloatTy() && ST->hasBF16())
230       return true;
231 
232     if (Ty->isHalfTy() || Ty->isFloatTy() || Ty->isDoubleTy())
233       return true;
234 
235     if (Ty->isIntegerTy(1) || Ty->isIntegerTy(8) || Ty->isIntegerTy(16) ||
236         Ty->isIntegerTy(32) || Ty->isIntegerTy(64))
237       return true;
238 
239     return false;
240   }
241 
isLegalMaskedLoadStore(Type * DataType,Align Alignment)242   bool isLegalMaskedLoadStore(Type *DataType, Align Alignment) {
243     if (!ST->hasSVE())
244       return false;
245 
246     // For fixed vectors, avoid scalarization if using SVE for them.
247     if (isa<FixedVectorType>(DataType) && !ST->useSVEForFixedLengthVectors())
248       return false; // Fall back to scalarization of masked operations.
249 
250     return !DataType->getScalarType()->isIntegerTy(1) &&
251            isElementTypeLegalForScalableVector(DataType->getScalarType());
252   }
253 
isLegalMaskedLoad(Type * DataType,Align Alignment)254   bool isLegalMaskedLoad(Type *DataType, Align Alignment) {
255     return isLegalMaskedLoadStore(DataType, Alignment);
256   }
257 
isLegalMaskedStore(Type * DataType,Align Alignment)258   bool isLegalMaskedStore(Type *DataType, Align Alignment) {
259     return isLegalMaskedLoadStore(DataType, Alignment);
260   }
261 
isLegalMaskedGatherScatter(Type * DataType)262   bool isLegalMaskedGatherScatter(Type *DataType) const {
263     if (!ST->hasSVE())
264       return false;
265 
266     // For fixed vectors, scalarize if not using SVE for them.
267     auto *DataTypeFVTy = dyn_cast<FixedVectorType>(DataType);
268     if (DataTypeFVTy && (!ST->useSVEForFixedLengthVectors() ||
269                          DataTypeFVTy->getNumElements() < 2))
270       return false;
271 
272     return !DataType->getScalarType()->isIntegerTy(1) &&
273            isElementTypeLegalForScalableVector(DataType->getScalarType());
274   }
275 
isLegalMaskedGather(Type * DataType,Align Alignment)276   bool isLegalMaskedGather(Type *DataType, Align Alignment) const {
277     return isLegalMaskedGatherScatter(DataType);
278   }
isLegalMaskedScatter(Type * DataType,Align Alignment)279   bool isLegalMaskedScatter(Type *DataType, Align Alignment) const {
280     return isLegalMaskedGatherScatter(DataType);
281   }
282 
isLegalNTStore(Type * DataType,Align Alignment)283   bool isLegalNTStore(Type *DataType, Align Alignment) {
284     // NOTE: The logic below is mostly geared towards LV, which calls it with
285     //       vectors with 2 elements. We might want to improve that, if other
286     //       users show up.
287     // Nontemporal vector stores can be directly lowered to STNP, if the vector
288     // can be halved so that each half fits into a register. That's the case if
289     // the element type fits into a register and the number of elements is a
290     // power of 2 > 1.
291     if (auto *DataTypeVTy = dyn_cast<VectorType>(DataType)) {
292       unsigned NumElements =
293           cast<FixedVectorType>(DataTypeVTy)->getNumElements();
294       unsigned EltSize = DataTypeVTy->getElementType()->getScalarSizeInBits();
295       return NumElements > 1 && isPowerOf2_64(NumElements) && EltSize >= 8 &&
296              EltSize <= 128 && isPowerOf2_64(EltSize);
297     }
298     return BaseT::isLegalNTStore(DataType, Alignment);
299   }
300 
enableOrderedReductions()301   bool enableOrderedReductions() const { return true; }
302 
303   InstructionCost getInterleavedMemoryOpCost(
304       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
305       Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
306       bool UseMaskForCond = false, bool UseMaskForGaps = false);
307 
308   bool
309   shouldConsiderAddressTypePromotion(const Instruction &I,
310                                      bool &AllowPromotionWithoutCommonHeader);
311 
shouldExpandReduction(const IntrinsicInst * II)312   bool shouldExpandReduction(const IntrinsicInst *II) const { return false; }
313 
getGISelRematGlobalCost()314   unsigned getGISelRematGlobalCost() const {
315     return 2;
316   }
317 
supportsScalableVectors()318   bool supportsScalableVectors() const { return ST->hasSVE(); }
319 
320   bool isLegalToVectorizeReduction(const RecurrenceDescriptor &RdxDesc,
321                                    ElementCount VF) const;
322 
323   InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
324                                              Optional<FastMathFlags> FMF,
325                                              TTI::TargetCostKind CostKind);
326 
327   InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
328                                  ArrayRef<int> Mask, int Index,
329                                  VectorType *SubTp);
330   /// @}
331 };
332 
333 } // end namespace llvm
334 
335 #endif // LLVM_LIB_TARGET_AARCH64_AARCH64TARGETTRANSFORMINFO_H
336