106f32e7eSjoerg //===- LoopVectorizationPlanner.h - Planner for LoopVectorization ---------===// 206f32e7eSjoerg // 306f32e7eSjoerg // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 406f32e7eSjoerg // See https://llvm.org/LICENSE.txt for license information. 506f32e7eSjoerg // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 606f32e7eSjoerg // 706f32e7eSjoerg //===----------------------------------------------------------------------===// 806f32e7eSjoerg /// 906f32e7eSjoerg /// \file 1006f32e7eSjoerg /// This file provides a LoopVectorizationPlanner class. 1106f32e7eSjoerg /// InnerLoopVectorizer vectorizes loops which contain only one basic 1206f32e7eSjoerg /// LoopVectorizationPlanner - drives the vectorization process after having 1306f32e7eSjoerg /// passed Legality checks. 1406f32e7eSjoerg /// The planner builds and optimizes the Vectorization Plans which record the 1506f32e7eSjoerg /// decisions how to vectorize the given loop. In particular, represent the 1606f32e7eSjoerg /// control-flow of the vectorized version, the replication of instructions that 1706f32e7eSjoerg /// are to be scalarized, and interleave access groups. 1806f32e7eSjoerg /// 1906f32e7eSjoerg /// Also provides a VPlan-based builder utility analogous to IRBuilder. 2006f32e7eSjoerg /// It provides an instruction-level API for generating VPInstructions while 2106f32e7eSjoerg /// abstracting away the Recipe manipulation details. 2206f32e7eSjoerg //===----------------------------------------------------------------------===// 2306f32e7eSjoerg 2406f32e7eSjoerg #ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H 2506f32e7eSjoerg #define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H 2606f32e7eSjoerg 2706f32e7eSjoerg #include "VPlan.h" 2806f32e7eSjoerg 2906f32e7eSjoerg namespace llvm { 3006f32e7eSjoerg 31*da58b97aSjoerg class LoopInfo; 32*da58b97aSjoerg class LoopVectorizationLegality; 33*da58b97aSjoerg class LoopVectorizationCostModel; 34*da58b97aSjoerg class PredicatedScalarEvolution; 35*da58b97aSjoerg class LoopVectorizationRequirements; 36*da58b97aSjoerg class LoopVectorizeHints; 37*da58b97aSjoerg class OptimizationRemarkEmitter; 38*da58b97aSjoerg class TargetTransformInfo; 39*da58b97aSjoerg class TargetLibraryInfo; 40*da58b97aSjoerg class VPRecipeBuilder; 41*da58b97aSjoerg 4206f32e7eSjoerg /// VPlan-based builder utility analogous to IRBuilder. 4306f32e7eSjoerg class VPBuilder { 4406f32e7eSjoerg VPBasicBlock *BB = nullptr; 4506f32e7eSjoerg VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator(); 4606f32e7eSjoerg createInstruction(unsigned Opcode,ArrayRef<VPValue * > Operands)4706f32e7eSjoerg VPInstruction *createInstruction(unsigned Opcode, 4806f32e7eSjoerg ArrayRef<VPValue *> Operands) { 4906f32e7eSjoerg VPInstruction *Instr = new VPInstruction(Opcode, Operands); 5006f32e7eSjoerg if (BB) 5106f32e7eSjoerg BB->insert(Instr, InsertPt); 5206f32e7eSjoerg return Instr; 5306f32e7eSjoerg } 5406f32e7eSjoerg createInstruction(unsigned Opcode,std::initializer_list<VPValue * > Operands)5506f32e7eSjoerg VPInstruction *createInstruction(unsigned Opcode, 5606f32e7eSjoerg std::initializer_list<VPValue *> Operands) { 5706f32e7eSjoerg return createInstruction(Opcode, ArrayRef<VPValue *>(Operands)); 5806f32e7eSjoerg } 5906f32e7eSjoerg 6006f32e7eSjoerg public: VPBuilder()6106f32e7eSjoerg VPBuilder() {} 6206f32e7eSjoerg 6306f32e7eSjoerg /// Clear the insertion point: created instructions will not be inserted into 6406f32e7eSjoerg /// a block. clearInsertionPoint()6506f32e7eSjoerg void clearInsertionPoint() { 6606f32e7eSjoerg BB = nullptr; 6706f32e7eSjoerg InsertPt = VPBasicBlock::iterator(); 6806f32e7eSjoerg } 6906f32e7eSjoerg getInsertBlock()7006f32e7eSjoerg VPBasicBlock *getInsertBlock() const { return BB; } getInsertPoint()7106f32e7eSjoerg VPBasicBlock::iterator getInsertPoint() const { return InsertPt; } 7206f32e7eSjoerg 7306f32e7eSjoerg /// InsertPoint - A saved insertion point. 7406f32e7eSjoerg class VPInsertPoint { 7506f32e7eSjoerg VPBasicBlock *Block = nullptr; 7606f32e7eSjoerg VPBasicBlock::iterator Point; 7706f32e7eSjoerg 7806f32e7eSjoerg public: 7906f32e7eSjoerg /// Creates a new insertion point which doesn't point to anything. 8006f32e7eSjoerg VPInsertPoint() = default; 8106f32e7eSjoerg 8206f32e7eSjoerg /// Creates a new insertion point at the given location. VPInsertPoint(VPBasicBlock * InsertBlock,VPBasicBlock::iterator InsertPoint)8306f32e7eSjoerg VPInsertPoint(VPBasicBlock *InsertBlock, VPBasicBlock::iterator InsertPoint) 8406f32e7eSjoerg : Block(InsertBlock), Point(InsertPoint) {} 8506f32e7eSjoerg 8606f32e7eSjoerg /// Returns true if this insert point is set. isSet()8706f32e7eSjoerg bool isSet() const { return Block != nullptr; } 8806f32e7eSjoerg getBlock()8906f32e7eSjoerg VPBasicBlock *getBlock() const { return Block; } getPoint()9006f32e7eSjoerg VPBasicBlock::iterator getPoint() const { return Point; } 9106f32e7eSjoerg }; 9206f32e7eSjoerg 9306f32e7eSjoerg /// Sets the current insert point to a previously-saved location. restoreIP(VPInsertPoint IP)9406f32e7eSjoerg void restoreIP(VPInsertPoint IP) { 9506f32e7eSjoerg if (IP.isSet()) 9606f32e7eSjoerg setInsertPoint(IP.getBlock(), IP.getPoint()); 9706f32e7eSjoerg else 9806f32e7eSjoerg clearInsertionPoint(); 9906f32e7eSjoerg } 10006f32e7eSjoerg 10106f32e7eSjoerg /// This specifies that created VPInstructions should be appended to the end 10206f32e7eSjoerg /// of the specified block. setInsertPoint(VPBasicBlock * TheBB)10306f32e7eSjoerg void setInsertPoint(VPBasicBlock *TheBB) { 10406f32e7eSjoerg assert(TheBB && "Attempting to set a null insert point"); 10506f32e7eSjoerg BB = TheBB; 10606f32e7eSjoerg InsertPt = BB->end(); 10706f32e7eSjoerg } 10806f32e7eSjoerg 10906f32e7eSjoerg /// This specifies that created instructions should be inserted at the 11006f32e7eSjoerg /// specified point. setInsertPoint(VPBasicBlock * TheBB,VPBasicBlock::iterator IP)11106f32e7eSjoerg void setInsertPoint(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) { 11206f32e7eSjoerg BB = TheBB; 11306f32e7eSjoerg InsertPt = IP; 11406f32e7eSjoerg } 11506f32e7eSjoerg 11606f32e7eSjoerg /// Insert and return the specified instruction. insert(VPInstruction * I)11706f32e7eSjoerg VPInstruction *insert(VPInstruction *I) const { 11806f32e7eSjoerg BB->insert(I, InsertPt); 11906f32e7eSjoerg return I; 12006f32e7eSjoerg } 12106f32e7eSjoerg 12206f32e7eSjoerg /// Create an N-ary operation with \p Opcode, \p Operands and set \p Inst as 12306f32e7eSjoerg /// its underlying Instruction. 12406f32e7eSjoerg VPValue *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands, 12506f32e7eSjoerg Instruction *Inst = nullptr) { 12606f32e7eSjoerg VPInstruction *NewVPInst = createInstruction(Opcode, Operands); 12706f32e7eSjoerg NewVPInst->setUnderlyingValue(Inst); 12806f32e7eSjoerg return NewVPInst; 12906f32e7eSjoerg } 13006f32e7eSjoerg VPValue *createNaryOp(unsigned Opcode, 13106f32e7eSjoerg std::initializer_list<VPValue *> Operands, 13206f32e7eSjoerg Instruction *Inst = nullptr) { 13306f32e7eSjoerg return createNaryOp(Opcode, ArrayRef<VPValue *>(Operands), Inst); 13406f32e7eSjoerg } 13506f32e7eSjoerg createNot(VPValue * Operand)13606f32e7eSjoerg VPValue *createNot(VPValue *Operand) { 13706f32e7eSjoerg return createInstruction(VPInstruction::Not, {Operand}); 13806f32e7eSjoerg } 13906f32e7eSjoerg createAnd(VPValue * LHS,VPValue * RHS)14006f32e7eSjoerg VPValue *createAnd(VPValue *LHS, VPValue *RHS) { 14106f32e7eSjoerg return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}); 14206f32e7eSjoerg } 14306f32e7eSjoerg createOr(VPValue * LHS,VPValue * RHS)14406f32e7eSjoerg VPValue *createOr(VPValue *LHS, VPValue *RHS) { 14506f32e7eSjoerg return createInstruction(Instruction::BinaryOps::Or, {LHS, RHS}); 14606f32e7eSjoerg } 14706f32e7eSjoerg createSelect(VPValue * Cond,VPValue * TrueVal,VPValue * FalseVal)148*da58b97aSjoerg VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal) { 149*da58b97aSjoerg return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal}); 150*da58b97aSjoerg } 151*da58b97aSjoerg 15206f32e7eSjoerg //===--------------------------------------------------------------------===// 15306f32e7eSjoerg // RAII helpers. 15406f32e7eSjoerg //===--------------------------------------------------------------------===// 15506f32e7eSjoerg 15606f32e7eSjoerg /// RAII object that stores the current insertion point and restores it when 15706f32e7eSjoerg /// the object is destroyed. 15806f32e7eSjoerg class InsertPointGuard { 15906f32e7eSjoerg VPBuilder &Builder; 16006f32e7eSjoerg VPBasicBlock *Block; 16106f32e7eSjoerg VPBasicBlock::iterator Point; 16206f32e7eSjoerg 16306f32e7eSjoerg public: InsertPointGuard(VPBuilder & B)16406f32e7eSjoerg InsertPointGuard(VPBuilder &B) 16506f32e7eSjoerg : Builder(B), Block(B.getInsertBlock()), Point(B.getInsertPoint()) {} 16606f32e7eSjoerg 16706f32e7eSjoerg InsertPointGuard(const InsertPointGuard &) = delete; 16806f32e7eSjoerg InsertPointGuard &operator=(const InsertPointGuard &) = delete; 16906f32e7eSjoerg ~InsertPointGuard()17006f32e7eSjoerg ~InsertPointGuard() { Builder.restoreIP(VPInsertPoint(Block, Point)); } 17106f32e7eSjoerg }; 17206f32e7eSjoerg }; 17306f32e7eSjoerg 17406f32e7eSjoerg /// TODO: The following VectorizationFactor was pulled out of 17506f32e7eSjoerg /// LoopVectorizationCostModel class. LV also deals with 17606f32e7eSjoerg /// VectorizerParams::VectorizationFactor and VectorizationCostTy. 17706f32e7eSjoerg /// We need to streamline them. 17806f32e7eSjoerg 17906f32e7eSjoerg /// Information about vectorization costs 18006f32e7eSjoerg struct VectorizationFactor { 18106f32e7eSjoerg // Vector width with best cost 182*da58b97aSjoerg ElementCount Width; 18306f32e7eSjoerg // Cost of the loop with that width 184*da58b97aSjoerg InstructionCost Cost; 185*da58b97aSjoerg VectorizationFactorVectorizationFactor186*da58b97aSjoerg VectorizationFactor(ElementCount Width, InstructionCost Cost) 187*da58b97aSjoerg : Width(Width), Cost(Cost) {} 18806f32e7eSjoerg 18906f32e7eSjoerg // Width 1 means no vectorization, cost 0 means uncomputed cost. DisabledVectorizationFactor190*da58b97aSjoerg static VectorizationFactor Disabled() { 191*da58b97aSjoerg return {ElementCount::getFixed(1), 0}; 192*da58b97aSjoerg } 19306f32e7eSjoerg 19406f32e7eSjoerg bool operator==(const VectorizationFactor &rhs) const { 19506f32e7eSjoerg return Width == rhs.Width && Cost == rhs.Cost; 19606f32e7eSjoerg } 197*da58b97aSjoerg 198*da58b97aSjoerg bool operator!=(const VectorizationFactor &rhs) const { 199*da58b97aSjoerg return !(*this == rhs); 200*da58b97aSjoerg } 201*da58b97aSjoerg }; 202*da58b97aSjoerg 203*da58b97aSjoerg /// A class that represents two vectorization factors (initialized with 0 by 204*da58b97aSjoerg /// default). One for fixed-width vectorization and one for scalable 205*da58b97aSjoerg /// vectorization. This can be used by the vectorizer to choose from a range of 206*da58b97aSjoerg /// fixed and/or scalable VFs in order to find the most cost-effective VF to 207*da58b97aSjoerg /// vectorize with. 208*da58b97aSjoerg struct FixedScalableVFPair { 209*da58b97aSjoerg ElementCount FixedVF; 210*da58b97aSjoerg ElementCount ScalableVF; 211*da58b97aSjoerg FixedScalableVFPairFixedScalableVFPair212*da58b97aSjoerg FixedScalableVFPair() 213*da58b97aSjoerg : FixedVF(ElementCount::getFixed(0)), 214*da58b97aSjoerg ScalableVF(ElementCount::getScalable(0)) {} FixedScalableVFPairFixedScalableVFPair215*da58b97aSjoerg FixedScalableVFPair(const ElementCount &Max) : FixedScalableVFPair() { 216*da58b97aSjoerg *(Max.isScalable() ? &ScalableVF : &FixedVF) = Max; 217*da58b97aSjoerg } FixedScalableVFPairFixedScalableVFPair218*da58b97aSjoerg FixedScalableVFPair(const ElementCount &FixedVF, 219*da58b97aSjoerg const ElementCount &ScalableVF) 220*da58b97aSjoerg : FixedVF(FixedVF), ScalableVF(ScalableVF) { 221*da58b97aSjoerg assert(!FixedVF.isScalable() && ScalableVF.isScalable() && 222*da58b97aSjoerg "Invalid scalable properties"); 223*da58b97aSjoerg } 224*da58b97aSjoerg getNoneFixedScalableVFPair225*da58b97aSjoerg static FixedScalableVFPair getNone() { return FixedScalableVFPair(); } 226*da58b97aSjoerg 227*da58b97aSjoerg /// \return true if either fixed- or scalable VF is non-zero. 228*da58b97aSjoerg explicit operator bool() const { return FixedVF || ScalableVF; } 229*da58b97aSjoerg 230*da58b97aSjoerg /// \return true if either fixed- or scalable VF is a valid vector VF. hasVectorFixedScalableVFPair231*da58b97aSjoerg bool hasVector() const { return FixedVF.isVector() || ScalableVF.isVector(); } 23206f32e7eSjoerg }; 23306f32e7eSjoerg 23406f32e7eSjoerg /// Planner drives the vectorization process after having passed 23506f32e7eSjoerg /// Legality checks. 23606f32e7eSjoerg class LoopVectorizationPlanner { 23706f32e7eSjoerg /// The loop that we evaluate. 23806f32e7eSjoerg Loop *OrigLoop; 23906f32e7eSjoerg 24006f32e7eSjoerg /// Loop Info analysis. 24106f32e7eSjoerg LoopInfo *LI; 24206f32e7eSjoerg 24306f32e7eSjoerg /// Target Library Info. 24406f32e7eSjoerg const TargetLibraryInfo *TLI; 24506f32e7eSjoerg 24606f32e7eSjoerg /// Target Transform Info. 24706f32e7eSjoerg const TargetTransformInfo *TTI; 24806f32e7eSjoerg 24906f32e7eSjoerg /// The legality analysis. 25006f32e7eSjoerg LoopVectorizationLegality *Legal; 25106f32e7eSjoerg 25206f32e7eSjoerg /// The profitability analysis. 25306f32e7eSjoerg LoopVectorizationCostModel &CM; 25406f32e7eSjoerg 255*da58b97aSjoerg /// The interleaved access analysis. 256*da58b97aSjoerg InterleavedAccessInfo &IAI; 257*da58b97aSjoerg 258*da58b97aSjoerg PredicatedScalarEvolution &PSE; 259*da58b97aSjoerg 260*da58b97aSjoerg const LoopVectorizeHints &Hints; 261*da58b97aSjoerg 262*da58b97aSjoerg LoopVectorizationRequirements &Requirements; 263*da58b97aSjoerg 264*da58b97aSjoerg OptimizationRemarkEmitter *ORE; 265*da58b97aSjoerg 26606f32e7eSjoerg SmallVector<VPlanPtr, 4> VPlans; 26706f32e7eSjoerg 26806f32e7eSjoerg /// A builder used to construct the current plan. 26906f32e7eSjoerg VPBuilder Builder; 27006f32e7eSjoerg 271*da58b97aSjoerg /// The best number of elements of the vector types used in the 272*da58b97aSjoerg /// transformed loop. BestVF = None means that vectorization is 273*da58b97aSjoerg /// disabled. 274*da58b97aSjoerg Optional<ElementCount> BestVF = None; 27506f32e7eSjoerg unsigned BestUF = 0; 27606f32e7eSjoerg 27706f32e7eSjoerg public: LoopVectorizationPlanner(Loop * L,LoopInfo * LI,const TargetLibraryInfo * TLI,const TargetTransformInfo * TTI,LoopVectorizationLegality * Legal,LoopVectorizationCostModel & CM,InterleavedAccessInfo & IAI,PredicatedScalarEvolution & PSE,const LoopVectorizeHints & Hints,LoopVectorizationRequirements & Requirements,OptimizationRemarkEmitter * ORE)27806f32e7eSjoerg LoopVectorizationPlanner(Loop *L, LoopInfo *LI, const TargetLibraryInfo *TLI, 27906f32e7eSjoerg const TargetTransformInfo *TTI, 28006f32e7eSjoerg LoopVectorizationLegality *Legal, 281*da58b97aSjoerg LoopVectorizationCostModel &CM, 282*da58b97aSjoerg InterleavedAccessInfo &IAI, 283*da58b97aSjoerg PredicatedScalarEvolution &PSE, 284*da58b97aSjoerg const LoopVectorizeHints &Hints, 285*da58b97aSjoerg LoopVectorizationRequirements &Requirements, 286*da58b97aSjoerg OptimizationRemarkEmitter *ORE) 287*da58b97aSjoerg : OrigLoop(L), LI(LI), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM), IAI(IAI), 288*da58b97aSjoerg PSE(PSE), Hints(Hints), Requirements(Requirements), ORE(ORE) {} 28906f32e7eSjoerg 29006f32e7eSjoerg /// Plan how to best vectorize, return the best VF and its cost, or None if 29106f32e7eSjoerg /// vectorization and interleaving should be avoided up front. 292*da58b97aSjoerg Optional<VectorizationFactor> plan(ElementCount UserVF, unsigned UserIC); 29306f32e7eSjoerg 29406f32e7eSjoerg /// Use the VPlan-native path to plan how to best vectorize, return the best 29506f32e7eSjoerg /// VF and its cost. 296*da58b97aSjoerg VectorizationFactor planInVPlanNativePath(ElementCount UserVF); 29706f32e7eSjoerg 29806f32e7eSjoerg /// Finalize the best decision and dispose of all other VPlans. 299*da58b97aSjoerg void setBestPlan(ElementCount VF, unsigned UF); 30006f32e7eSjoerg 30106f32e7eSjoerg /// Generate the IR code for the body of the vectorized loop according to the 30206f32e7eSjoerg /// best selected VPlan. 30306f32e7eSjoerg void executePlan(InnerLoopVectorizer &LB, DominatorTree *DT); 30406f32e7eSjoerg 305*da58b97aSjoerg #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 306*da58b97aSjoerg void printPlans(raw_ostream &O); 307*da58b97aSjoerg #endif 308*da58b97aSjoerg 309*da58b97aSjoerg /// Look through the existing plans and return true if we have one with all 310*da58b97aSjoerg /// the vectorization factors in question. hasPlanWithVFs(const ArrayRef<ElementCount> VFs)311*da58b97aSjoerg bool hasPlanWithVFs(const ArrayRef<ElementCount> VFs) const { 312*da58b97aSjoerg return any_of(VPlans, [&](const VPlanPtr &Plan) { 313*da58b97aSjoerg return all_of(VFs, [&](const ElementCount &VF) { 314*da58b97aSjoerg return Plan->hasVF(VF); 315*da58b97aSjoerg }); 316*da58b97aSjoerg }); 31706f32e7eSjoerg } 31806f32e7eSjoerg 31906f32e7eSjoerg /// Test a \p Predicate on a \p Range of VF's. Return the value of applying 32006f32e7eSjoerg /// \p Predicate on Range.Start, possibly decreasing Range.End such that the 32106f32e7eSjoerg /// returned value holds for the entire \p Range. 32206f32e7eSjoerg static bool 323*da58b97aSjoerg getDecisionAndClampRange(const std::function<bool(ElementCount)> &Predicate, 32406f32e7eSjoerg VFRange &Range); 32506f32e7eSjoerg 32606f32e7eSjoerg protected: 32706f32e7eSjoerg /// Collect the instructions from the original loop that would be trivially 32806f32e7eSjoerg /// dead in the vectorized loop if generated. 32906f32e7eSjoerg void collectTriviallyDeadInstructions( 33006f32e7eSjoerg SmallPtrSetImpl<Instruction *> &DeadInstructions); 33106f32e7eSjoerg 33206f32e7eSjoerg /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive, 33306f32e7eSjoerg /// according to the information gathered by Legal when it checked if it is 33406f32e7eSjoerg /// legal to vectorize the loop. 335*da58b97aSjoerg void buildVPlans(ElementCount MinVF, ElementCount MaxVF); 33606f32e7eSjoerg 33706f32e7eSjoerg private: 33806f32e7eSjoerg /// Build a VPlan according to the information gathered by Legal. \return a 33906f32e7eSjoerg /// VPlan for vectorization factors \p Range.Start and up to \p Range.End 34006f32e7eSjoerg /// exclusive, possibly decreasing \p Range.End. 34106f32e7eSjoerg VPlanPtr buildVPlan(VFRange &Range); 34206f32e7eSjoerg 34306f32e7eSjoerg /// Build a VPlan using VPRecipes according to the information gather by 34406f32e7eSjoerg /// Legal. This method is only used for the legacy inner loop vectorizer. 345*da58b97aSjoerg VPlanPtr buildVPlanWithVPRecipes( 346*da58b97aSjoerg VFRange &Range, SmallPtrSetImpl<Instruction *> &DeadInstructions, 347*da58b97aSjoerg const DenseMap<Instruction *, Instruction *> &SinkAfter); 34806f32e7eSjoerg 34906f32e7eSjoerg /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive, 35006f32e7eSjoerg /// according to the information gathered by Legal when it checked if it is 35106f32e7eSjoerg /// legal to vectorize the loop. This method creates VPlans using VPRecipes. 352*da58b97aSjoerg void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF); 353*da58b97aSjoerg 354*da58b97aSjoerg /// Adjust the recipes for any inloop reductions. The chain of instructions 355*da58b97aSjoerg /// leading from the loop exit instr to the phi need to be converted to 356*da58b97aSjoerg /// reductions, with one operand being vector and the other being the scalar 357*da58b97aSjoerg /// reduction chain. 358*da58b97aSjoerg void adjustRecipesForInLoopReductions(VPlanPtr &Plan, 359*da58b97aSjoerg VPRecipeBuilder &RecipeBuilder); 36006f32e7eSjoerg }; 36106f32e7eSjoerg 36206f32e7eSjoerg } // namespace llvm 36306f32e7eSjoerg 36406f32e7eSjoerg #endif // LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H 365