1 //===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements a set of utility VPlan to VPlan transformations.
11 ///
12 //===----------------------------------------------------------------------===//
13
14 #include "VPlanTransforms.h"
15 #include "llvm/ADT/PostOrderIterator.h"
16
17 using namespace llvm;
18
VPInstructionsToVPRecipes(Loop * OrigLoop,VPlanPtr & Plan,LoopVectorizationLegality::InductionList & Inductions,SmallPtrSetImpl<Instruction * > & DeadInstructions,ScalarEvolution & SE)19 void VPlanTransforms::VPInstructionsToVPRecipes(
20 Loop *OrigLoop, VPlanPtr &Plan,
21 LoopVectorizationLegality::InductionList &Inductions,
22 SmallPtrSetImpl<Instruction *> &DeadInstructions, ScalarEvolution &SE) {
23
24 auto *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
25 ReversePostOrderTraversal<VPBlockBase *> RPOT(TopRegion->getEntry());
26
27 for (VPBlockBase *Base : RPOT) {
28 // Do not widen instructions in pre-header and exit blocks.
29 if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0)
30 continue;
31
32 VPBasicBlock *VPBB = Base->getEntryBasicBlock();
33 // Introduce each ingredient into VPlan.
34 for (auto I = VPBB->begin(), E = VPBB->end(); I != E;) {
35 VPRecipeBase *Ingredient = &*I++;
36 VPValue *VPV = Ingredient->getVPSingleValue();
37 Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
38 if (DeadInstructions.count(Inst)) {
39 VPValue DummyValue;
40 VPV->replaceAllUsesWith(&DummyValue);
41 Ingredient->eraseFromParent();
42 continue;
43 }
44
45 VPRecipeBase *NewRecipe = nullptr;
46 if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(Ingredient)) {
47 auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
48 InductionDescriptor II = Inductions.lookup(Phi);
49 if (II.getKind() == InductionDescriptor::IK_IntInduction ||
50 II.getKind() == InductionDescriptor::IK_FpInduction) {
51 VPValue *Start = Plan->getOrAddVPValue(II.getStartValue());
52 NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, nullptr);
53 } else {
54 Plan->addVPValue(Phi, VPPhi);
55 continue;
56 }
57 } else {
58 assert(isa<VPInstruction>(Ingredient) &&
59 "only VPInstructions expected here");
60 assert(!isa<PHINode>(Inst) && "phis should be handled above");
61 // Create VPWidenMemoryInstructionRecipe for loads and stores.
62 if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
63 NewRecipe = new VPWidenMemoryInstructionRecipe(
64 *Load, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
65 nullptr /*Mask*/);
66 } else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
67 NewRecipe = new VPWidenMemoryInstructionRecipe(
68 *Store, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
69 Plan->getOrAddVPValue(Store->getValueOperand()),
70 nullptr /*Mask*/);
71 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
72 NewRecipe = new VPWidenGEPRecipe(
73 GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop);
74 } else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
75 NewRecipe = new VPWidenCallRecipe(
76 *CI, Plan->mapToVPValues(CI->arg_operands()));
77 } else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
78 bool InvariantCond =
79 SE.isLoopInvariant(SE.getSCEV(SI->getOperand(0)), OrigLoop);
80 NewRecipe = new VPWidenSelectRecipe(
81 *SI, Plan->mapToVPValues(SI->operands()), InvariantCond);
82 } else {
83 NewRecipe =
84 new VPWidenRecipe(*Inst, Plan->mapToVPValues(Inst->operands()));
85 }
86 }
87
88 NewRecipe->insertBefore(Ingredient);
89 if (NewRecipe->getNumDefinedValues() == 1)
90 VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
91 else
92 assert(NewRecipe->getNumDefinedValues() == 0 &&
93 "Only recpies with zero or one defined values expected");
94 Ingredient->eraseFromParent();
95 Plan->removeVPValueFor(Inst);
96 for (auto *Def : NewRecipe->definedValues()) {
97 Plan->addVPValue(Inst, Def);
98 }
99 }
100 }
101 }
102