1 //===- ConstantHoisting.cpp - Prepare code for expensive constants --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass identifies expensive constants to hoist and coalesces them to
10 // better prepare it for SelectionDAG-based code generation. This works around
11 // the limitations of the basic-block-at-a-time approach.
12 //
13 // First it scans all instructions for integer constants and calculates its
14 // cost. If the constant can be folded into the instruction (the cost is
15 // TCC_Free) or the cost is just a simple operation (TCC_BASIC), then we don't
16 // consider it expensive and leave it alone. This is the default behavior and
17 // the default implementation of getIntImmCostInst will always return TCC_Free.
18 //
19 // If the cost is more than TCC_BASIC, then the integer constant can't be folded
20 // into the instruction and it might be beneficial to hoist the constant.
21 // Similar constants are coalesced to reduce register pressure and
22 // materialization code.
23 //
24 // When a constant is hoisted, it is also hidden behind a bitcast to force it to
25 // be live-out of the basic block. Otherwise the constant would be just
26 // duplicated and each basic block would have its own copy in the SelectionDAG.
27 // The SelectionDAG recognizes such constants as opaque and doesn't perform
28 // certain transformations on them, which would create a new expensive constant.
29 //
30 // This optimization is only applied to integer constants in instructions and
31 // simple (this means not nested) constant cast expressions. For example:
32 // %0 = load i64* inttoptr (i64 big_constant to i64*)
33 //===----------------------------------------------------------------------===//
34 
35 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
36 #include "llvm/ADT/APInt.h"
37 #include "llvm/ADT/DenseMap.h"
38 #include "llvm/ADT/SmallPtrSet.h"
39 #include "llvm/ADT/SmallVector.h"
40 #include "llvm/ADT/Statistic.h"
41 #include "llvm/Analysis/BlockFrequencyInfo.h"
42 #include "llvm/Analysis/ProfileSummaryInfo.h"
43 #include "llvm/Analysis/TargetTransformInfo.h"
44 #include "llvm/IR/BasicBlock.h"
45 #include "llvm/IR/Constants.h"
46 #include "llvm/IR/DebugInfoMetadata.h"
47 #include "llvm/IR/Dominators.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/InstrTypes.h"
50 #include "llvm/IR/Instruction.h"
51 #include "llvm/IR/Instructions.h"
52 #include "llvm/IR/IntrinsicInst.h"
53 #include "llvm/IR/Operator.h"
54 #include "llvm/IR/Value.h"
55 #include "llvm/InitializePasses.h"
56 #include "llvm/Pass.h"
57 #include "llvm/Support/BlockFrequency.h"
58 #include "llvm/Support/Casting.h"
59 #include "llvm/Support/CommandLine.h"
60 #include "llvm/Support/Debug.h"
61 #include "llvm/Support/raw_ostream.h"
62 #include "llvm/Transforms/Scalar.h"
63 #include "llvm/Transforms/Utils/Local.h"
64 #include "llvm/Transforms/Utils/SizeOpts.h"
65 #include <algorithm>
66 #include <cassert>
67 #include <cstdint>
68 #include <iterator>
69 #include <tuple>
70 #include <utility>
71 
72 using namespace llvm;
73 using namespace consthoist;
74 
75 #define DEBUG_TYPE "consthoist"
76 
77 STATISTIC(NumConstantsHoisted, "Number of constants hoisted");
78 STATISTIC(NumConstantsRebased, "Number of constants rebased");
79 
80 static cl::opt<bool> ConstHoistWithBlockFrequency(
81     "consthoist-with-block-frequency", cl::init(true), cl::Hidden,
82     cl::desc("Enable the use of the block frequency analysis to reduce the "
83              "chance to execute const materialization more frequently than "
84              "without hoisting."));
85 
86 static cl::opt<bool> ConstHoistGEP(
87     "consthoist-gep", cl::init(false), cl::Hidden,
88     cl::desc("Try hoisting constant gep expressions"));
89 
90 static cl::opt<unsigned>
91 MinNumOfDependentToRebase("consthoist-min-num-to-rebase",
92     cl::desc("Do not rebase if number of dependent constants of a Base is less "
93              "than this number."),
94     cl::init(0), cl::Hidden);
95 
96 namespace {
97 
98 /// The constant hoisting pass.
99 class ConstantHoistingLegacyPass : public FunctionPass {
100 public:
101   static char ID; // Pass identification, replacement for typeid
102 
ConstantHoistingLegacyPass()103   ConstantHoistingLegacyPass() : FunctionPass(ID) {
104     initializeConstantHoistingLegacyPassPass(*PassRegistry::getPassRegistry());
105   }
106 
107   bool runOnFunction(Function &Fn) override;
108 
getPassName() const109   StringRef getPassName() const override { return "Constant Hoisting"; }
110 
getAnalysisUsage(AnalysisUsage & AU) const111   void getAnalysisUsage(AnalysisUsage &AU) const override {
112     AU.setPreservesCFG();
113     if (ConstHoistWithBlockFrequency)
114       AU.addRequired<BlockFrequencyInfoWrapperPass>();
115     AU.addRequired<DominatorTreeWrapperPass>();
116     AU.addRequired<ProfileSummaryInfoWrapperPass>();
117     AU.addRequired<TargetTransformInfoWrapperPass>();
118   }
119 
120 private:
121   ConstantHoistingPass Impl;
122 };
123 
124 } // end anonymous namespace
125 
126 char ConstantHoistingLegacyPass::ID = 0;
127 
128 INITIALIZE_PASS_BEGIN(ConstantHoistingLegacyPass, "consthoist",
129                       "Constant Hoisting", false, false)
INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)130 INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
131 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
132 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
133 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
134 INITIALIZE_PASS_END(ConstantHoistingLegacyPass, "consthoist",
135                     "Constant Hoisting", false, false)
136 
137 FunctionPass *llvm::createConstantHoistingPass() {
138   return new ConstantHoistingLegacyPass();
139 }
140 
141 /// Perform the constant hoisting optimization for the given function.
runOnFunction(Function & Fn)142 bool ConstantHoistingLegacyPass::runOnFunction(Function &Fn) {
143   if (skipFunction(Fn))
144     return false;
145 
146   LLVM_DEBUG(dbgs() << "********** Begin Constant Hoisting **********\n");
147   LLVM_DEBUG(dbgs() << "********** Function: " << Fn.getName() << '\n');
148 
149   bool MadeChange =
150       Impl.runImpl(Fn, getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn),
151                    getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
152                    ConstHoistWithBlockFrequency
153                        ? &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI()
154                        : nullptr,
155                    Fn.getEntryBlock(),
156                    &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI());
157 
158   if (MadeChange) {
159     LLVM_DEBUG(dbgs() << "********** Function after Constant Hoisting: "
160                       << Fn.getName() << '\n');
161     LLVM_DEBUG(dbgs() << Fn);
162   }
163   LLVM_DEBUG(dbgs() << "********** End Constant Hoisting **********\n");
164 
165   return MadeChange;
166 }
167 
168 /// Find the constant materialization insertion point.
findMatInsertPt(Instruction * Inst,unsigned Idx) const169 Instruction *ConstantHoistingPass::findMatInsertPt(Instruction *Inst,
170                                                    unsigned Idx) const {
171   // If the operand is a cast instruction, then we have to materialize the
172   // constant before the cast instruction.
173   if (Idx != ~0U) {
174     Value *Opnd = Inst->getOperand(Idx);
175     if (auto CastInst = dyn_cast<Instruction>(Opnd))
176       if (CastInst->isCast())
177         return CastInst;
178   }
179 
180   // The simple and common case. This also includes constant expressions.
181   if (!isa<PHINode>(Inst) && !Inst->isEHPad())
182     return Inst;
183 
184   // We can't insert directly before a phi node or an eh pad. Insert before
185   // the terminator of the incoming or dominating block.
186   assert(Entry != Inst->getParent() && "PHI or landing pad in entry block!");
187   BasicBlock *InsertionBlock = nullptr;
188   if (Idx != ~0U && isa<PHINode>(Inst)) {
189     InsertionBlock = cast<PHINode>(Inst)->getIncomingBlock(Idx);
190     if (!InsertionBlock->isEHPad()) {
191       return InsertionBlock->getTerminator();
192     }
193   } else {
194     InsertionBlock = Inst->getParent();
195   }
196 
197   // This must be an EH pad. Iterate over immediate dominators until we find a
198   // non-EH pad. We need to skip over catchswitch blocks, which are both EH pads
199   // and terminators.
200   auto *IDom = DT->getNode(InsertionBlock)->getIDom();
201   while (IDom->getBlock()->isEHPad()) {
202     assert(Entry != IDom->getBlock() && "eh pad in entry block");
203     IDom = IDom->getIDom();
204   }
205 
206   return IDom->getBlock()->getTerminator();
207 }
208 
209 /// Given \p BBs as input, find another set of BBs which collectively
210 /// dominates \p BBs and have the minimal sum of frequencies. Return the BB
211 /// set found in \p BBs.
findBestInsertionSet(DominatorTree & DT,BlockFrequencyInfo & BFI,BasicBlock * Entry,SetVector<BasicBlock * > & BBs)212 static void findBestInsertionSet(DominatorTree &DT, BlockFrequencyInfo &BFI,
213                                  BasicBlock *Entry,
214                                  SetVector<BasicBlock *> &BBs) {
215   assert(!BBs.count(Entry) && "Assume Entry is not in BBs");
216   // Nodes on the current path to the root.
217   SmallPtrSet<BasicBlock *, 8> Path;
218   // Candidates includes any block 'BB' in set 'BBs' that is not strictly
219   // dominated by any other blocks in set 'BBs', and all nodes in the path
220   // in the dominator tree from Entry to 'BB'.
221   SmallPtrSet<BasicBlock *, 16> Candidates;
222   for (auto *BB : BBs) {
223     // Ignore unreachable basic blocks.
224     if (!DT.isReachableFromEntry(BB))
225       continue;
226     Path.clear();
227     // Walk up the dominator tree until Entry or another BB in BBs
228     // is reached. Insert the nodes on the way to the Path.
229     BasicBlock *Node = BB;
230     // The "Path" is a candidate path to be added into Candidates set.
231     bool isCandidate = false;
232     do {
233       Path.insert(Node);
234       if (Node == Entry || Candidates.count(Node)) {
235         isCandidate = true;
236         break;
237       }
238       assert(DT.getNode(Node)->getIDom() &&
239              "Entry doens't dominate current Node");
240       Node = DT.getNode(Node)->getIDom()->getBlock();
241     } while (!BBs.count(Node));
242 
243     // If isCandidate is false, Node is another Block in BBs dominating
244     // current 'BB'. Drop the nodes on the Path.
245     if (!isCandidate)
246       continue;
247 
248     // Add nodes on the Path into Candidates.
249     Candidates.insert(Path.begin(), Path.end());
250   }
251 
252   // Sort the nodes in Candidates in top-down order and save the nodes
253   // in Orders.
254   unsigned Idx = 0;
255   SmallVector<BasicBlock *, 16> Orders;
256   Orders.push_back(Entry);
257   while (Idx != Orders.size()) {
258     BasicBlock *Node = Orders[Idx++];
259     for (auto *ChildDomNode : DT.getNode(Node)->children()) {
260       if (Candidates.count(ChildDomNode->getBlock()))
261         Orders.push_back(ChildDomNode->getBlock());
262     }
263   }
264 
265   // Visit Orders in bottom-up order.
266   using InsertPtsCostPair =
267       std::pair<SetVector<BasicBlock *>, BlockFrequency>;
268 
269   // InsertPtsMap is a map from a BB to the best insertion points for the
270   // subtree of BB (subtree not including the BB itself).
271   DenseMap<BasicBlock *, InsertPtsCostPair> InsertPtsMap;
272   InsertPtsMap.reserve(Orders.size() + 1);
273   for (BasicBlock *Node : llvm::reverse(Orders)) {
274     bool NodeInBBs = BBs.count(Node);
275     auto &InsertPts = InsertPtsMap[Node].first;
276     BlockFrequency &InsertPtsFreq = InsertPtsMap[Node].second;
277 
278     // Return the optimal insert points in BBs.
279     if (Node == Entry) {
280       BBs.clear();
281       if (InsertPtsFreq > BFI.getBlockFreq(Node) ||
282           (InsertPtsFreq == BFI.getBlockFreq(Node) && InsertPts.size() > 1))
283         BBs.insert(Entry);
284       else
285         BBs.insert(InsertPts.begin(), InsertPts.end());
286       break;
287     }
288 
289     BasicBlock *Parent = DT.getNode(Node)->getIDom()->getBlock();
290     // Initially, ParentInsertPts is empty and ParentPtsFreq is 0. Every child
291     // will update its parent's ParentInsertPts and ParentPtsFreq.
292     auto &ParentInsertPts = InsertPtsMap[Parent].first;
293     BlockFrequency &ParentPtsFreq = InsertPtsMap[Parent].second;
294     // Choose to insert in Node or in subtree of Node.
295     // Don't hoist to EHPad because we may not find a proper place to insert
296     // in EHPad.
297     // If the total frequency of InsertPts is the same as the frequency of the
298     // target Node, and InsertPts contains more than one nodes, choose hoisting
299     // to reduce code size.
300     if (NodeInBBs ||
301         (!Node->isEHPad() &&
302          (InsertPtsFreq > BFI.getBlockFreq(Node) ||
303           (InsertPtsFreq == BFI.getBlockFreq(Node) && InsertPts.size() > 1)))) {
304       ParentInsertPts.insert(Node);
305       ParentPtsFreq += BFI.getBlockFreq(Node);
306     } else {
307       ParentInsertPts.insert(InsertPts.begin(), InsertPts.end());
308       ParentPtsFreq += InsertPtsFreq;
309     }
310   }
311 }
312 
313 /// Find an insertion point that dominates all uses.
findConstantInsertionPoint(const ConstantInfo & ConstInfo) const314 SetVector<Instruction *> ConstantHoistingPass::findConstantInsertionPoint(
315     const ConstantInfo &ConstInfo) const {
316   assert(!ConstInfo.RebasedConstants.empty() && "Invalid constant info entry.");
317   // Collect all basic blocks.
318   SetVector<BasicBlock *> BBs;
319   SetVector<Instruction *> InsertPts;
320   for (auto const &RCI : ConstInfo.RebasedConstants)
321     for (auto const &U : RCI.Uses)
322       BBs.insert(findMatInsertPt(U.Inst, U.OpndIdx)->getParent());
323 
324   if (BBs.count(Entry)) {
325     InsertPts.insert(&Entry->front());
326     return InsertPts;
327   }
328 
329   if (BFI) {
330     findBestInsertionSet(*DT, *BFI, Entry, BBs);
331     for (auto *BB : BBs) {
332       BasicBlock::iterator InsertPt = BB->begin();
333       for (; isa<PHINode>(InsertPt) || InsertPt->isEHPad(); ++InsertPt)
334         ;
335       InsertPts.insert(&*InsertPt);
336     }
337     return InsertPts;
338   }
339 
340   while (BBs.size() >= 2) {
341     BasicBlock *BB, *BB1, *BB2;
342     BB1 = BBs.pop_back_val();
343     BB2 = BBs.pop_back_val();
344     BB = DT->findNearestCommonDominator(BB1, BB2);
345     if (BB == Entry) {
346       InsertPts.insert(&Entry->front());
347       return InsertPts;
348     }
349     BBs.insert(BB);
350   }
351   assert((BBs.size() == 1) && "Expected only one element.");
352   Instruction &FirstInst = (*BBs.begin())->front();
353   InsertPts.insert(findMatInsertPt(&FirstInst));
354   return InsertPts;
355 }
356 
357 /// Record constant integer ConstInt for instruction Inst at operand
358 /// index Idx.
359 ///
360 /// The operand at index Idx is not necessarily the constant integer itself. It
361 /// could also be a cast instruction or a constant expression that uses the
362 /// constant integer.
collectConstantCandidates(ConstCandMapType & ConstCandMap,Instruction * Inst,unsigned Idx,ConstantInt * ConstInt)363 void ConstantHoistingPass::collectConstantCandidates(
364     ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx,
365     ConstantInt *ConstInt) {
366   InstructionCost Cost;
367   // Ask the target about the cost of materializing the constant for the given
368   // instruction and operand index.
369   if (auto IntrInst = dyn_cast<IntrinsicInst>(Inst))
370     Cost = TTI->getIntImmCostIntrin(IntrInst->getIntrinsicID(), Idx,
371                                     ConstInt->getValue(), ConstInt->getType(),
372                                     TargetTransformInfo::TCK_SizeAndLatency);
373   else
374     Cost = TTI->getIntImmCostInst(
375         Inst->getOpcode(), Idx, ConstInt->getValue(), ConstInt->getType(),
376         TargetTransformInfo::TCK_SizeAndLatency, Inst);
377 
378   // Ignore cheap integer constants.
379   if (Cost > TargetTransformInfo::TCC_Basic) {
380     ConstCandMapType::iterator Itr;
381     bool Inserted;
382     ConstPtrUnionType Cand = ConstInt;
383     std::tie(Itr, Inserted) = ConstCandMap.insert(std::make_pair(Cand, 0));
384     if (Inserted) {
385       ConstIntCandVec.push_back(ConstantCandidate(ConstInt));
386       Itr->second = ConstIntCandVec.size() - 1;
387     }
388     ConstIntCandVec[Itr->second].addUser(Inst, Idx, *Cost.getValue());
389     LLVM_DEBUG(if (isa<ConstantInt>(Inst->getOperand(Idx))) dbgs()
390                    << "Collect constant " << *ConstInt << " from " << *Inst
391                    << " with cost " << Cost << '\n';
392                else dbgs() << "Collect constant " << *ConstInt
393                            << " indirectly from " << *Inst << " via "
394                            << *Inst->getOperand(Idx) << " with cost " << Cost
395                            << '\n';);
396   }
397 }
398 
399 /// Record constant GEP expression for instruction Inst at operand index Idx.
collectConstantCandidates(ConstCandMapType & ConstCandMap,Instruction * Inst,unsigned Idx,ConstantExpr * ConstExpr)400 void ConstantHoistingPass::collectConstantCandidates(
401     ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx,
402     ConstantExpr *ConstExpr) {
403   // TODO: Handle vector GEPs
404   if (ConstExpr->getType()->isVectorTy())
405     return;
406 
407   GlobalVariable *BaseGV = dyn_cast<GlobalVariable>(ConstExpr->getOperand(0));
408   if (!BaseGV)
409     return;
410 
411   // Get offset from the base GV.
412   PointerType *GVPtrTy = cast<PointerType>(BaseGV->getType());
413   IntegerType *PtrIntTy = DL->getIntPtrType(*Ctx, GVPtrTy->getAddressSpace());
414   APInt Offset(DL->getTypeSizeInBits(PtrIntTy), /*val*/0, /*isSigned*/true);
415   auto *GEPO = cast<GEPOperator>(ConstExpr);
416 
417   // TODO: If we have a mix of inbounds and non-inbounds GEPs, then basing a
418   // non-inbounds GEP on an inbounds GEP is potentially incorrect. Restrict to
419   // inbounds GEP for now -- alternatively, we could drop inbounds from the
420   // constant expression,
421   if (!GEPO->isInBounds())
422     return;
423 
424   if (!GEPO->accumulateConstantOffset(*DL, Offset))
425     return;
426 
427   if (!Offset.isIntN(32))
428     return;
429 
430   // A constant GEP expression that has a GlobalVariable as base pointer is
431   // usually lowered to a load from constant pool. Such operation is unlikely
432   // to be cheaper than compute it by <Base + Offset>, which can be lowered to
433   // an ADD instruction or folded into Load/Store instruction.
434   InstructionCost Cost =
435       TTI->getIntImmCostInst(Instruction::Add, 1, Offset, PtrIntTy,
436                              TargetTransformInfo::TCK_SizeAndLatency, Inst);
437   ConstCandVecType &ExprCandVec = ConstGEPCandMap[BaseGV];
438   ConstCandMapType::iterator Itr;
439   bool Inserted;
440   ConstPtrUnionType Cand = ConstExpr;
441   std::tie(Itr, Inserted) = ConstCandMap.insert(std::make_pair(Cand, 0));
442   if (Inserted) {
443     ExprCandVec.push_back(ConstantCandidate(
444         ConstantInt::get(Type::getInt32Ty(*Ctx), Offset.getLimitedValue()),
445         ConstExpr));
446     Itr->second = ExprCandVec.size() - 1;
447   }
448   ExprCandVec[Itr->second].addUser(Inst, Idx, *Cost.getValue());
449 }
450 
451 /// Check the operand for instruction Inst at index Idx.
collectConstantCandidates(ConstCandMapType & ConstCandMap,Instruction * Inst,unsigned Idx)452 void ConstantHoistingPass::collectConstantCandidates(
453     ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx) {
454   Value *Opnd = Inst->getOperand(Idx);
455 
456   // Visit constant integers.
457   if (auto ConstInt = dyn_cast<ConstantInt>(Opnd)) {
458     collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
459     return;
460   }
461 
462   // Visit cast instructions that have constant integers.
463   if (auto CastInst = dyn_cast<Instruction>(Opnd)) {
464     // Only visit cast instructions, which have been skipped. All other
465     // instructions should have already been visited.
466     if (!CastInst->isCast())
467       return;
468 
469     if (auto *ConstInt = dyn_cast<ConstantInt>(CastInst->getOperand(0))) {
470       // Pretend the constant is directly used by the instruction and ignore
471       // the cast instruction.
472       collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
473       return;
474     }
475   }
476 
477   // Visit constant expressions that have constant integers.
478   if (auto ConstExpr = dyn_cast<ConstantExpr>(Opnd)) {
479     // Handle constant gep expressions.
480     if (ConstHoistGEP && isa<GEPOperator>(ConstExpr))
481       collectConstantCandidates(ConstCandMap, Inst, Idx, ConstExpr);
482 
483     // Only visit constant cast expressions.
484     if (!ConstExpr->isCast())
485       return;
486 
487     if (auto ConstInt = dyn_cast<ConstantInt>(ConstExpr->getOperand(0))) {
488       // Pretend the constant is directly used by the instruction and ignore
489       // the constant expression.
490       collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
491       return;
492     }
493   }
494 }
495 
496 /// Scan the instruction for expensive integer constants and record them
497 /// in the constant candidate vector.
collectConstantCandidates(ConstCandMapType & ConstCandMap,Instruction * Inst)498 void ConstantHoistingPass::collectConstantCandidates(
499     ConstCandMapType &ConstCandMap, Instruction *Inst) {
500   // Skip all cast instructions. They are visited indirectly later on.
501   if (Inst->isCast())
502     return;
503 
504   // Scan all operands.
505   for (unsigned Idx = 0, E = Inst->getNumOperands(); Idx != E; ++Idx) {
506     // The cost of materializing the constants (defined in
507     // `TargetTransformInfo::getIntImmCostInst`) for instructions which only
508     // take constant variables is lower than `TargetTransformInfo::TCC_Basic`.
509     // So it's safe for us to collect constant candidates from all
510     // IntrinsicInsts.
511     if (canReplaceOperandWithVariable(Inst, Idx)) {
512       collectConstantCandidates(ConstCandMap, Inst, Idx);
513     }
514   } // end of for all operands
515 }
516 
517 /// Collect all integer constants in the function that cannot be folded
518 /// into an instruction itself.
collectConstantCandidates(Function & Fn)519 void ConstantHoistingPass::collectConstantCandidates(Function &Fn) {
520   ConstCandMapType ConstCandMap;
521   for (BasicBlock &BB : Fn) {
522     // Ignore unreachable basic blocks.
523     if (!DT->isReachableFromEntry(&BB))
524       continue;
525     for (Instruction &Inst : BB)
526       collectConstantCandidates(ConstCandMap, &Inst);
527   }
528 }
529 
530 // This helper function is necessary to deal with values that have different
531 // bit widths (APInt Operator- does not like that). If the value cannot be
532 // represented in uint64 we return an "empty" APInt. This is then interpreted
533 // as the value is not in range.
calculateOffsetDiff(const APInt & V1,const APInt & V2)534 static std::optional<APInt> calculateOffsetDiff(const APInt &V1,
535                                                 const APInt &V2) {
536   std::optional<APInt> Res;
537   unsigned BW = V1.getBitWidth() > V2.getBitWidth() ?
538                 V1.getBitWidth() : V2.getBitWidth();
539   uint64_t LimVal1 = V1.getLimitedValue();
540   uint64_t LimVal2 = V2.getLimitedValue();
541 
542   if (LimVal1 == ~0ULL || LimVal2 == ~0ULL)
543     return Res;
544 
545   uint64_t Diff = LimVal1 - LimVal2;
546   return APInt(BW, Diff, true);
547 }
548 
549 // From a list of constants, one needs to picked as the base and the other
550 // constants will be transformed into an offset from that base constant. The
551 // question is which we can pick best? For example, consider these constants
552 // and their number of uses:
553 //
554 //  Constants| 2 | 4 | 12 | 42 |
555 //  NumUses  | 3 | 2 |  8 |  7 |
556 //
557 // Selecting constant 12 because it has the most uses will generate negative
558 // offsets for constants 2 and 4 (i.e. -10 and -8 respectively). If negative
559 // offsets lead to less optimal code generation, then there might be better
560 // solutions. Suppose immediates in the range of 0..35 are most optimally
561 // supported by the architecture, then selecting constant 2 is most optimal
562 // because this will generate offsets: 0, 2, 10, 40. Offsets 0, 2 and 10 are in
563 // range 0..35, and thus 3 + 2 + 8 = 13 uses are in range. Selecting 12 would
564 // have only 8 uses in range, so choosing 2 as a base is more optimal. Thus, in
565 // selecting the base constant the range of the offsets is a very important
566 // factor too that we take into account here. This algorithm calculates a total
567 // costs for selecting a constant as the base and substract the costs if
568 // immediates are out of range. It has quadratic complexity, so we call this
569 // function only when we're optimising for size and there are less than 100
570 // constants, we fall back to the straightforward algorithm otherwise
571 // which does not do all the offset calculations.
572 unsigned
maximizeConstantsInRange(ConstCandVecType::iterator S,ConstCandVecType::iterator E,ConstCandVecType::iterator & MaxCostItr)573 ConstantHoistingPass::maximizeConstantsInRange(ConstCandVecType::iterator S,
574                                            ConstCandVecType::iterator E,
575                                            ConstCandVecType::iterator &MaxCostItr) {
576   unsigned NumUses = 0;
577 
578   bool OptForSize = Entry->getParent()->hasOptSize() ||
579                     llvm::shouldOptimizeForSize(Entry->getParent(), PSI, BFI,
580                                                 PGSOQueryType::IRPass);
581   if (!OptForSize || std::distance(S,E) > 100) {
582     for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
583       NumUses += ConstCand->Uses.size();
584       if (ConstCand->CumulativeCost > MaxCostItr->CumulativeCost)
585         MaxCostItr = ConstCand;
586     }
587     return NumUses;
588   }
589 
590   LLVM_DEBUG(dbgs() << "== Maximize constants in range ==\n");
591   InstructionCost MaxCost = -1;
592   for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
593     auto Value = ConstCand->ConstInt->getValue();
594     Type *Ty = ConstCand->ConstInt->getType();
595     InstructionCost Cost = 0;
596     NumUses += ConstCand->Uses.size();
597     LLVM_DEBUG(dbgs() << "= Constant: " << ConstCand->ConstInt->getValue()
598                       << "\n");
599 
600     for (auto User : ConstCand->Uses) {
601       unsigned Opcode = User.Inst->getOpcode();
602       unsigned OpndIdx = User.OpndIdx;
603       Cost += TTI->getIntImmCostInst(Opcode, OpndIdx, Value, Ty,
604                                      TargetTransformInfo::TCK_SizeAndLatency);
605       LLVM_DEBUG(dbgs() << "Cost: " << Cost << "\n");
606 
607       for (auto C2 = S; C2 != E; ++C2) {
608         std::optional<APInt> Diff = calculateOffsetDiff(
609             C2->ConstInt->getValue(), ConstCand->ConstInt->getValue());
610         if (Diff) {
611           const InstructionCost ImmCosts =
612               TTI->getIntImmCodeSizeCost(Opcode, OpndIdx, *Diff, Ty);
613           Cost -= ImmCosts;
614           LLVM_DEBUG(dbgs() << "Offset " << *Diff << " "
615                             << "has penalty: " << ImmCosts << "\n"
616                             << "Adjusted cost: " << Cost << "\n");
617         }
618       }
619     }
620     LLVM_DEBUG(dbgs() << "Cumulative cost: " << Cost << "\n");
621     if (Cost > MaxCost) {
622       MaxCost = Cost;
623       MaxCostItr = ConstCand;
624       LLVM_DEBUG(dbgs() << "New candidate: " << MaxCostItr->ConstInt->getValue()
625                         << "\n");
626     }
627   }
628   return NumUses;
629 }
630 
631 /// Find the base constant within the given range and rebase all other
632 /// constants with respect to the base constant.
findAndMakeBaseConstant(ConstCandVecType::iterator S,ConstCandVecType::iterator E,SmallVectorImpl<consthoist::ConstantInfo> & ConstInfoVec)633 void ConstantHoistingPass::findAndMakeBaseConstant(
634     ConstCandVecType::iterator S, ConstCandVecType::iterator E,
635     SmallVectorImpl<consthoist::ConstantInfo> &ConstInfoVec) {
636   auto MaxCostItr = S;
637   unsigned NumUses = maximizeConstantsInRange(S, E, MaxCostItr);
638 
639   // Don't hoist constants that have only one use.
640   if (NumUses <= 1)
641     return;
642 
643   ConstantInt *ConstInt = MaxCostItr->ConstInt;
644   ConstantExpr *ConstExpr = MaxCostItr->ConstExpr;
645   ConstantInfo ConstInfo;
646   ConstInfo.BaseInt = ConstInt;
647   ConstInfo.BaseExpr = ConstExpr;
648   Type *Ty = ConstInt->getType();
649 
650   // Rebase the constants with respect to the base constant.
651   for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
652     APInt Diff = ConstCand->ConstInt->getValue() - ConstInt->getValue();
653     Constant *Offset = Diff == 0 ? nullptr : ConstantInt::get(Ty, Diff);
654     Type *ConstTy =
655         ConstCand->ConstExpr ? ConstCand->ConstExpr->getType() : nullptr;
656     ConstInfo.RebasedConstants.push_back(
657       RebasedConstantInfo(std::move(ConstCand->Uses), Offset, ConstTy));
658   }
659   ConstInfoVec.push_back(std::move(ConstInfo));
660 }
661 
662 /// Finds and combines constant candidates that can be easily
663 /// rematerialized with an add from a common base constant.
findBaseConstants(GlobalVariable * BaseGV)664 void ConstantHoistingPass::findBaseConstants(GlobalVariable *BaseGV) {
665   // If BaseGV is nullptr, find base among candidate constant integers;
666   // Otherwise find base among constant GEPs that share the same BaseGV.
667   ConstCandVecType &ConstCandVec = BaseGV ?
668       ConstGEPCandMap[BaseGV] : ConstIntCandVec;
669   ConstInfoVecType &ConstInfoVec = BaseGV ?
670       ConstGEPInfoMap[BaseGV] : ConstIntInfoVec;
671 
672   // Sort the constants by value and type. This invalidates the mapping!
673   llvm::stable_sort(ConstCandVec, [](const ConstantCandidate &LHS,
674                                      const ConstantCandidate &RHS) {
675     if (LHS.ConstInt->getType() != RHS.ConstInt->getType())
676       return LHS.ConstInt->getType()->getBitWidth() <
677              RHS.ConstInt->getType()->getBitWidth();
678     return LHS.ConstInt->getValue().ult(RHS.ConstInt->getValue());
679   });
680 
681   // Simple linear scan through the sorted constant candidate vector for viable
682   // merge candidates.
683   auto MinValItr = ConstCandVec.begin();
684   for (auto CC = std::next(ConstCandVec.begin()), E = ConstCandVec.end();
685        CC != E; ++CC) {
686     if (MinValItr->ConstInt->getType() == CC->ConstInt->getType()) {
687       Type *MemUseValTy = nullptr;
688       for (auto &U : CC->Uses) {
689         auto *UI = U.Inst;
690         if (LoadInst *LI = dyn_cast<LoadInst>(UI)) {
691           MemUseValTy = LI->getType();
692           break;
693         } else if (StoreInst *SI = dyn_cast<StoreInst>(UI)) {
694           // Make sure the constant is used as pointer operand of the StoreInst.
695           if (SI->getPointerOperand() == SI->getOperand(U.OpndIdx)) {
696             MemUseValTy = SI->getValueOperand()->getType();
697             break;
698           }
699         }
700       }
701 
702       // Check if the constant is in range of an add with immediate.
703       APInt Diff = CC->ConstInt->getValue() - MinValItr->ConstInt->getValue();
704       if ((Diff.getBitWidth() <= 64) &&
705           TTI->isLegalAddImmediate(Diff.getSExtValue()) &&
706           // Check if Diff can be used as offset in addressing mode of the user
707           // memory instruction.
708           (!MemUseValTy || TTI->isLegalAddressingMode(MemUseValTy,
709            /*BaseGV*/nullptr, /*BaseOffset*/Diff.getSExtValue(),
710            /*HasBaseReg*/true, /*Scale*/0)))
711         continue;
712     }
713     // We either have now a different constant type or the constant is not in
714     // range of an add with immediate anymore.
715     findAndMakeBaseConstant(MinValItr, CC, ConstInfoVec);
716     // Start a new base constant search.
717     MinValItr = CC;
718   }
719   // Finalize the last base constant search.
720   findAndMakeBaseConstant(MinValItr, ConstCandVec.end(), ConstInfoVec);
721 }
722 
723 /// Updates the operand at Idx in instruction Inst with the result of
724 ///        instruction Mat. If the instruction is a PHI node then special
725 ///        handling for duplicate values from the same incoming basic block is
726 ///        required.
727 /// \return The update will always succeed, but the return value indicated if
728 ///         Mat was used for the update or not.
updateOperand(Instruction * Inst,unsigned Idx,Instruction * Mat)729 static bool updateOperand(Instruction *Inst, unsigned Idx, Instruction *Mat) {
730   if (auto PHI = dyn_cast<PHINode>(Inst)) {
731     // Check if any previous operand of the PHI node has the same incoming basic
732     // block. This is a very odd case that happens when the incoming basic block
733     // has a switch statement. In this case use the same value as the previous
734     // operand(s), otherwise we will fail verification due to different values.
735     // The values are actually the same, but the variable names are different
736     // and the verifier doesn't like that.
737     BasicBlock *IncomingBB = PHI->getIncomingBlock(Idx);
738     for (unsigned i = 0; i < Idx; ++i) {
739       if (PHI->getIncomingBlock(i) == IncomingBB) {
740         Value *IncomingVal = PHI->getIncomingValue(i);
741         Inst->setOperand(Idx, IncomingVal);
742         return false;
743       }
744     }
745   }
746 
747   Inst->setOperand(Idx, Mat);
748   return true;
749 }
750 
751 /// Emit materialization code for all rebased constants and update their
752 /// users.
emitBaseConstants(Instruction * Base,Constant * Offset,Type * Ty,const ConstantUser & ConstUser)753 void ConstantHoistingPass::emitBaseConstants(Instruction *Base,
754                                              Constant *Offset,
755                                              Type *Ty,
756                                              const ConstantUser &ConstUser) {
757   Instruction *Mat = Base;
758 
759   // The same offset can be dereferenced to different types in nested struct.
760   if (!Offset && Ty && Ty != Base->getType())
761     Offset = ConstantInt::get(Type::getInt32Ty(*Ctx), 0);
762 
763   if (Offset) {
764     Instruction *InsertionPt = findMatInsertPt(ConstUser.Inst,
765                                                ConstUser.OpndIdx);
766     if (Ty) {
767       // Constant being rebased is a ConstantExpr.
768       PointerType *Int8PtrTy = Type::getInt8PtrTy(*Ctx,
769           cast<PointerType>(Ty)->getAddressSpace());
770       Base = new BitCastInst(Base, Int8PtrTy, "base_bitcast", InsertionPt);
771       Mat = GetElementPtrInst::Create(Type::getInt8Ty(*Ctx), Base,
772           Offset, "mat_gep", InsertionPt);
773       Mat = new BitCastInst(Mat, Ty, "mat_bitcast", InsertionPt);
774     } else
775       // Constant being rebased is a ConstantInt.
776       Mat = BinaryOperator::Create(Instruction::Add, Base, Offset,
777                                  "const_mat", InsertionPt);
778 
779     LLVM_DEBUG(dbgs() << "Materialize constant (" << *Base->getOperand(0)
780                       << " + " << *Offset << ") in BB "
781                       << Mat->getParent()->getName() << '\n'
782                       << *Mat << '\n');
783     Mat->setDebugLoc(ConstUser.Inst->getDebugLoc());
784   }
785   Value *Opnd = ConstUser.Inst->getOperand(ConstUser.OpndIdx);
786 
787   // Visit constant integer.
788   if (isa<ConstantInt>(Opnd)) {
789     LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n');
790     if (!updateOperand(ConstUser.Inst, ConstUser.OpndIdx, Mat) && Offset)
791       Mat->eraseFromParent();
792     LLVM_DEBUG(dbgs() << "To    : " << *ConstUser.Inst << '\n');
793     return;
794   }
795 
796   // Visit cast instruction.
797   if (auto CastInst = dyn_cast<Instruction>(Opnd)) {
798     assert(CastInst->isCast() && "Expected an cast instruction!");
799     // Check if we already have visited this cast instruction before to avoid
800     // unnecessary cloning.
801     Instruction *&ClonedCastInst = ClonedCastMap[CastInst];
802     if (!ClonedCastInst) {
803       ClonedCastInst = CastInst->clone();
804       ClonedCastInst->setOperand(0, Mat);
805       ClonedCastInst->insertAfter(CastInst);
806       // Use the same debug location as the original cast instruction.
807       ClonedCastInst->setDebugLoc(CastInst->getDebugLoc());
808       LLVM_DEBUG(dbgs() << "Clone instruction: " << *CastInst << '\n'
809                         << "To               : " << *ClonedCastInst << '\n');
810     }
811 
812     LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n');
813     updateOperand(ConstUser.Inst, ConstUser.OpndIdx, ClonedCastInst);
814     LLVM_DEBUG(dbgs() << "To    : " << *ConstUser.Inst << '\n');
815     return;
816   }
817 
818   // Visit constant expression.
819   if (auto ConstExpr = dyn_cast<ConstantExpr>(Opnd)) {
820     if (isa<GEPOperator>(ConstExpr)) {
821       // Operand is a ConstantGEP, replace it.
822       updateOperand(ConstUser.Inst, ConstUser.OpndIdx, Mat);
823       return;
824     }
825 
826     // Aside from constant GEPs, only constant cast expressions are collected.
827     assert(ConstExpr->isCast() && "ConstExpr should be a cast");
828     Instruction *ConstExprInst = ConstExpr->getAsInstruction(
829         findMatInsertPt(ConstUser.Inst, ConstUser.OpndIdx));
830     ConstExprInst->setOperand(0, Mat);
831 
832     // Use the same debug location as the instruction we are about to update.
833     ConstExprInst->setDebugLoc(ConstUser.Inst->getDebugLoc());
834 
835     LLVM_DEBUG(dbgs() << "Create instruction: " << *ConstExprInst << '\n'
836                       << "From              : " << *ConstExpr << '\n');
837     LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n');
838     if (!updateOperand(ConstUser.Inst, ConstUser.OpndIdx, ConstExprInst)) {
839       ConstExprInst->eraseFromParent();
840       if (Offset)
841         Mat->eraseFromParent();
842     }
843     LLVM_DEBUG(dbgs() << "To    : " << *ConstUser.Inst << '\n');
844     return;
845   }
846 }
847 
848 /// Hoist and hide the base constant behind a bitcast and emit
849 /// materialization code for derived constants.
emitBaseConstants(GlobalVariable * BaseGV)850 bool ConstantHoistingPass::emitBaseConstants(GlobalVariable *BaseGV) {
851   bool MadeChange = false;
852   SmallVectorImpl<consthoist::ConstantInfo> &ConstInfoVec =
853       BaseGV ? ConstGEPInfoMap[BaseGV] : ConstIntInfoVec;
854   for (auto const &ConstInfo : ConstInfoVec) {
855     SetVector<Instruction *> IPSet = findConstantInsertionPoint(ConstInfo);
856     // We can have an empty set if the function contains unreachable blocks.
857     if (IPSet.empty())
858       continue;
859 
860     unsigned UsesNum = 0;
861     unsigned ReBasesNum = 0;
862     unsigned NotRebasedNum = 0;
863     for (Instruction *IP : IPSet) {
864       // First, collect constants depending on this IP of the base.
865       unsigned Uses = 0;
866       using RebasedUse = std::tuple<Constant *, Type *, ConstantUser>;
867       SmallVector<RebasedUse, 4> ToBeRebased;
868       for (auto const &RCI : ConstInfo.RebasedConstants) {
869         for (auto const &U : RCI.Uses) {
870           Uses++;
871           BasicBlock *OrigMatInsertBB =
872               findMatInsertPt(U.Inst, U.OpndIdx)->getParent();
873           // If Base constant is to be inserted in multiple places,
874           // generate rebase for U using the Base dominating U.
875           if (IPSet.size() == 1 ||
876               DT->dominates(IP->getParent(), OrigMatInsertBB))
877             ToBeRebased.push_back(RebasedUse(RCI.Offset, RCI.Ty, U));
878         }
879       }
880       UsesNum = Uses;
881 
882       // If only few constants depend on this IP of base, skip rebasing,
883       // assuming the base and the rebased have the same materialization cost.
884       if (ToBeRebased.size() < MinNumOfDependentToRebase) {
885         NotRebasedNum += ToBeRebased.size();
886         continue;
887       }
888 
889       // Emit an instance of the base at this IP.
890       Instruction *Base = nullptr;
891       // Hoist and hide the base constant behind a bitcast.
892       if (ConstInfo.BaseExpr) {
893         assert(BaseGV && "A base constant expression must have an base GV");
894         Type *Ty = ConstInfo.BaseExpr->getType();
895         Base = new BitCastInst(ConstInfo.BaseExpr, Ty, "const", IP);
896       } else {
897         IntegerType *Ty = ConstInfo.BaseInt->getType();
898         Base = new BitCastInst(ConstInfo.BaseInt, Ty, "const", IP);
899       }
900 
901       Base->setDebugLoc(IP->getDebugLoc());
902 
903       LLVM_DEBUG(dbgs() << "Hoist constant (" << *ConstInfo.BaseInt
904                         << ") to BB " << IP->getParent()->getName() << '\n'
905                         << *Base << '\n');
906 
907       // Emit materialization code for rebased constants depending on this IP.
908       for (auto const &R : ToBeRebased) {
909         Constant *Off = std::get<0>(R);
910         Type *Ty = std::get<1>(R);
911         ConstantUser U = std::get<2>(R);
912         emitBaseConstants(Base, Off, Ty, U);
913         ReBasesNum++;
914         // Use the same debug location as the last user of the constant.
915         Base->setDebugLoc(DILocation::getMergedLocation(
916             Base->getDebugLoc(), U.Inst->getDebugLoc()));
917       }
918       assert(!Base->use_empty() && "The use list is empty!?");
919       assert(isa<Instruction>(Base->user_back()) &&
920              "All uses should be instructions.");
921     }
922     (void)UsesNum;
923     (void)ReBasesNum;
924     (void)NotRebasedNum;
925     // Expect all uses are rebased after rebase is done.
926     assert(UsesNum == (ReBasesNum + NotRebasedNum) &&
927            "Not all uses are rebased");
928 
929     NumConstantsHoisted++;
930 
931     // Base constant is also included in ConstInfo.RebasedConstants, so
932     // deduct 1 from ConstInfo.RebasedConstants.size().
933     NumConstantsRebased += ConstInfo.RebasedConstants.size() - 1;
934 
935     MadeChange = true;
936   }
937   return MadeChange;
938 }
939 
940 /// Check all cast instructions we made a copy of and remove them if they
941 /// have no more users.
deleteDeadCastInst() const942 void ConstantHoistingPass::deleteDeadCastInst() const {
943   for (auto const &I : ClonedCastMap)
944     if (I.first->use_empty())
945       I.first->eraseFromParent();
946 }
947 
948 /// Optimize expensive integer constants in the given function.
runImpl(Function & Fn,TargetTransformInfo & TTI,DominatorTree & DT,BlockFrequencyInfo * BFI,BasicBlock & Entry,ProfileSummaryInfo * PSI)949 bool ConstantHoistingPass::runImpl(Function &Fn, TargetTransformInfo &TTI,
950                                    DominatorTree &DT, BlockFrequencyInfo *BFI,
951                                    BasicBlock &Entry, ProfileSummaryInfo *PSI) {
952   this->TTI = &TTI;
953   this->DT = &DT;
954   this->BFI = BFI;
955   this->DL = &Fn.getParent()->getDataLayout();
956   this->Ctx = &Fn.getContext();
957   this->Entry = &Entry;
958   this->PSI = PSI;
959   // Collect all constant candidates.
960   collectConstantCandidates(Fn);
961 
962   // Combine constants that can be easily materialized with an add from a common
963   // base constant.
964   if (!ConstIntCandVec.empty())
965     findBaseConstants(nullptr);
966   for (const auto &MapEntry : ConstGEPCandMap)
967     if (!MapEntry.second.empty())
968       findBaseConstants(MapEntry.first);
969 
970   // Finally hoist the base constant and emit materialization code for dependent
971   // constants.
972   bool MadeChange = false;
973   if (!ConstIntInfoVec.empty())
974     MadeChange = emitBaseConstants(nullptr);
975   for (const auto &MapEntry : ConstGEPInfoMap)
976     if (!MapEntry.second.empty())
977       MadeChange |= emitBaseConstants(MapEntry.first);
978 
979 
980   // Cleanup dead instructions.
981   deleteDeadCastInst();
982 
983   cleanup();
984 
985   return MadeChange;
986 }
987 
run(Function & F,FunctionAnalysisManager & AM)988 PreservedAnalyses ConstantHoistingPass::run(Function &F,
989                                             FunctionAnalysisManager &AM) {
990   auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
991   auto &TTI = AM.getResult<TargetIRAnalysis>(F);
992   auto BFI = ConstHoistWithBlockFrequency
993                  ? &AM.getResult<BlockFrequencyAnalysis>(F)
994                  : nullptr;
995   auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
996   auto *PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
997   if (!runImpl(F, TTI, DT, BFI, F.getEntryBlock(), PSI))
998     return PreservedAnalyses::all();
999 
1000   PreservedAnalyses PA;
1001   PA.preserveSet<CFGAnalyses>();
1002   return PA;
1003 }
1004