1 //===- ConstantHoisting.cpp - Prepare code for expensive constants --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass identifies expensive constants to hoist and coalesces them to
10 // better prepare it for SelectionDAG-based code generation. This works around
11 // the limitations of the basic-block-at-a-time approach.
12 //
13 // First it scans all instructions for integer constants and calculates its
14 // cost. If the constant can be folded into the instruction (the cost is
15 // TCC_Free) or the cost is just a simple operation (TCC_BASIC), then we don't
16 // consider it expensive and leave it alone. This is the default behavior and
17 // the default implementation of getIntImmCostInst will always return TCC_Free.
18 //
19 // If the cost is more than TCC_BASIC, then the integer constant can't be folded
20 // into the instruction and it might be beneficial to hoist the constant.
21 // Similar constants are coalesced to reduce register pressure and
22 // materialization code.
23 //
24 // When a constant is hoisted, it is also hidden behind a bitcast to force it to
25 // be live-out of the basic block. Otherwise the constant would be just
26 // duplicated and each basic block would have its own copy in the SelectionDAG.
27 // The SelectionDAG recognizes such constants as opaque and doesn't perform
28 // certain transformations on them, which would create a new expensive constant.
29 //
30 // This optimization is only applied to integer constants in instructions and
31 // simple (this means not nested) constant cast expressions. For example:
32 // %0 = load i64* inttoptr (i64 big_constant to i64*)
33 //===----------------------------------------------------------------------===//
34
35 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
36 #include "llvm/ADT/APInt.h"
37 #include "llvm/ADT/DenseMap.h"
38 #include "llvm/ADT/None.h"
39 #include "llvm/ADT/Optional.h"
40 #include "llvm/ADT/SmallPtrSet.h"
41 #include "llvm/ADT/SmallVector.h"
42 #include "llvm/ADT/Statistic.h"
43 #include "llvm/Analysis/BlockFrequencyInfo.h"
44 #include "llvm/Analysis/ProfileSummaryInfo.h"
45 #include "llvm/Analysis/TargetTransformInfo.h"
46 #include "llvm/IR/BasicBlock.h"
47 #include "llvm/IR/Constants.h"
48 #include "llvm/IR/DebugInfoMetadata.h"
49 #include "llvm/IR/Dominators.h"
50 #include "llvm/IR/Function.h"
51 #include "llvm/IR/InstrTypes.h"
52 #include "llvm/IR/Instruction.h"
53 #include "llvm/IR/Instructions.h"
54 #include "llvm/IR/IntrinsicInst.h"
55 #include "llvm/IR/Value.h"
56 #include "llvm/InitializePasses.h"
57 #include "llvm/Pass.h"
58 #include "llvm/Support/BlockFrequency.h"
59 #include "llvm/Support/Casting.h"
60 #include "llvm/Support/CommandLine.h"
61 #include "llvm/Support/Debug.h"
62 #include "llvm/Support/raw_ostream.h"
63 #include "llvm/Transforms/Scalar.h"
64 #include "llvm/Transforms/Utils/Local.h"
65 #include "llvm/Transforms/Utils/SizeOpts.h"
66 #include <algorithm>
67 #include <cassert>
68 #include <cstdint>
69 #include <iterator>
70 #include <tuple>
71 #include <utility>
72
73 using namespace llvm;
74 using namespace consthoist;
75
76 #define DEBUG_TYPE "consthoist"
77
78 STATISTIC(NumConstantsHoisted, "Number of constants hoisted");
79 STATISTIC(NumConstantsRebased, "Number of constants rebased");
80
81 static cl::opt<bool> ConstHoistWithBlockFrequency(
82 "consthoist-with-block-frequency", cl::init(true), cl::Hidden,
83 cl::desc("Enable the use of the block frequency analysis to reduce the "
84 "chance to execute const materialization more frequently than "
85 "without hoisting."));
86
87 static cl::opt<bool> ConstHoistGEP(
88 "consthoist-gep", cl::init(false), cl::Hidden,
89 cl::desc("Try hoisting constant gep expressions"));
90
91 static cl::opt<unsigned>
92 MinNumOfDependentToRebase("consthoist-min-num-to-rebase",
93 cl::desc("Do not rebase if number of dependent constants of a Base is less "
94 "than this number."),
95 cl::init(0), cl::Hidden);
96
97 namespace {
98
99 /// The constant hoisting pass.
100 class ConstantHoistingLegacyPass : public FunctionPass {
101 public:
102 static char ID; // Pass identification, replacement for typeid
103
ConstantHoistingLegacyPass()104 ConstantHoistingLegacyPass() : FunctionPass(ID) {
105 initializeConstantHoistingLegacyPassPass(*PassRegistry::getPassRegistry());
106 }
107
108 bool runOnFunction(Function &Fn) override;
109
getPassName() const110 StringRef getPassName() const override { return "Constant Hoisting"; }
111
getAnalysisUsage(AnalysisUsage & AU) const112 void getAnalysisUsage(AnalysisUsage &AU) const override {
113 AU.setPreservesCFG();
114 if (ConstHoistWithBlockFrequency)
115 AU.addRequired<BlockFrequencyInfoWrapperPass>();
116 AU.addRequired<DominatorTreeWrapperPass>();
117 AU.addRequired<ProfileSummaryInfoWrapperPass>();
118 AU.addRequired<TargetTransformInfoWrapperPass>();
119 }
120
121 private:
122 ConstantHoistingPass Impl;
123 };
124
125 } // end anonymous namespace
126
127 char ConstantHoistingLegacyPass::ID = 0;
128
129 INITIALIZE_PASS_BEGIN(ConstantHoistingLegacyPass, "consthoist",
130 "Constant Hoisting", false, false)
INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)131 INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
132 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
133 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
134 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
135 INITIALIZE_PASS_END(ConstantHoistingLegacyPass, "consthoist",
136 "Constant Hoisting", false, false)
137
138 FunctionPass *llvm::createConstantHoistingPass() {
139 return new ConstantHoistingLegacyPass();
140 }
141
142 /// Perform the constant hoisting optimization for the given function.
runOnFunction(Function & Fn)143 bool ConstantHoistingLegacyPass::runOnFunction(Function &Fn) {
144 if (skipFunction(Fn))
145 return false;
146
147 LLVM_DEBUG(dbgs() << "********** Begin Constant Hoisting **********\n");
148 LLVM_DEBUG(dbgs() << "********** Function: " << Fn.getName() << '\n');
149
150 bool MadeChange =
151 Impl.runImpl(Fn, getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn),
152 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
153 ConstHoistWithBlockFrequency
154 ? &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI()
155 : nullptr,
156 Fn.getEntryBlock(),
157 &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI());
158
159 if (MadeChange) {
160 LLVM_DEBUG(dbgs() << "********** Function after Constant Hoisting: "
161 << Fn.getName() << '\n');
162 LLVM_DEBUG(dbgs() << Fn);
163 }
164 LLVM_DEBUG(dbgs() << "********** End Constant Hoisting **********\n");
165
166 return MadeChange;
167 }
168
169 /// Find the constant materialization insertion point.
findMatInsertPt(Instruction * Inst,unsigned Idx) const170 Instruction *ConstantHoistingPass::findMatInsertPt(Instruction *Inst,
171 unsigned Idx) const {
172 // If the operand is a cast instruction, then we have to materialize the
173 // constant before the cast instruction.
174 if (Idx != ~0U) {
175 Value *Opnd = Inst->getOperand(Idx);
176 if (auto CastInst = dyn_cast<Instruction>(Opnd))
177 if (CastInst->isCast())
178 return CastInst;
179 }
180
181 // The simple and common case. This also includes constant expressions.
182 if (!isa<PHINode>(Inst) && !Inst->isEHPad())
183 return Inst;
184
185 // We can't insert directly before a phi node or an eh pad. Insert before
186 // the terminator of the incoming or dominating block.
187 assert(Entry != Inst->getParent() && "PHI or landing pad in entry block!");
188 if (Idx != ~0U && isa<PHINode>(Inst))
189 return cast<PHINode>(Inst)->getIncomingBlock(Idx)->getTerminator();
190
191 // This must be an EH pad. Iterate over immediate dominators until we find a
192 // non-EH pad. We need to skip over catchswitch blocks, which are both EH pads
193 // and terminators.
194 auto IDom = DT->getNode(Inst->getParent())->getIDom();
195 while (IDom->getBlock()->isEHPad()) {
196 assert(Entry != IDom->getBlock() && "eh pad in entry block");
197 IDom = IDom->getIDom();
198 }
199
200 return IDom->getBlock()->getTerminator();
201 }
202
203 /// Given \p BBs as input, find another set of BBs which collectively
204 /// dominates \p BBs and have the minimal sum of frequencies. Return the BB
205 /// set found in \p BBs.
findBestInsertionSet(DominatorTree & DT,BlockFrequencyInfo & BFI,BasicBlock * Entry,SetVector<BasicBlock * > & BBs)206 static void findBestInsertionSet(DominatorTree &DT, BlockFrequencyInfo &BFI,
207 BasicBlock *Entry,
208 SetVector<BasicBlock *> &BBs) {
209 assert(!BBs.count(Entry) && "Assume Entry is not in BBs");
210 // Nodes on the current path to the root.
211 SmallPtrSet<BasicBlock *, 8> Path;
212 // Candidates includes any block 'BB' in set 'BBs' that is not strictly
213 // dominated by any other blocks in set 'BBs', and all nodes in the path
214 // in the dominator tree from Entry to 'BB'.
215 SmallPtrSet<BasicBlock *, 16> Candidates;
216 for (auto BB : BBs) {
217 // Ignore unreachable basic blocks.
218 if (!DT.isReachableFromEntry(BB))
219 continue;
220 Path.clear();
221 // Walk up the dominator tree until Entry or another BB in BBs
222 // is reached. Insert the nodes on the way to the Path.
223 BasicBlock *Node = BB;
224 // The "Path" is a candidate path to be added into Candidates set.
225 bool isCandidate = false;
226 do {
227 Path.insert(Node);
228 if (Node == Entry || Candidates.count(Node)) {
229 isCandidate = true;
230 break;
231 }
232 assert(DT.getNode(Node)->getIDom() &&
233 "Entry doens't dominate current Node");
234 Node = DT.getNode(Node)->getIDom()->getBlock();
235 } while (!BBs.count(Node));
236
237 // If isCandidate is false, Node is another Block in BBs dominating
238 // current 'BB'. Drop the nodes on the Path.
239 if (!isCandidate)
240 continue;
241
242 // Add nodes on the Path into Candidates.
243 Candidates.insert(Path.begin(), Path.end());
244 }
245
246 // Sort the nodes in Candidates in top-down order and save the nodes
247 // in Orders.
248 unsigned Idx = 0;
249 SmallVector<BasicBlock *, 16> Orders;
250 Orders.push_back(Entry);
251 while (Idx != Orders.size()) {
252 BasicBlock *Node = Orders[Idx++];
253 for (auto ChildDomNode : DT.getNode(Node)->children()) {
254 if (Candidates.count(ChildDomNode->getBlock()))
255 Orders.push_back(ChildDomNode->getBlock());
256 }
257 }
258
259 // Visit Orders in bottom-up order.
260 using InsertPtsCostPair =
261 std::pair<SetVector<BasicBlock *>, BlockFrequency>;
262
263 // InsertPtsMap is a map from a BB to the best insertion points for the
264 // subtree of BB (subtree not including the BB itself).
265 DenseMap<BasicBlock *, InsertPtsCostPair> InsertPtsMap;
266 InsertPtsMap.reserve(Orders.size() + 1);
267 for (auto RIt = Orders.rbegin(); RIt != Orders.rend(); RIt++) {
268 BasicBlock *Node = *RIt;
269 bool NodeInBBs = BBs.count(Node);
270 auto &InsertPts = InsertPtsMap[Node].first;
271 BlockFrequency &InsertPtsFreq = InsertPtsMap[Node].second;
272
273 // Return the optimal insert points in BBs.
274 if (Node == Entry) {
275 BBs.clear();
276 if (InsertPtsFreq > BFI.getBlockFreq(Node) ||
277 (InsertPtsFreq == BFI.getBlockFreq(Node) && InsertPts.size() > 1))
278 BBs.insert(Entry);
279 else
280 BBs.insert(InsertPts.begin(), InsertPts.end());
281 break;
282 }
283
284 BasicBlock *Parent = DT.getNode(Node)->getIDom()->getBlock();
285 // Initially, ParentInsertPts is empty and ParentPtsFreq is 0. Every child
286 // will update its parent's ParentInsertPts and ParentPtsFreq.
287 auto &ParentInsertPts = InsertPtsMap[Parent].first;
288 BlockFrequency &ParentPtsFreq = InsertPtsMap[Parent].second;
289 // Choose to insert in Node or in subtree of Node.
290 // Don't hoist to EHPad because we may not find a proper place to insert
291 // in EHPad.
292 // If the total frequency of InsertPts is the same as the frequency of the
293 // target Node, and InsertPts contains more than one nodes, choose hoisting
294 // to reduce code size.
295 if (NodeInBBs ||
296 (!Node->isEHPad() &&
297 (InsertPtsFreq > BFI.getBlockFreq(Node) ||
298 (InsertPtsFreq == BFI.getBlockFreq(Node) && InsertPts.size() > 1)))) {
299 ParentInsertPts.insert(Node);
300 ParentPtsFreq += BFI.getBlockFreq(Node);
301 } else {
302 ParentInsertPts.insert(InsertPts.begin(), InsertPts.end());
303 ParentPtsFreq += InsertPtsFreq;
304 }
305 }
306 }
307
308 /// Find an insertion point that dominates all uses.
findConstantInsertionPoint(const ConstantInfo & ConstInfo) const309 SetVector<Instruction *> ConstantHoistingPass::findConstantInsertionPoint(
310 const ConstantInfo &ConstInfo) const {
311 assert(!ConstInfo.RebasedConstants.empty() && "Invalid constant info entry.");
312 // Collect all basic blocks.
313 SetVector<BasicBlock *> BBs;
314 SetVector<Instruction *> InsertPts;
315 for (auto const &RCI : ConstInfo.RebasedConstants)
316 for (auto const &U : RCI.Uses)
317 BBs.insert(findMatInsertPt(U.Inst, U.OpndIdx)->getParent());
318
319 if (BBs.count(Entry)) {
320 InsertPts.insert(&Entry->front());
321 return InsertPts;
322 }
323
324 if (BFI) {
325 findBestInsertionSet(*DT, *BFI, Entry, BBs);
326 for (auto BB : BBs) {
327 BasicBlock::iterator InsertPt = BB->begin();
328 for (; isa<PHINode>(InsertPt) || InsertPt->isEHPad(); ++InsertPt)
329 ;
330 InsertPts.insert(&*InsertPt);
331 }
332 return InsertPts;
333 }
334
335 while (BBs.size() >= 2) {
336 BasicBlock *BB, *BB1, *BB2;
337 BB1 = BBs.pop_back_val();
338 BB2 = BBs.pop_back_val();
339 BB = DT->findNearestCommonDominator(BB1, BB2);
340 if (BB == Entry) {
341 InsertPts.insert(&Entry->front());
342 return InsertPts;
343 }
344 BBs.insert(BB);
345 }
346 assert((BBs.size() == 1) && "Expected only one element.");
347 Instruction &FirstInst = (*BBs.begin())->front();
348 InsertPts.insert(findMatInsertPt(&FirstInst));
349 return InsertPts;
350 }
351
352 /// Record constant integer ConstInt for instruction Inst at operand
353 /// index Idx.
354 ///
355 /// The operand at index Idx is not necessarily the constant integer itself. It
356 /// could also be a cast instruction or a constant expression that uses the
357 /// constant integer.
collectConstantCandidates(ConstCandMapType & ConstCandMap,Instruction * Inst,unsigned Idx,ConstantInt * ConstInt)358 void ConstantHoistingPass::collectConstantCandidates(
359 ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx,
360 ConstantInt *ConstInt) {
361 unsigned Cost;
362 // Ask the target about the cost of materializing the constant for the given
363 // instruction and operand index.
364 if (auto IntrInst = dyn_cast<IntrinsicInst>(Inst))
365 Cost = TTI->getIntImmCostIntrin(IntrInst->getIntrinsicID(), Idx,
366 ConstInt->getValue(), ConstInt->getType(),
367 TargetTransformInfo::TCK_SizeAndLatency);
368 else
369 Cost = TTI->getIntImmCostInst(
370 Inst->getOpcode(), Idx, ConstInt->getValue(), ConstInt->getType(),
371 TargetTransformInfo::TCK_SizeAndLatency, Inst);
372
373 // Ignore cheap integer constants.
374 if (Cost > TargetTransformInfo::TCC_Basic) {
375 ConstCandMapType::iterator Itr;
376 bool Inserted;
377 ConstPtrUnionType Cand = ConstInt;
378 std::tie(Itr, Inserted) = ConstCandMap.insert(std::make_pair(Cand, 0));
379 if (Inserted) {
380 ConstIntCandVec.push_back(ConstantCandidate(ConstInt));
381 Itr->second = ConstIntCandVec.size() - 1;
382 }
383 ConstIntCandVec[Itr->second].addUser(Inst, Idx, Cost);
384 LLVM_DEBUG(if (isa<ConstantInt>(Inst->getOperand(Idx))) dbgs()
385 << "Collect constant " << *ConstInt << " from " << *Inst
386 << " with cost " << Cost << '\n';
387 else dbgs() << "Collect constant " << *ConstInt
388 << " indirectly from " << *Inst << " via "
389 << *Inst->getOperand(Idx) << " with cost " << Cost
390 << '\n';);
391 }
392 }
393
394 /// Record constant GEP expression for instruction Inst at operand index Idx.
collectConstantCandidates(ConstCandMapType & ConstCandMap,Instruction * Inst,unsigned Idx,ConstantExpr * ConstExpr)395 void ConstantHoistingPass::collectConstantCandidates(
396 ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx,
397 ConstantExpr *ConstExpr) {
398 // TODO: Handle vector GEPs
399 if (ConstExpr->getType()->isVectorTy())
400 return;
401
402 GlobalVariable *BaseGV = dyn_cast<GlobalVariable>(ConstExpr->getOperand(0));
403 if (!BaseGV)
404 return;
405
406 // Get offset from the base GV.
407 PointerType *GVPtrTy = cast<PointerType>(BaseGV->getType());
408 IntegerType *PtrIntTy = DL->getIntPtrType(*Ctx, GVPtrTy->getAddressSpace());
409 APInt Offset(DL->getTypeSizeInBits(PtrIntTy), /*val*/0, /*isSigned*/true);
410 auto *GEPO = cast<GEPOperator>(ConstExpr);
411 if (!GEPO->accumulateConstantOffset(*DL, Offset))
412 return;
413
414 if (!Offset.isIntN(32))
415 return;
416
417 // A constant GEP expression that has a GlobalVariable as base pointer is
418 // usually lowered to a load from constant pool. Such operation is unlikely
419 // to be cheaper than compute it by <Base + Offset>, which can be lowered to
420 // an ADD instruction or folded into Load/Store instruction.
421 int Cost =
422 TTI->getIntImmCostInst(Instruction::Add, 1, Offset, PtrIntTy,
423 TargetTransformInfo::TCK_SizeAndLatency, Inst);
424 ConstCandVecType &ExprCandVec = ConstGEPCandMap[BaseGV];
425 ConstCandMapType::iterator Itr;
426 bool Inserted;
427 ConstPtrUnionType Cand = ConstExpr;
428 std::tie(Itr, Inserted) = ConstCandMap.insert(std::make_pair(Cand, 0));
429 if (Inserted) {
430 ExprCandVec.push_back(ConstantCandidate(
431 ConstantInt::get(Type::getInt32Ty(*Ctx), Offset.getLimitedValue()),
432 ConstExpr));
433 Itr->second = ExprCandVec.size() - 1;
434 }
435 ExprCandVec[Itr->second].addUser(Inst, Idx, Cost);
436 }
437
438 /// Check the operand for instruction Inst at index Idx.
collectConstantCandidates(ConstCandMapType & ConstCandMap,Instruction * Inst,unsigned Idx)439 void ConstantHoistingPass::collectConstantCandidates(
440 ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx) {
441 Value *Opnd = Inst->getOperand(Idx);
442
443 // Visit constant integers.
444 if (auto ConstInt = dyn_cast<ConstantInt>(Opnd)) {
445 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
446 return;
447 }
448
449 // Visit cast instructions that have constant integers.
450 if (auto CastInst = dyn_cast<Instruction>(Opnd)) {
451 // Only visit cast instructions, which have been skipped. All other
452 // instructions should have already been visited.
453 if (!CastInst->isCast())
454 return;
455
456 if (auto *ConstInt = dyn_cast<ConstantInt>(CastInst->getOperand(0))) {
457 // Pretend the constant is directly used by the instruction and ignore
458 // the cast instruction.
459 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
460 return;
461 }
462 }
463
464 // Visit constant expressions that have constant integers.
465 if (auto ConstExpr = dyn_cast<ConstantExpr>(Opnd)) {
466 // Handle constant gep expressions.
467 if (ConstHoistGEP && ConstExpr->isGEPWithNoNotionalOverIndexing())
468 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstExpr);
469
470 // Only visit constant cast expressions.
471 if (!ConstExpr->isCast())
472 return;
473
474 if (auto ConstInt = dyn_cast<ConstantInt>(ConstExpr->getOperand(0))) {
475 // Pretend the constant is directly used by the instruction and ignore
476 // the constant expression.
477 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
478 return;
479 }
480 }
481 }
482
483 /// Scan the instruction for expensive integer constants and record them
484 /// in the constant candidate vector.
collectConstantCandidates(ConstCandMapType & ConstCandMap,Instruction * Inst)485 void ConstantHoistingPass::collectConstantCandidates(
486 ConstCandMapType &ConstCandMap, Instruction *Inst) {
487 // Skip all cast instructions. They are visited indirectly later on.
488 if (Inst->isCast())
489 return;
490
491 // Scan all operands.
492 for (unsigned Idx = 0, E = Inst->getNumOperands(); Idx != E; ++Idx) {
493 // The cost of materializing the constants (defined in
494 // `TargetTransformInfo::getIntImmCostInst`) for instructions which only
495 // take constant variables is lower than `TargetTransformInfo::TCC_Basic`.
496 // So it's safe for us to collect constant candidates from all
497 // IntrinsicInsts.
498 if (canReplaceOperandWithVariable(Inst, Idx)) {
499 collectConstantCandidates(ConstCandMap, Inst, Idx);
500 }
501 } // end of for all operands
502 }
503
504 /// Collect all integer constants in the function that cannot be folded
505 /// into an instruction itself.
collectConstantCandidates(Function & Fn)506 void ConstantHoistingPass::collectConstantCandidates(Function &Fn) {
507 ConstCandMapType ConstCandMap;
508 for (BasicBlock &BB : Fn) {
509 // Ignore unreachable basic blocks.
510 if (!DT->isReachableFromEntry(&BB))
511 continue;
512 for (Instruction &Inst : BB)
513 collectConstantCandidates(ConstCandMap, &Inst);
514 }
515 }
516
517 // This helper function is necessary to deal with values that have different
518 // bit widths (APInt Operator- does not like that). If the value cannot be
519 // represented in uint64 we return an "empty" APInt. This is then interpreted
520 // as the value is not in range.
calculateOffsetDiff(const APInt & V1,const APInt & V2)521 static Optional<APInt> calculateOffsetDiff(const APInt &V1, const APInt &V2) {
522 Optional<APInt> Res = None;
523 unsigned BW = V1.getBitWidth() > V2.getBitWidth() ?
524 V1.getBitWidth() : V2.getBitWidth();
525 uint64_t LimVal1 = V1.getLimitedValue();
526 uint64_t LimVal2 = V2.getLimitedValue();
527
528 if (LimVal1 == ~0ULL || LimVal2 == ~0ULL)
529 return Res;
530
531 uint64_t Diff = LimVal1 - LimVal2;
532 return APInt(BW, Diff, true);
533 }
534
535 // From a list of constants, one needs to picked as the base and the other
536 // constants will be transformed into an offset from that base constant. The
537 // question is which we can pick best? For example, consider these constants
538 // and their number of uses:
539 //
540 // Constants| 2 | 4 | 12 | 42 |
541 // NumUses | 3 | 2 | 8 | 7 |
542 //
543 // Selecting constant 12 because it has the most uses will generate negative
544 // offsets for constants 2 and 4 (i.e. -10 and -8 respectively). If negative
545 // offsets lead to less optimal code generation, then there might be better
546 // solutions. Suppose immediates in the range of 0..35 are most optimally
547 // supported by the architecture, then selecting constant 2 is most optimal
548 // because this will generate offsets: 0, 2, 10, 40. Offsets 0, 2 and 10 are in
549 // range 0..35, and thus 3 + 2 + 8 = 13 uses are in range. Selecting 12 would
550 // have only 8 uses in range, so choosing 2 as a base is more optimal. Thus, in
551 // selecting the base constant the range of the offsets is a very important
552 // factor too that we take into account here. This algorithm calculates a total
553 // costs for selecting a constant as the base and substract the costs if
554 // immediates are out of range. It has quadratic complexity, so we call this
555 // function only when we're optimising for size and there are less than 100
556 // constants, we fall back to the straightforward algorithm otherwise
557 // which does not do all the offset calculations.
558 unsigned
maximizeConstantsInRange(ConstCandVecType::iterator S,ConstCandVecType::iterator E,ConstCandVecType::iterator & MaxCostItr)559 ConstantHoistingPass::maximizeConstantsInRange(ConstCandVecType::iterator S,
560 ConstCandVecType::iterator E,
561 ConstCandVecType::iterator &MaxCostItr) {
562 unsigned NumUses = 0;
563
564 bool OptForSize = Entry->getParent()->hasOptSize() ||
565 llvm::shouldOptimizeForSize(Entry->getParent(), PSI, BFI,
566 PGSOQueryType::IRPass);
567 if (!OptForSize || std::distance(S,E) > 100) {
568 for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
569 NumUses += ConstCand->Uses.size();
570 if (ConstCand->CumulativeCost > MaxCostItr->CumulativeCost)
571 MaxCostItr = ConstCand;
572 }
573 return NumUses;
574 }
575
576 LLVM_DEBUG(dbgs() << "== Maximize constants in range ==\n");
577 int MaxCost = -1;
578 for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
579 auto Value = ConstCand->ConstInt->getValue();
580 Type *Ty = ConstCand->ConstInt->getType();
581 int Cost = 0;
582 NumUses += ConstCand->Uses.size();
583 LLVM_DEBUG(dbgs() << "= Constant: " << ConstCand->ConstInt->getValue()
584 << "\n");
585
586 for (auto User : ConstCand->Uses) {
587 unsigned Opcode = User.Inst->getOpcode();
588 unsigned OpndIdx = User.OpndIdx;
589 Cost += TTI->getIntImmCostInst(Opcode, OpndIdx, Value, Ty,
590 TargetTransformInfo::TCK_SizeAndLatency);
591 LLVM_DEBUG(dbgs() << "Cost: " << Cost << "\n");
592
593 for (auto C2 = S; C2 != E; ++C2) {
594 Optional<APInt> Diff = calculateOffsetDiff(
595 C2->ConstInt->getValue(),
596 ConstCand->ConstInt->getValue());
597 if (Diff) {
598 const int ImmCosts =
599 TTI->getIntImmCodeSizeCost(Opcode, OpndIdx, Diff.getValue(), Ty);
600 Cost -= ImmCosts;
601 LLVM_DEBUG(dbgs() << "Offset " << Diff.getValue() << " "
602 << "has penalty: " << ImmCosts << "\n"
603 << "Adjusted cost: " << Cost << "\n");
604 }
605 }
606 }
607 LLVM_DEBUG(dbgs() << "Cumulative cost: " << Cost << "\n");
608 if (Cost > MaxCost) {
609 MaxCost = Cost;
610 MaxCostItr = ConstCand;
611 LLVM_DEBUG(dbgs() << "New candidate: " << MaxCostItr->ConstInt->getValue()
612 << "\n");
613 }
614 }
615 return NumUses;
616 }
617
618 /// Find the base constant within the given range and rebase all other
619 /// constants with respect to the base constant.
findAndMakeBaseConstant(ConstCandVecType::iterator S,ConstCandVecType::iterator E,SmallVectorImpl<consthoist::ConstantInfo> & ConstInfoVec)620 void ConstantHoistingPass::findAndMakeBaseConstant(
621 ConstCandVecType::iterator S, ConstCandVecType::iterator E,
622 SmallVectorImpl<consthoist::ConstantInfo> &ConstInfoVec) {
623 auto MaxCostItr = S;
624 unsigned NumUses = maximizeConstantsInRange(S, E, MaxCostItr);
625
626 // Don't hoist constants that have only one use.
627 if (NumUses <= 1)
628 return;
629
630 ConstantInt *ConstInt = MaxCostItr->ConstInt;
631 ConstantExpr *ConstExpr = MaxCostItr->ConstExpr;
632 ConstantInfo ConstInfo;
633 ConstInfo.BaseInt = ConstInt;
634 ConstInfo.BaseExpr = ConstExpr;
635 Type *Ty = ConstInt->getType();
636
637 // Rebase the constants with respect to the base constant.
638 for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
639 APInt Diff = ConstCand->ConstInt->getValue() - ConstInt->getValue();
640 Constant *Offset = Diff == 0 ? nullptr : ConstantInt::get(Ty, Diff);
641 Type *ConstTy =
642 ConstCand->ConstExpr ? ConstCand->ConstExpr->getType() : nullptr;
643 ConstInfo.RebasedConstants.push_back(
644 RebasedConstantInfo(std::move(ConstCand->Uses), Offset, ConstTy));
645 }
646 ConstInfoVec.push_back(std::move(ConstInfo));
647 }
648
649 /// Finds and combines constant candidates that can be easily
650 /// rematerialized with an add from a common base constant.
findBaseConstants(GlobalVariable * BaseGV)651 void ConstantHoistingPass::findBaseConstants(GlobalVariable *BaseGV) {
652 // If BaseGV is nullptr, find base among candidate constant integers;
653 // Otherwise find base among constant GEPs that share the same BaseGV.
654 ConstCandVecType &ConstCandVec = BaseGV ?
655 ConstGEPCandMap[BaseGV] : ConstIntCandVec;
656 ConstInfoVecType &ConstInfoVec = BaseGV ?
657 ConstGEPInfoMap[BaseGV] : ConstIntInfoVec;
658
659 // Sort the constants by value and type. This invalidates the mapping!
660 llvm::stable_sort(ConstCandVec, [](const ConstantCandidate &LHS,
661 const ConstantCandidate &RHS) {
662 if (LHS.ConstInt->getType() != RHS.ConstInt->getType())
663 return LHS.ConstInt->getType()->getBitWidth() <
664 RHS.ConstInt->getType()->getBitWidth();
665 return LHS.ConstInt->getValue().ult(RHS.ConstInt->getValue());
666 });
667
668 // Simple linear scan through the sorted constant candidate vector for viable
669 // merge candidates.
670 auto MinValItr = ConstCandVec.begin();
671 for (auto CC = std::next(ConstCandVec.begin()), E = ConstCandVec.end();
672 CC != E; ++CC) {
673 if (MinValItr->ConstInt->getType() == CC->ConstInt->getType()) {
674 Type *MemUseValTy = nullptr;
675 for (auto &U : CC->Uses) {
676 auto *UI = U.Inst;
677 if (LoadInst *LI = dyn_cast<LoadInst>(UI)) {
678 MemUseValTy = LI->getType();
679 break;
680 } else if (StoreInst *SI = dyn_cast<StoreInst>(UI)) {
681 // Make sure the constant is used as pointer operand of the StoreInst.
682 if (SI->getPointerOperand() == SI->getOperand(U.OpndIdx)) {
683 MemUseValTy = SI->getValueOperand()->getType();
684 break;
685 }
686 }
687 }
688
689 // Check if the constant is in range of an add with immediate.
690 APInt Diff = CC->ConstInt->getValue() - MinValItr->ConstInt->getValue();
691 if ((Diff.getBitWidth() <= 64) &&
692 TTI->isLegalAddImmediate(Diff.getSExtValue()) &&
693 // Check if Diff can be used as offset in addressing mode of the user
694 // memory instruction.
695 (!MemUseValTy || TTI->isLegalAddressingMode(MemUseValTy,
696 /*BaseGV*/nullptr, /*BaseOffset*/Diff.getSExtValue(),
697 /*HasBaseReg*/true, /*Scale*/0)))
698 continue;
699 }
700 // We either have now a different constant type or the constant is not in
701 // range of an add with immediate anymore.
702 findAndMakeBaseConstant(MinValItr, CC, ConstInfoVec);
703 // Start a new base constant search.
704 MinValItr = CC;
705 }
706 // Finalize the last base constant search.
707 findAndMakeBaseConstant(MinValItr, ConstCandVec.end(), ConstInfoVec);
708 }
709
710 /// Updates the operand at Idx in instruction Inst with the result of
711 /// instruction Mat. If the instruction is a PHI node then special
712 /// handling for duplicate values form the same incoming basic block is
713 /// required.
714 /// \return The update will always succeed, but the return value indicated if
715 /// Mat was used for the update or not.
updateOperand(Instruction * Inst,unsigned Idx,Instruction * Mat)716 static bool updateOperand(Instruction *Inst, unsigned Idx, Instruction *Mat) {
717 if (auto PHI = dyn_cast<PHINode>(Inst)) {
718 // Check if any previous operand of the PHI node has the same incoming basic
719 // block. This is a very odd case that happens when the incoming basic block
720 // has a switch statement. In this case use the same value as the previous
721 // operand(s), otherwise we will fail verification due to different values.
722 // The values are actually the same, but the variable names are different
723 // and the verifier doesn't like that.
724 BasicBlock *IncomingBB = PHI->getIncomingBlock(Idx);
725 for (unsigned i = 0; i < Idx; ++i) {
726 if (PHI->getIncomingBlock(i) == IncomingBB) {
727 Value *IncomingVal = PHI->getIncomingValue(i);
728 Inst->setOperand(Idx, IncomingVal);
729 return false;
730 }
731 }
732 }
733
734 Inst->setOperand(Idx, Mat);
735 return true;
736 }
737
738 /// Emit materialization code for all rebased constants and update their
739 /// users.
emitBaseConstants(Instruction * Base,Constant * Offset,Type * Ty,const ConstantUser & ConstUser)740 void ConstantHoistingPass::emitBaseConstants(Instruction *Base,
741 Constant *Offset,
742 Type *Ty,
743 const ConstantUser &ConstUser) {
744 Instruction *Mat = Base;
745
746 // The same offset can be dereferenced to different types in nested struct.
747 if (!Offset && Ty && Ty != Base->getType())
748 Offset = ConstantInt::get(Type::getInt32Ty(*Ctx), 0);
749
750 if (Offset) {
751 Instruction *InsertionPt = findMatInsertPt(ConstUser.Inst,
752 ConstUser.OpndIdx);
753 if (Ty) {
754 // Constant being rebased is a ConstantExpr.
755 PointerType *Int8PtrTy = Type::getInt8PtrTy(*Ctx,
756 cast<PointerType>(Ty)->getAddressSpace());
757 Base = new BitCastInst(Base, Int8PtrTy, "base_bitcast", InsertionPt);
758 Mat = GetElementPtrInst::Create(Int8PtrTy->getElementType(), Base,
759 Offset, "mat_gep", InsertionPt);
760 Mat = new BitCastInst(Mat, Ty, "mat_bitcast", InsertionPt);
761 } else
762 // Constant being rebased is a ConstantInt.
763 Mat = BinaryOperator::Create(Instruction::Add, Base, Offset,
764 "const_mat", InsertionPt);
765
766 LLVM_DEBUG(dbgs() << "Materialize constant (" << *Base->getOperand(0)
767 << " + " << *Offset << ") in BB "
768 << Mat->getParent()->getName() << '\n'
769 << *Mat << '\n');
770 Mat->setDebugLoc(ConstUser.Inst->getDebugLoc());
771 }
772 Value *Opnd = ConstUser.Inst->getOperand(ConstUser.OpndIdx);
773
774 // Visit constant integer.
775 if (isa<ConstantInt>(Opnd)) {
776 LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n');
777 if (!updateOperand(ConstUser.Inst, ConstUser.OpndIdx, Mat) && Offset)
778 Mat->eraseFromParent();
779 LLVM_DEBUG(dbgs() << "To : " << *ConstUser.Inst << '\n');
780 return;
781 }
782
783 // Visit cast instruction.
784 if (auto CastInst = dyn_cast<Instruction>(Opnd)) {
785 assert(CastInst->isCast() && "Expected an cast instruction!");
786 // Check if we already have visited this cast instruction before to avoid
787 // unnecessary cloning.
788 Instruction *&ClonedCastInst = ClonedCastMap[CastInst];
789 if (!ClonedCastInst) {
790 ClonedCastInst = CastInst->clone();
791 ClonedCastInst->setOperand(0, Mat);
792 ClonedCastInst->insertAfter(CastInst);
793 // Use the same debug location as the original cast instruction.
794 ClonedCastInst->setDebugLoc(CastInst->getDebugLoc());
795 LLVM_DEBUG(dbgs() << "Clone instruction: " << *CastInst << '\n'
796 << "To : " << *ClonedCastInst << '\n');
797 }
798
799 LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n');
800 updateOperand(ConstUser.Inst, ConstUser.OpndIdx, ClonedCastInst);
801 LLVM_DEBUG(dbgs() << "To : " << *ConstUser.Inst << '\n');
802 return;
803 }
804
805 // Visit constant expression.
806 if (auto ConstExpr = dyn_cast<ConstantExpr>(Opnd)) {
807 if (ConstExpr->isGEPWithNoNotionalOverIndexing()) {
808 // Operand is a ConstantGEP, replace it.
809 updateOperand(ConstUser.Inst, ConstUser.OpndIdx, Mat);
810 return;
811 }
812
813 // Aside from constant GEPs, only constant cast expressions are collected.
814 assert(ConstExpr->isCast() && "ConstExpr should be a cast");
815 Instruction *ConstExprInst = ConstExpr->getAsInstruction();
816 ConstExprInst->setOperand(0, Mat);
817 ConstExprInst->insertBefore(findMatInsertPt(ConstUser.Inst,
818 ConstUser.OpndIdx));
819
820 // Use the same debug location as the instruction we are about to update.
821 ConstExprInst->setDebugLoc(ConstUser.Inst->getDebugLoc());
822
823 LLVM_DEBUG(dbgs() << "Create instruction: " << *ConstExprInst << '\n'
824 << "From : " << *ConstExpr << '\n');
825 LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n');
826 if (!updateOperand(ConstUser.Inst, ConstUser.OpndIdx, ConstExprInst)) {
827 ConstExprInst->eraseFromParent();
828 if (Offset)
829 Mat->eraseFromParent();
830 }
831 LLVM_DEBUG(dbgs() << "To : " << *ConstUser.Inst << '\n');
832 return;
833 }
834 }
835
836 /// Hoist and hide the base constant behind a bitcast and emit
837 /// materialization code for derived constants.
emitBaseConstants(GlobalVariable * BaseGV)838 bool ConstantHoistingPass::emitBaseConstants(GlobalVariable *BaseGV) {
839 bool MadeChange = false;
840 SmallVectorImpl<consthoist::ConstantInfo> &ConstInfoVec =
841 BaseGV ? ConstGEPInfoMap[BaseGV] : ConstIntInfoVec;
842 for (auto const &ConstInfo : ConstInfoVec) {
843 SetVector<Instruction *> IPSet = findConstantInsertionPoint(ConstInfo);
844 // We can have an empty set if the function contains unreachable blocks.
845 if (IPSet.empty())
846 continue;
847
848 unsigned UsesNum = 0;
849 unsigned ReBasesNum = 0;
850 unsigned NotRebasedNum = 0;
851 for (Instruction *IP : IPSet) {
852 // First, collect constants depending on this IP of the base.
853 unsigned Uses = 0;
854 using RebasedUse = std::tuple<Constant *, Type *, ConstantUser>;
855 SmallVector<RebasedUse, 4> ToBeRebased;
856 for (auto const &RCI : ConstInfo.RebasedConstants) {
857 for (auto const &U : RCI.Uses) {
858 Uses++;
859 BasicBlock *OrigMatInsertBB =
860 findMatInsertPt(U.Inst, U.OpndIdx)->getParent();
861 // If Base constant is to be inserted in multiple places,
862 // generate rebase for U using the Base dominating U.
863 if (IPSet.size() == 1 ||
864 DT->dominates(IP->getParent(), OrigMatInsertBB))
865 ToBeRebased.push_back(RebasedUse(RCI.Offset, RCI.Ty, U));
866 }
867 }
868 UsesNum = Uses;
869
870 // If only few constants depend on this IP of base, skip rebasing,
871 // assuming the base and the rebased have the same materialization cost.
872 if (ToBeRebased.size() < MinNumOfDependentToRebase) {
873 NotRebasedNum += ToBeRebased.size();
874 continue;
875 }
876
877 // Emit an instance of the base at this IP.
878 Instruction *Base = nullptr;
879 // Hoist and hide the base constant behind a bitcast.
880 if (ConstInfo.BaseExpr) {
881 assert(BaseGV && "A base constant expression must have an base GV");
882 Type *Ty = ConstInfo.BaseExpr->getType();
883 Base = new BitCastInst(ConstInfo.BaseExpr, Ty, "const", IP);
884 } else {
885 IntegerType *Ty = ConstInfo.BaseInt->getType();
886 Base = new BitCastInst(ConstInfo.BaseInt, Ty, "const", IP);
887 }
888
889 Base->setDebugLoc(IP->getDebugLoc());
890
891 LLVM_DEBUG(dbgs() << "Hoist constant (" << *ConstInfo.BaseInt
892 << ") to BB " << IP->getParent()->getName() << '\n'
893 << *Base << '\n');
894
895 // Emit materialization code for rebased constants depending on this IP.
896 for (auto const &R : ToBeRebased) {
897 Constant *Off = std::get<0>(R);
898 Type *Ty = std::get<1>(R);
899 ConstantUser U = std::get<2>(R);
900 emitBaseConstants(Base, Off, Ty, U);
901 ReBasesNum++;
902 // Use the same debug location as the last user of the constant.
903 Base->setDebugLoc(DILocation::getMergedLocation(
904 Base->getDebugLoc(), U.Inst->getDebugLoc()));
905 }
906 assert(!Base->use_empty() && "The use list is empty!?");
907 assert(isa<Instruction>(Base->user_back()) &&
908 "All uses should be instructions.");
909 }
910 (void)UsesNum;
911 (void)ReBasesNum;
912 (void)NotRebasedNum;
913 // Expect all uses are rebased after rebase is done.
914 assert(UsesNum == (ReBasesNum + NotRebasedNum) &&
915 "Not all uses are rebased");
916
917 NumConstantsHoisted++;
918
919 // Base constant is also included in ConstInfo.RebasedConstants, so
920 // deduct 1 from ConstInfo.RebasedConstants.size().
921 NumConstantsRebased += ConstInfo.RebasedConstants.size() - 1;
922
923 MadeChange = true;
924 }
925 return MadeChange;
926 }
927
928 /// Check all cast instructions we made a copy of and remove them if they
929 /// have no more users.
deleteDeadCastInst() const930 void ConstantHoistingPass::deleteDeadCastInst() const {
931 for (auto const &I : ClonedCastMap)
932 if (I.first->use_empty())
933 I.first->eraseFromParent();
934 }
935
936 /// Optimize expensive integer constants in the given function.
runImpl(Function & Fn,TargetTransformInfo & TTI,DominatorTree & DT,BlockFrequencyInfo * BFI,BasicBlock & Entry,ProfileSummaryInfo * PSI)937 bool ConstantHoistingPass::runImpl(Function &Fn, TargetTransformInfo &TTI,
938 DominatorTree &DT, BlockFrequencyInfo *BFI,
939 BasicBlock &Entry, ProfileSummaryInfo *PSI) {
940 this->TTI = &TTI;
941 this->DT = &DT;
942 this->BFI = BFI;
943 this->DL = &Fn.getParent()->getDataLayout();
944 this->Ctx = &Fn.getContext();
945 this->Entry = &Entry;
946 this->PSI = PSI;
947 // Collect all constant candidates.
948 collectConstantCandidates(Fn);
949
950 // Combine constants that can be easily materialized with an add from a common
951 // base constant.
952 if (!ConstIntCandVec.empty())
953 findBaseConstants(nullptr);
954 for (const auto &MapEntry : ConstGEPCandMap)
955 if (!MapEntry.second.empty())
956 findBaseConstants(MapEntry.first);
957
958 // Finally hoist the base constant and emit materialization code for dependent
959 // constants.
960 bool MadeChange = false;
961 if (!ConstIntInfoVec.empty())
962 MadeChange = emitBaseConstants(nullptr);
963 for (const auto &MapEntry : ConstGEPInfoMap)
964 if (!MapEntry.second.empty())
965 MadeChange |= emitBaseConstants(MapEntry.first);
966
967
968 // Cleanup dead instructions.
969 deleteDeadCastInst();
970
971 cleanup();
972
973 return MadeChange;
974 }
975
run(Function & F,FunctionAnalysisManager & AM)976 PreservedAnalyses ConstantHoistingPass::run(Function &F,
977 FunctionAnalysisManager &AM) {
978 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
979 auto &TTI = AM.getResult<TargetIRAnalysis>(F);
980 auto BFI = ConstHoistWithBlockFrequency
981 ? &AM.getResult<BlockFrequencyAnalysis>(F)
982 : nullptr;
983 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
984 auto *PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
985 if (!runImpl(F, TTI, DT, BFI, F.getEntryBlock(), PSI))
986 return PreservedAnalyses::all();
987
988 PreservedAnalyses PA;
989 PA.preserveSet<CFGAnalyses>();
990 return PA;
991 }
992