1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10 // It also builds the data structures and initialization code needed for
11 // updating execution counts and emitting the profile at runtime.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/ADT/Twine.h"
21 #include "llvm/Analysis/BlockFrequencyInfo.h"
22 #include "llvm/Analysis/BranchProbabilityInfo.h"
23 #include "llvm/Analysis/LoopInfo.h"
24 #include "llvm/Analysis/TargetLibraryInfo.h"
25 #include "llvm/IR/Attributes.h"
26 #include "llvm/IR/BasicBlock.h"
27 #include "llvm/IR/Constant.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DerivedTypes.h"
30 #include "llvm/IR/Dominators.h"
31 #include "llvm/IR/Function.h"
32 #include "llvm/IR/GlobalValue.h"
33 #include "llvm/IR/GlobalVariable.h"
34 #include "llvm/IR/IRBuilder.h"
35 #include "llvm/IR/Instruction.h"
36 #include "llvm/IR/Instructions.h"
37 #include "llvm/IR/IntrinsicInst.h"
38 #include "llvm/IR/Module.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/InitializePasses.h"
41 #include "llvm/Pass.h"
42 #include "llvm/ProfileData/InstrProf.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/CommandLine.h"
45 #include "llvm/Support/Error.h"
46 #include "llvm/Support/ErrorHandling.h"
47 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
48 #include "llvm/Transforms/Utils/ModuleUtils.h"
49 #include "llvm/Transforms/Utils/SSAUpdater.h"
50 #include <algorithm>
51 #include <cassert>
52 #include <cstddef>
53 #include <cstdint>
54 #include <string>
55
56 using namespace llvm;
57
58 #define DEBUG_TYPE "instrprof"
59
60 namespace {
61
62 cl::opt<bool> DoHashBasedCounterSplit(
63 "hash-based-counter-split",
64 cl::desc("Rename counter variable of a comdat function based on cfg hash"),
65 cl::init(true));
66
67 cl::opt<bool> RuntimeCounterRelocation(
68 "runtime-counter-relocation",
69 cl::desc("Enable relocating counters at runtime."),
70 cl::init(false));
71
72 cl::opt<bool> ValueProfileStaticAlloc(
73 "vp-static-alloc",
74 cl::desc("Do static counter allocation for value profiler"),
75 cl::init(true));
76
77 cl::opt<double> NumCountersPerValueSite(
78 "vp-counters-per-site",
79 cl::desc("The average number of profile counters allocated "
80 "per value profiling site."),
81 // This is set to a very small value because in real programs, only
82 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
83 // For those sites with non-zero profile, the average number of targets
84 // is usually smaller than 2.
85 cl::init(1.0));
86
87 cl::opt<bool> AtomicCounterUpdateAll(
88 "instrprof-atomic-counter-update-all", cl::ZeroOrMore,
89 cl::desc("Make all profile counter updates atomic (for testing only)"),
90 cl::init(false));
91
92 cl::opt<bool> AtomicCounterUpdatePromoted(
93 "atomic-counter-update-promoted", cl::ZeroOrMore,
94 cl::desc("Do counter update using atomic fetch add "
95 " for promoted counters only"),
96 cl::init(false));
97
98 cl::opt<bool> AtomicFirstCounter(
99 "atomic-first-counter", cl::ZeroOrMore,
100 cl::desc("Use atomic fetch add for first counter in a function (usually "
101 "the entry counter)"),
102 cl::init(false));
103
104 // If the option is not specified, the default behavior about whether
105 // counter promotion is done depends on how instrumentaiton lowering
106 // pipeline is setup, i.e., the default value of true of this option
107 // does not mean the promotion will be done by default. Explicitly
108 // setting this option can override the default behavior.
109 cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
110 cl::desc("Do counter register promotion"),
111 cl::init(false));
112 cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
113 cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20),
114 cl::desc("Max number counter promotions per loop to avoid"
115 " increasing register pressure too much"));
116
117 // A debug option
118 cl::opt<int>
119 MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
120 cl::desc("Max number of allowed counter promotions"));
121
122 cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
123 cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3),
124 cl::desc("The max number of exiting blocks of a loop to allow "
125 " speculative counter promotion"));
126
127 cl::opt<bool> SpeculativeCounterPromotionToLoop(
128 cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false),
129 cl::desc("When the option is false, if the target block is in a loop, "
130 "the promotion will be disallowed unless the promoted counter "
131 " update can be further/iteratively promoted into an acyclic "
132 " region."));
133
134 cl::opt<bool> IterativeCounterPromotion(
135 cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
136 cl::desc("Allow counter promotion across the whole loop nest."));
137
138 cl::opt<bool> SkipRetExitBlock(
139 cl::ZeroOrMore, "skip-ret-exit-block", cl::init(true),
140 cl::desc("Suppress counter promotion if exit blocks contain ret."));
141
142 class InstrProfilingLegacyPass : public ModulePass {
143 InstrProfiling InstrProf;
144
145 public:
146 static char ID;
147
InstrProfilingLegacyPass()148 InstrProfilingLegacyPass() : ModulePass(ID) {}
InstrProfilingLegacyPass(const InstrProfOptions & Options,bool IsCS=false)149 InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false)
150 : ModulePass(ID), InstrProf(Options, IsCS) {
151 initializeInstrProfilingLegacyPassPass(*PassRegistry::getPassRegistry());
152 }
153
getPassName() const154 StringRef getPassName() const override {
155 return "Frontend instrumentation-based coverage lowering";
156 }
157
runOnModule(Module & M)158 bool runOnModule(Module &M) override {
159 auto GetTLI = [this](Function &F) -> TargetLibraryInfo & {
160 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
161 };
162 return InstrProf.run(M, GetTLI);
163 }
164
getAnalysisUsage(AnalysisUsage & AU) const165 void getAnalysisUsage(AnalysisUsage &AU) const override {
166 AU.setPreservesCFG();
167 AU.addRequired<TargetLibraryInfoWrapperPass>();
168 }
169 };
170
171 ///
172 /// A helper class to promote one counter RMW operation in the loop
173 /// into register update.
174 ///
175 /// RWM update for the counter will be sinked out of the loop after
176 /// the transformation.
177 ///
178 class PGOCounterPromoterHelper : public LoadAndStorePromoter {
179 public:
PGOCounterPromoterHelper(Instruction * L,Instruction * S,SSAUpdater & SSA,Value * Init,BasicBlock * PH,ArrayRef<BasicBlock * > ExitBlocks,ArrayRef<Instruction * > InsertPts,DenseMap<Loop *,SmallVector<LoadStorePair,8>> & LoopToCands,LoopInfo & LI)180 PGOCounterPromoterHelper(
181 Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,
182 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
183 ArrayRef<Instruction *> InsertPts,
184 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
185 LoopInfo &LI)
186 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
187 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
188 assert(isa<LoadInst>(L));
189 assert(isa<StoreInst>(S));
190 SSA.AddAvailableValue(PH, Init);
191 }
192
doExtraRewritesBeforeFinalDeletion()193 void doExtraRewritesBeforeFinalDeletion() override {
194 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
195 BasicBlock *ExitBlock = ExitBlocks[i];
196 Instruction *InsertPos = InsertPts[i];
197 // Get LiveIn value into the ExitBlock. If there are multiple
198 // predecessors, the value is defined by a PHI node in this
199 // block.
200 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
201 Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
202 Type *Ty = LiveInValue->getType();
203 IRBuilder<> Builder(InsertPos);
204 if (AtomicCounterUpdatePromoted)
205 // automic update currently can only be promoted across the current
206 // loop, not the whole loop nest.
207 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
208 MaybeAlign(),
209 AtomicOrdering::SequentiallyConsistent);
210 else {
211 LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
212 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
213 auto *NewStore = Builder.CreateStore(NewVal, Addr);
214
215 // Now update the parent loop's candidate list:
216 if (IterativeCounterPromotion) {
217 auto *TargetLoop = LI.getLoopFor(ExitBlock);
218 if (TargetLoop)
219 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
220 }
221 }
222 }
223 }
224
225 private:
226 Instruction *Store;
227 ArrayRef<BasicBlock *> ExitBlocks;
228 ArrayRef<Instruction *> InsertPts;
229 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
230 LoopInfo &LI;
231 };
232
233 /// A helper class to do register promotion for all profile counter
234 /// updates in a loop.
235 ///
236 class PGOCounterPromoter {
237 public:
PGOCounterPromoter(DenseMap<Loop *,SmallVector<LoadStorePair,8>> & LoopToCands,Loop & CurLoop,LoopInfo & LI,BlockFrequencyInfo * BFI)238 PGOCounterPromoter(
239 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
240 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
241 : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop),
242 LI(LI), BFI(BFI) {
243
244 // Skip collection of ExitBlocks and InsertPts for loops that will not be
245 // able to have counters promoted.
246 SmallVector<BasicBlock *, 8> LoopExitBlocks;
247 SmallPtrSet<BasicBlock *, 8> BlockSet;
248
249 L.getExitBlocks(LoopExitBlocks);
250 if (!isPromotionPossible(&L, LoopExitBlocks))
251 return;
252
253 for (BasicBlock *ExitBlock : LoopExitBlocks) {
254 if (BlockSet.insert(ExitBlock).second) {
255 ExitBlocks.push_back(ExitBlock);
256 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
257 }
258 }
259 }
260
run(int64_t * NumPromoted)261 bool run(int64_t *NumPromoted) {
262 // Skip 'infinite' loops:
263 if (ExitBlocks.size() == 0)
264 return false;
265
266 // Skip if any of the ExitBlocks contains a ret instruction.
267 // This is to prevent dumping of incomplete profile -- if the
268 // the loop is a long running loop and dump is called in the middle
269 // of the loop, the result profile is incomplete.
270 // FIXME: add other heuristics to detect long running loops.
271 if (SkipRetExitBlock) {
272 for (auto BB : ExitBlocks)
273 if (isa<ReturnInst>(BB->getTerminator()))
274 return false;
275 }
276
277 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
278 if (MaxProm == 0)
279 return false;
280
281 unsigned Promoted = 0;
282 for (auto &Cand : LoopToCandidates[&L]) {
283
284 SmallVector<PHINode *, 4> NewPHIs;
285 SSAUpdater SSA(&NewPHIs);
286 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
287
288 // If BFI is set, we will use it to guide the promotions.
289 if (BFI) {
290 auto *BB = Cand.first->getParent();
291 auto InstrCount = BFI->getBlockProfileCount(BB);
292 if (!InstrCount)
293 continue;
294 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
295 // If the average loop trip count is not greater than 1.5, we skip
296 // promotion.
297 if (PreheaderCount &&
298 (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2))
299 continue;
300 }
301
302 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
303 L.getLoopPreheader(), ExitBlocks,
304 InsertPts, LoopToCandidates, LI);
305 Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
306 Promoted++;
307 if (Promoted >= MaxProm)
308 break;
309
310 (*NumPromoted)++;
311 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
312 break;
313 }
314
315 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
316 << L.getLoopDepth() << ")\n");
317 return Promoted != 0;
318 }
319
320 private:
allowSpeculativeCounterPromotion(Loop * LP)321 bool allowSpeculativeCounterPromotion(Loop *LP) {
322 SmallVector<BasicBlock *, 8> ExitingBlocks;
323 L.getExitingBlocks(ExitingBlocks);
324 // Not considierered speculative.
325 if (ExitingBlocks.size() == 1)
326 return true;
327 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
328 return false;
329 return true;
330 }
331
332 // Check whether the loop satisfies the basic conditions needed to perform
333 // Counter Promotions.
isPromotionPossible(Loop * LP,const SmallVectorImpl<BasicBlock * > & LoopExitBlocks)334 bool isPromotionPossible(Loop *LP,
335 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
336 // We can't insert into a catchswitch.
337 if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
338 return isa<CatchSwitchInst>(Exit->getTerminator());
339 }))
340 return false;
341
342 if (!LP->hasDedicatedExits())
343 return false;
344
345 BasicBlock *PH = LP->getLoopPreheader();
346 if (!PH)
347 return false;
348
349 return true;
350 }
351
352 // Returns the max number of Counter Promotions for LP.
getMaxNumOfPromotionsInLoop(Loop * LP)353 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
354 SmallVector<BasicBlock *, 8> LoopExitBlocks;
355 LP->getExitBlocks(LoopExitBlocks);
356 if (!isPromotionPossible(LP, LoopExitBlocks))
357 return 0;
358
359 SmallVector<BasicBlock *, 8> ExitingBlocks;
360 LP->getExitingBlocks(ExitingBlocks);
361
362 // If BFI is set, we do more aggressive promotions based on BFI.
363 if (BFI)
364 return (unsigned)-1;
365
366 // Not considierered speculative.
367 if (ExitingBlocks.size() == 1)
368 return MaxNumOfPromotionsPerLoop;
369
370 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
371 return 0;
372
373 // Whether the target block is in a loop does not matter:
374 if (SpeculativeCounterPromotionToLoop)
375 return MaxNumOfPromotionsPerLoop;
376
377 // Now check the target block:
378 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
379 for (auto *TargetBlock : LoopExitBlocks) {
380 auto *TargetLoop = LI.getLoopFor(TargetBlock);
381 if (!TargetLoop)
382 continue;
383 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
384 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
385 MaxProm =
386 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
387 PendingCandsInTarget);
388 }
389 return MaxProm;
390 }
391
392 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
393 SmallVector<BasicBlock *, 8> ExitBlocks;
394 SmallVector<Instruction *, 8> InsertPts;
395 Loop &L;
396 LoopInfo &LI;
397 BlockFrequencyInfo *BFI;
398 };
399
400 enum class ValueProfilingCallType {
401 // Individual values are tracked. Currently used for indiret call target
402 // profiling.
403 Default,
404
405 // MemOp: the memop size value profiling.
406 MemOp
407 };
408
409 } // end anonymous namespace
410
run(Module & M,ModuleAnalysisManager & AM)411 PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
412 FunctionAnalysisManager &FAM =
413 AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
414 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
415 return FAM.getResult<TargetLibraryAnalysis>(F);
416 };
417 if (!run(M, GetTLI))
418 return PreservedAnalyses::all();
419
420 return PreservedAnalyses::none();
421 }
422
423 char InstrProfilingLegacyPass::ID = 0;
424 INITIALIZE_PASS_BEGIN(
425 InstrProfilingLegacyPass, "instrprof",
426 "Frontend instrumentation-based coverage lowering.", false, false)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)427 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
428 INITIALIZE_PASS_END(
429 InstrProfilingLegacyPass, "instrprof",
430 "Frontend instrumentation-based coverage lowering.", false, false)
431
432 ModulePass *
433 llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options,
434 bool IsCS) {
435 return new InstrProfilingLegacyPass(Options, IsCS);
436 }
437
castToIncrementInst(Instruction * Instr)438 static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
439 InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
440 if (Inc)
441 return Inc;
442 return dyn_cast<InstrProfIncrementInst>(Instr);
443 }
444
lowerIntrinsics(Function * F)445 bool InstrProfiling::lowerIntrinsics(Function *F) {
446 bool MadeChange = false;
447 PromotionCandidates.clear();
448 for (BasicBlock &BB : *F) {
449 for (auto I = BB.begin(), E = BB.end(); I != E;) {
450 auto Instr = I++;
451 InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
452 if (Inc) {
453 lowerIncrement(Inc);
454 MadeChange = true;
455 } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
456 lowerValueProfileInst(Ind);
457 MadeChange = true;
458 }
459 }
460 }
461
462 if (!MadeChange)
463 return false;
464
465 promoteCounterLoadStores(F);
466 return true;
467 }
468
isRuntimeCounterRelocationEnabled() const469 bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
470 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
471 return RuntimeCounterRelocation;
472
473 return TT.isOSFuchsia();
474 }
475
isCounterPromotionEnabled() const476 bool InstrProfiling::isCounterPromotionEnabled() const {
477 if (DoCounterPromotion.getNumOccurrences() > 0)
478 return DoCounterPromotion;
479
480 return Options.DoCounterPromotion;
481 }
482
promoteCounterLoadStores(Function * F)483 void InstrProfiling::promoteCounterLoadStores(Function *F) {
484 if (!isCounterPromotionEnabled())
485 return;
486
487 DominatorTree DT(*F);
488 LoopInfo LI(DT);
489 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
490
491 std::unique_ptr<BlockFrequencyInfo> BFI;
492 if (Options.UseBFIInPromotion) {
493 std::unique_ptr<BranchProbabilityInfo> BPI;
494 BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
495 BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
496 }
497
498 for (const auto &LoadStore : PromotionCandidates) {
499 auto *CounterLoad = LoadStore.first;
500 auto *CounterStore = LoadStore.second;
501 BasicBlock *BB = CounterLoad->getParent();
502 Loop *ParentLoop = LI.getLoopFor(BB);
503 if (!ParentLoop)
504 continue;
505 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
506 }
507
508 SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
509
510 // Do a post-order traversal of the loops so that counter updates can be
511 // iteratively hoisted outside the loop nest.
512 for (auto *Loop : llvm::reverse(Loops)) {
513 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
514 Promoter.run(&TotalCountersPromoted);
515 }
516 }
517
518 /// Check if the module contains uses of any profiling intrinsics.
containsProfilingIntrinsics(Module & M)519 static bool containsProfilingIntrinsics(Module &M) {
520 if (auto *F = M.getFunction(
521 Intrinsic::getName(llvm::Intrinsic::instrprof_increment)))
522 if (!F->use_empty())
523 return true;
524 if (auto *F = M.getFunction(
525 Intrinsic::getName(llvm::Intrinsic::instrprof_increment_step)))
526 if (!F->use_empty())
527 return true;
528 if (auto *F = M.getFunction(
529 Intrinsic::getName(llvm::Intrinsic::instrprof_value_profile)))
530 if (!F->use_empty())
531 return true;
532 return false;
533 }
534
run(Module & M,std::function<const TargetLibraryInfo & (Function & F)> GetTLI)535 bool InstrProfiling::run(
536 Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
537 this->M = &M;
538 this->GetTLI = std::move(GetTLI);
539 NamesVar = nullptr;
540 NamesSize = 0;
541 ProfileDataMap.clear();
542 CompilerUsedVars.clear();
543 UsedVars.clear();
544 TT = Triple(M.getTargetTriple());
545
546 // Emit the runtime hook even if no counters are present.
547 bool MadeChange = emitRuntimeHook();
548
549 // Improve compile time by avoiding linear scans when there is no work.
550 GlobalVariable *CoverageNamesVar =
551 M.getNamedGlobal(getCoverageUnusedNamesVarName());
552 if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
553 return MadeChange;
554
555 // We did not know how many value sites there would be inside
556 // the instrumented function. This is counting the number of instrumented
557 // target value sites to enter it as field in the profile data variable.
558 for (Function &F : M) {
559 InstrProfIncrementInst *FirstProfIncInst = nullptr;
560 for (BasicBlock &BB : F)
561 for (auto I = BB.begin(), E = BB.end(); I != E; I++)
562 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
563 computeNumValueSiteCounts(Ind);
564 else if (FirstProfIncInst == nullptr)
565 FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
566
567 // Value profiling intrinsic lowering requires per-function profile data
568 // variable to be created first.
569 if (FirstProfIncInst != nullptr)
570 static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
571 }
572
573 for (Function &F : M)
574 MadeChange |= lowerIntrinsics(&F);
575
576 if (CoverageNamesVar) {
577 lowerCoverageData(CoverageNamesVar);
578 MadeChange = true;
579 }
580
581 if (!MadeChange)
582 return false;
583
584 emitVNodes();
585 emitNameData();
586 emitRegistration();
587 emitUses();
588 emitInitialization();
589 return true;
590 }
591
getOrInsertValueProfilingCall(Module & M,const TargetLibraryInfo & TLI,ValueProfilingCallType CallType=ValueProfilingCallType::Default)592 static FunctionCallee getOrInsertValueProfilingCall(
593 Module &M, const TargetLibraryInfo &TLI,
594 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
595 LLVMContext &Ctx = M.getContext();
596 auto *ReturnTy = Type::getVoidTy(M.getContext());
597
598 AttributeList AL;
599 if (auto AK = TLI.getExtAttrForI32Param(false))
600 AL = AL.addParamAttribute(M.getContext(), 2, AK);
601
602 assert((CallType == ValueProfilingCallType::Default ||
603 CallType == ValueProfilingCallType::MemOp) &&
604 "Must be Default or MemOp");
605 Type *ParamTypes[] = {
606 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
607 #include "llvm/ProfileData/InstrProfData.inc"
608 };
609 auto *ValueProfilingCallTy =
610 FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
611 StringRef FuncName = CallType == ValueProfilingCallType::Default
612 ? getInstrProfValueProfFuncName()
613 : getInstrProfValueProfMemOpFuncName();
614 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
615 }
616
computeNumValueSiteCounts(InstrProfValueProfileInst * Ind)617 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
618 GlobalVariable *Name = Ind->getName();
619 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
620 uint64_t Index = Ind->getIndex()->getZExtValue();
621 auto It = ProfileDataMap.find(Name);
622 if (It == ProfileDataMap.end()) {
623 PerFunctionProfileData PD;
624 PD.NumValueSites[ValueKind] = Index + 1;
625 ProfileDataMap[Name] = PD;
626 } else if (It->second.NumValueSites[ValueKind] <= Index)
627 It->second.NumValueSites[ValueKind] = Index + 1;
628 }
629
lowerValueProfileInst(InstrProfValueProfileInst * Ind)630 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
631 GlobalVariable *Name = Ind->getName();
632 auto It = ProfileDataMap.find(Name);
633 assert(It != ProfileDataMap.end() && It->second.DataVar &&
634 "value profiling detected in function with no counter incerement");
635
636 GlobalVariable *DataVar = It->second.DataVar;
637 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
638 uint64_t Index = Ind->getIndex()->getZExtValue();
639 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
640 Index += It->second.NumValueSites[Kind];
641
642 IRBuilder<> Builder(Ind);
643 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
644 llvm::InstrProfValueKind::IPVK_MemOPSize);
645 CallInst *Call = nullptr;
646 auto *TLI = &GetTLI(*Ind->getFunction());
647
648 // To support value profiling calls within Windows exception handlers, funclet
649 // information contained within operand bundles needs to be copied over to
650 // the library call. This is required for the IR to be processed by the
651 // WinEHPrepare pass.
652 SmallVector<OperandBundleDef, 1> OpBundles;
653 Ind->getOperandBundlesAsDefs(OpBundles);
654 if (!IsMemOpSize) {
655 Value *Args[3] = {Ind->getTargetValue(),
656 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
657 Builder.getInt32(Index)};
658 Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
659 OpBundles);
660 } else {
661 Value *Args[3] = {Ind->getTargetValue(),
662 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
663 Builder.getInt32(Index)};
664 Call = Builder.CreateCall(
665 getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
666 Args, OpBundles);
667 }
668 if (auto AK = TLI->getExtAttrForI32Param(false))
669 Call->addParamAttr(2, AK);
670 Ind->replaceAllUsesWith(Call);
671 Ind->eraseFromParent();
672 }
673
lowerIncrement(InstrProfIncrementInst * Inc)674 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
675 GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
676
677 IRBuilder<> Builder(Inc);
678 uint64_t Index = Inc->getIndex()->getZExtValue();
679 Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(),
680 Counters, 0, Index);
681
682 if (isRuntimeCounterRelocationEnabled()) {
683 Type *Int64Ty = Type::getInt64Ty(M->getContext());
684 Type *Int64PtrTy = Type::getInt64PtrTy(M->getContext());
685 Function *Fn = Inc->getParent()->getParent();
686 Instruction &I = Fn->getEntryBlock().front();
687 LoadInst *LI = dyn_cast<LoadInst>(&I);
688 if (!LI) {
689 IRBuilder<> Builder(&I);
690 Type *Int64Ty = Type::getInt64Ty(M->getContext());
691 GlobalVariable *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
692 if (!Bias) {
693 Bias = new GlobalVariable(*M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
694 Constant::getNullValue(Int64Ty),
695 getInstrProfCounterBiasVarName());
696 Bias->setVisibility(GlobalVariable::HiddenVisibility);
697 }
698 LI = Builder.CreateLoad(Int64Ty, Bias);
699 }
700 auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), LI);
701 Addr = Builder.CreateIntToPtr(Add, Int64PtrTy);
702 }
703
704 if (Options.Atomic || AtomicCounterUpdateAll ||
705 (Index == 0 && AtomicFirstCounter)) {
706 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
707 MaybeAlign(), AtomicOrdering::Monotonic);
708 } else {
709 Value *IncStep = Inc->getStep();
710 Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
711 auto *Count = Builder.CreateAdd(Load, Inc->getStep());
712 auto *Store = Builder.CreateStore(Count, Addr);
713 if (isCounterPromotionEnabled())
714 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
715 }
716 Inc->eraseFromParent();
717 }
718
lowerCoverageData(GlobalVariable * CoverageNamesVar)719 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
720 ConstantArray *Names =
721 cast<ConstantArray>(CoverageNamesVar->getInitializer());
722 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
723 Constant *NC = Names->getOperand(I);
724 Value *V = NC->stripPointerCasts();
725 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
726 GlobalVariable *Name = cast<GlobalVariable>(V);
727
728 Name->setLinkage(GlobalValue::PrivateLinkage);
729 ReferencedNames.push_back(Name);
730 NC->dropAllReferences();
731 }
732 CoverageNamesVar->eraseFromParent();
733 }
734
735 /// Get the name of a profiling variable for a particular function.
getVarName(InstrProfIncrementInst * Inc,StringRef Prefix)736 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
737 StringRef NamePrefix = getInstrProfNameVarPrefix();
738 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
739 Function *F = Inc->getParent()->getParent();
740 Module *M = F->getParent();
741 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
742 !canRenameComdatFunc(*F))
743 return (Prefix + Name).str();
744 uint64_t FuncHash = Inc->getHash()->getZExtValue();
745 SmallVector<char, 24> HashPostfix;
746 if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
747 return (Prefix + Name).str();
748 return (Prefix + Name + "." + Twine(FuncHash)).str();
749 }
750
getIntModuleFlagOrZero(Module * M,StringRef Flag)751 static uint64_t getIntModuleFlagOrZero(Module *M, StringRef Flag) {
752 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M->getModuleFlag(Flag));
753 if (!MD)
754 return 0;
755
756 // If the flag is a ConstantAsMetadata, it should be an integer representable
757 // in 64-bits.
758 return cast<ConstantInt>(MD->getValue())->getZExtValue();
759 }
760
shouldRecordFunctionAddr(Function * F)761 static inline bool shouldRecordFunctionAddr(Function *F) {
762 // Only record function addresses if IR PGO is enabled or if clang value
763 // profiling is enabled. Recording function addresses greatly increases object
764 // file size, because it prevents the inliner from deleting functions that
765 // have been inlined everywhere.
766 if (!isIRPGOFlagSet(F->getParent()) &&
767 getIntModuleFlagOrZero(F->getParent(), "EnableValueProfiling") == 0) {
768 return false;
769 }
770
771 // Check the linkage
772 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
773 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
774 !HasAvailableExternallyLinkage)
775 return true;
776
777 // A function marked 'alwaysinline' with available_externally linkage can't
778 // have its address taken. Doing so would create an undefined external ref to
779 // the function, which would fail to link.
780 if (HasAvailableExternallyLinkage &&
781 F->hasFnAttribute(Attribute::AlwaysInline))
782 return false;
783
784 // Prohibit function address recording if the function is both internal and
785 // COMDAT. This avoids the profile data variable referencing internal symbols
786 // in COMDAT.
787 if (F->hasLocalLinkage() && F->hasComdat())
788 return false;
789
790 // Check uses of this function for other than direct calls or invokes to it.
791 // Inline virtual functions have linkeOnceODR linkage. When a key method
792 // exists, the vtable will only be emitted in the TU where the key method
793 // is defined. In a TU where vtable is not available, the function won't
794 // be 'addresstaken'. If its address is not recorded here, the profile data
795 // with missing address may be picked by the linker leading to missing
796 // indirect call target info.
797 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
798 }
799
needsRuntimeRegistrationOfSectionRange(const Triple & TT)800 static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
801 // Don't do this for Darwin. compiler-rt uses linker magic.
802 if (TT.isOSDarwin())
803 return false;
804 // Use linker script magic to get data/cnts/name start/end.
805 if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
806 TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS4CPU() ||
807 TT.isOSWindows())
808 return false;
809
810 return true;
811 }
812
813 GlobalVariable *
getOrCreateRegionCounters(InstrProfIncrementInst * Inc)814 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
815 GlobalVariable *NamePtr = Inc->getName();
816 auto It = ProfileDataMap.find(NamePtr);
817 PerFunctionProfileData PD;
818 if (It != ProfileDataMap.end()) {
819 if (It->second.RegionCounters)
820 return It->second.RegionCounters;
821 PD = It->second;
822 }
823
824 // Match the linkage and visibility of the name global. COFF supports using
825 // comdats with internal symbols, so do that if we can.
826 Function *Fn = Inc->getParent()->getParent();
827 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
828 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
829 if (TT.isOSBinFormatCOFF()) {
830 Linkage = GlobalValue::InternalLinkage;
831 Visibility = GlobalValue::DefaultVisibility;
832 }
833
834 // Move the name variable to the right section. Place them in a COMDAT group
835 // if the associated function is a COMDAT. This will make sure that only one
836 // copy of counters of the COMDAT function will be emitted after linking. Keep
837 // in mind that this pass may run before the inliner, so we need to create a
838 // new comdat group for the counters and profiling data. If we use the comdat
839 // of the parent function, that will result in relocations against discarded
840 // sections.
841 bool NeedComdat = needsComdatForCounter(*Fn, *M);
842 if (NeedComdat) {
843 if (TT.isOSBinFormatCOFF()) {
844 // For COFF, put the counters, data, and values each into their own
845 // comdats. We can't use a group because the Visual C++ linker will
846 // report duplicate symbol errors if there are multiple external symbols
847 // with the same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
848 Linkage = GlobalValue::LinkOnceODRLinkage;
849 Visibility = GlobalValue::HiddenVisibility;
850 }
851 }
852 std::string DataVarName = getVarName(Inc, getInstrProfDataVarPrefix());
853 auto MaybeSetComdat = [=](GlobalVariable *GV) {
854 // For ELF, when not using COMDAT, put counters, data and values into
855 // a noduplicates COMDAT which is lowered to a zero-flag section group.
856 // This allows linker GC to discard the entire group when the function
857 // is discarded.
858 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
859 if (UseComdat) {
860 auto GroupName = TT.isOSBinFormatCOFF() ? GV->getName() : DataVarName;
861 Comdat *C = M->getOrInsertComdat(GroupName);
862 if (!NeedComdat)
863 C->setSelectionKind(Comdat::NoDuplicates);
864 GV->setComdat(C);
865 }
866 };
867
868 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
869 LLVMContext &Ctx = M->getContext();
870 ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
871
872 // Create the counters variable.
873 auto *CounterPtr =
874 new GlobalVariable(*M, CounterTy, false, Linkage,
875 Constant::getNullValue(CounterTy),
876 getVarName(Inc, getInstrProfCountersVarPrefix()));
877 CounterPtr->setVisibility(Visibility);
878 CounterPtr->setSection(
879 getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
880 CounterPtr->setAlignment(Align(8));
881 MaybeSetComdat(CounterPtr);
882 CounterPtr->setLinkage(Linkage);
883
884 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
885 // Allocate statically the array of pointers to value profile nodes for
886 // the current function.
887 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
888 if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(TT)) {
889 uint64_t NS = 0;
890 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
891 NS += PD.NumValueSites[Kind];
892 if (NS) {
893 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
894
895 auto *ValuesVar =
896 new GlobalVariable(*M, ValuesTy, false, Linkage,
897 Constant::getNullValue(ValuesTy),
898 getVarName(Inc, getInstrProfValuesVarPrefix()));
899 ValuesVar->setVisibility(Visibility);
900 ValuesVar->setSection(
901 getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
902 ValuesVar->setAlignment(Align(8));
903 MaybeSetComdat(ValuesVar);
904 ValuesPtrExpr =
905 ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
906 }
907 }
908
909 // Create data variable.
910 auto *Int16Ty = Type::getInt16Ty(Ctx);
911 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
912 Type *DataTypes[] = {
913 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
914 #include "llvm/ProfileData/InstrProfData.inc"
915 };
916 auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
917
918 Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
919 ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
920 : ConstantPointerNull::get(Int8PtrTy);
921
922 Constant *Int16ArrayVals[IPVK_Last + 1];
923 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
924 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
925
926 Constant *DataVals[] = {
927 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
928 #include "llvm/ProfileData/InstrProfData.inc"
929 };
930 auto *Data =
931 new GlobalVariable(*M, DataTy, false, Linkage,
932 ConstantStruct::get(DataTy, DataVals), DataVarName);
933 Data->setVisibility(Visibility);
934 Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
935 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
936 MaybeSetComdat(Data);
937 Data->setLinkage(Linkage);
938
939 PD.RegionCounters = CounterPtr;
940 PD.DataVar = Data;
941 ProfileDataMap[NamePtr] = PD;
942
943 // Mark the data variable as used so that it isn't stripped out.
944 CompilerUsedVars.push_back(Data);
945 // Now that the linkage set by the FE has been passed to the data and counter
946 // variables, reset Name variable's linkage and visibility to private so that
947 // it can be removed later by the compiler.
948 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
949 // Collect the referenced names to be used by emitNameData.
950 ReferencedNames.push_back(NamePtr);
951
952 return CounterPtr;
953 }
954
emitVNodes()955 void InstrProfiling::emitVNodes() {
956 if (!ValueProfileStaticAlloc)
957 return;
958
959 // For now only support this on platforms that do
960 // not require runtime registration to discover
961 // named section start/end.
962 if (needsRuntimeRegistrationOfSectionRange(TT))
963 return;
964
965 size_t TotalNS = 0;
966 for (auto &PD : ProfileDataMap) {
967 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
968 TotalNS += PD.second.NumValueSites[Kind];
969 }
970
971 if (!TotalNS)
972 return;
973
974 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
975 // Heuristic for small programs with very few total value sites.
976 // The default value of vp-counters-per-site is chosen based on
977 // the observation that large apps usually have a low percentage
978 // of value sites that actually have any profile data, and thus
979 // the average number of counters per site is low. For small
980 // apps with very few sites, this may not be true. Bump up the
981 // number of counters in this case.
982 #define INSTR_PROF_MIN_VAL_COUNTS 10
983 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
984 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
985
986 auto &Ctx = M->getContext();
987 Type *VNodeTypes[] = {
988 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
989 #include "llvm/ProfileData/InstrProfData.inc"
990 };
991 auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
992
993 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
994 auto *VNodesVar = new GlobalVariable(
995 *M, VNodesTy, false, GlobalValue::PrivateLinkage,
996 Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
997 VNodesVar->setSection(
998 getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
999 // VNodesVar is used by runtime but not referenced via relocation by other
1000 // sections. Conservatively make it linker retained.
1001 UsedVars.push_back(VNodesVar);
1002 }
1003
emitNameData()1004 void InstrProfiling::emitNameData() {
1005 std::string UncompressedData;
1006
1007 if (ReferencedNames.empty())
1008 return;
1009
1010 std::string CompressedNameStr;
1011 if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1012 DoInstrProfNameCompression)) {
1013 report_fatal_error(toString(std::move(E)), false);
1014 }
1015
1016 auto &Ctx = M->getContext();
1017 auto *NamesVal = ConstantDataArray::getString(
1018 Ctx, StringRef(CompressedNameStr), false);
1019 NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
1020 GlobalValue::PrivateLinkage, NamesVal,
1021 getInstrProfNamesVarName());
1022 NamesSize = CompressedNameStr.size();
1023 NamesVar->setSection(
1024 getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1025 // On COFF, it's important to reduce the alignment down to 1 to prevent the
1026 // linker from inserting padding before the start of the names section or
1027 // between names entries.
1028 NamesVar->setAlignment(Align(1));
1029 // NamesVar is used by runtime but not referenced via relocation by other
1030 // sections. Conservatively make it linker retained.
1031 UsedVars.push_back(NamesVar);
1032
1033 for (auto *NamePtr : ReferencedNames)
1034 NamePtr->eraseFromParent();
1035 }
1036
emitRegistration()1037 void InstrProfiling::emitRegistration() {
1038 if (!needsRuntimeRegistrationOfSectionRange(TT))
1039 return;
1040
1041 // Construct the function.
1042 auto *VoidTy = Type::getVoidTy(M->getContext());
1043 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
1044 auto *Int64Ty = Type::getInt64Ty(M->getContext());
1045 auto *RegisterFTy = FunctionType::get(VoidTy, false);
1046 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1047 getInstrProfRegFuncsName(), M);
1048 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1049 if (Options.NoRedZone)
1050 RegisterF->addFnAttr(Attribute::NoRedZone);
1051
1052 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1053 auto *RuntimeRegisterF =
1054 Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
1055 getInstrProfRegFuncName(), M);
1056
1057 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1058 for (Value *Data : CompilerUsedVars)
1059 if (!isa<Function>(Data))
1060 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1061 for (Value *Data : UsedVars)
1062 if (Data != NamesVar && !isa<Function>(Data))
1063 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1064
1065 if (NamesVar) {
1066 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1067 auto *NamesRegisterTy =
1068 FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
1069 auto *NamesRegisterF =
1070 Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
1071 getInstrProfNamesRegFuncName(), M);
1072 IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
1073 IRB.getInt64(NamesSize)});
1074 }
1075
1076 IRB.CreateRetVoid();
1077 }
1078
emitRuntimeHook()1079 bool InstrProfiling::emitRuntimeHook() {
1080 // We expect the linker to be invoked with -u<hook_var> flag for Linux or
1081 // Fuchsia, in which case there is no need to emit the user function.
1082 if (TT.isOSLinux() || TT.isOSFuchsia())
1083 return false;
1084
1085 // If the module's provided its own runtime, we don't need to do anything.
1086 if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1087 return false;
1088
1089 // Declare an external variable that will pull in the runtime initialization.
1090 auto *Int32Ty = Type::getInt32Ty(M->getContext());
1091 auto *Var =
1092 new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
1093 nullptr, getInstrProfRuntimeHookVarName());
1094
1095 // Make a function that uses it.
1096 auto *User = Function::Create(FunctionType::get(Int32Ty, false),
1097 GlobalValue::LinkOnceODRLinkage,
1098 getInstrProfRuntimeHookVarUseFuncName(), M);
1099 User->addFnAttr(Attribute::NoInline);
1100 if (Options.NoRedZone)
1101 User->addFnAttr(Attribute::NoRedZone);
1102 User->setVisibility(GlobalValue::HiddenVisibility);
1103 if (TT.supportsCOMDAT())
1104 User->setComdat(M->getOrInsertComdat(User->getName()));
1105
1106 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1107 auto *Load = IRB.CreateLoad(Int32Ty, Var);
1108 IRB.CreateRet(Load);
1109
1110 // Mark the user variable as used so that it isn't stripped out.
1111 CompilerUsedVars.push_back(User);
1112 return true;
1113 }
1114
emitUses()1115 void InstrProfiling::emitUses() {
1116 // The metadata sections are parallel arrays. Optimizers (e.g.
1117 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1118 // we conservatively retain all unconditionally in the compiler.
1119 //
1120 // On ELF, the linker can guarantee the associated sections will be retained
1121 // or discarded as a unit, so llvm.compiler.used is sufficient. Otherwise,
1122 // conservatively make all of them retained by the linker.
1123 if (TT.isOSBinFormatELF())
1124 appendToCompilerUsed(*M, CompilerUsedVars);
1125 else
1126 appendToUsed(*M, CompilerUsedVars);
1127
1128 // We do not add proper references from used metadata sections to NamesVar and
1129 // VNodesVar, so we have to be conservative and place them in llvm.used
1130 // regardless of the target,
1131 appendToUsed(*M, UsedVars);
1132 }
1133
emitInitialization()1134 void InstrProfiling::emitInitialization() {
1135 // Create ProfileFileName variable. Don't don't this for the
1136 // context-sensitive instrumentation lowering: This lowering is after
1137 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1138 // have already create the variable before LTO/ThinLTO linking.
1139 if (!IsCS)
1140 createProfileFileNameVar(*M, Options.InstrProfileOutput);
1141 Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1142 if (!RegisterF)
1143 return;
1144
1145 // Create the initialization function.
1146 auto *VoidTy = Type::getVoidTy(M->getContext());
1147 auto *F = Function::Create(FunctionType::get(VoidTy, false),
1148 GlobalValue::InternalLinkage,
1149 getInstrProfInitFuncName(), M);
1150 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1151 F->addFnAttr(Attribute::NoInline);
1152 if (Options.NoRedZone)
1153 F->addFnAttr(Attribute::NoRedZone);
1154
1155 // Add the basic block and the necessary calls.
1156 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1157 IRB.CreateCall(RegisterF, {});
1158 IRB.CreateRetVoid();
1159
1160 appendToGlobalCtors(*M, F, 0);
1161 }
1162