10b57cec5SDimitry Andric //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling. 100b57cec5SDimitry Andric // It also builds the data structures and initialization code needed for 110b57cec5SDimitry Andric // updating execution counts and emitting the profile at runtime. 120b57cec5SDimitry Andric // 130b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 140b57cec5SDimitry Andric 150b57cec5SDimitry Andric #include "llvm/Transforms/Instrumentation/InstrProfiling.h" 160b57cec5SDimitry Andric #include "llvm/ADT/ArrayRef.h" 170b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h" 180b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h" 190b57cec5SDimitry Andric #include "llvm/ADT/Triple.h" 200b57cec5SDimitry Andric #include "llvm/ADT/Twine.h" 210b57cec5SDimitry Andric #include "llvm/Analysis/BlockFrequencyInfo.h" 220b57cec5SDimitry Andric #include "llvm/Analysis/BranchProbabilityInfo.h" 230b57cec5SDimitry Andric #include "llvm/Analysis/LoopInfo.h" 240b57cec5SDimitry Andric #include "llvm/Analysis/TargetLibraryInfo.h" 250b57cec5SDimitry Andric #include "llvm/IR/Attributes.h" 260b57cec5SDimitry Andric #include "llvm/IR/BasicBlock.h" 270b57cec5SDimitry Andric #include "llvm/IR/Constant.h" 280b57cec5SDimitry Andric #include "llvm/IR/Constants.h" 290eae32dcSDimitry Andric #include "llvm/IR/DIBuilder.h" 300b57cec5SDimitry Andric #include "llvm/IR/DerivedTypes.h" 310eae32dcSDimitry Andric #include "llvm/IR/DiagnosticInfo.h" 320b57cec5SDimitry Andric #include "llvm/IR/Dominators.h" 330b57cec5SDimitry Andric #include "llvm/IR/Function.h" 340b57cec5SDimitry Andric #include "llvm/IR/GlobalValue.h" 350b57cec5SDimitry Andric #include "llvm/IR/GlobalVariable.h" 360b57cec5SDimitry Andric #include "llvm/IR/IRBuilder.h" 370b57cec5SDimitry Andric #include "llvm/IR/Instruction.h" 380b57cec5SDimitry Andric #include "llvm/IR/Instructions.h" 390b57cec5SDimitry Andric #include "llvm/IR/IntrinsicInst.h" 400b57cec5SDimitry Andric #include "llvm/IR/Module.h" 410b57cec5SDimitry Andric #include "llvm/IR/Type.h" 42480093f4SDimitry Andric #include "llvm/InitializePasses.h" 430b57cec5SDimitry Andric #include "llvm/Pass.h" 440b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProf.h" 450eae32dcSDimitry Andric #include "llvm/ProfileData/InstrProfCorrelator.h" 460b57cec5SDimitry Andric #include "llvm/Support/Casting.h" 470b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h" 480b57cec5SDimitry Andric #include "llvm/Support/Error.h" 490b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h" 500b57cec5SDimitry Andric #include "llvm/Transforms/Utils/ModuleUtils.h" 510b57cec5SDimitry Andric #include "llvm/Transforms/Utils/SSAUpdater.h" 520b57cec5SDimitry Andric #include <algorithm> 530b57cec5SDimitry Andric #include <cassert> 540b57cec5SDimitry Andric #include <cstdint> 550b57cec5SDimitry Andric #include <string> 560b57cec5SDimitry Andric 570b57cec5SDimitry Andric using namespace llvm; 580b57cec5SDimitry Andric 590b57cec5SDimitry Andric #define DEBUG_TYPE "instrprof" 600b57cec5SDimitry Andric 610eae32dcSDimitry Andric namespace llvm { 620eae32dcSDimitry Andric cl::opt<bool> 6381ad6265SDimitry Andric DebugInfoCorrelate("debug-info-correlate", 640eae32dcSDimitry Andric cl::desc("Use debug info to correlate profiles."), 650eae32dcSDimitry Andric cl::init(false)); 660eae32dcSDimitry Andric } // namespace llvm 670eae32dcSDimitry Andric 680b57cec5SDimitry Andric namespace { 690b57cec5SDimitry Andric 700b57cec5SDimitry Andric cl::opt<bool> DoHashBasedCounterSplit( 710b57cec5SDimitry Andric "hash-based-counter-split", 720b57cec5SDimitry Andric cl::desc("Rename counter variable of a comdat function based on cfg hash"), 730b57cec5SDimitry Andric cl::init(true)); 740b57cec5SDimitry Andric 754824e7fdSDimitry Andric cl::opt<bool> 764824e7fdSDimitry Andric RuntimeCounterRelocation("runtime-counter-relocation", 775ffd83dbSDimitry Andric cl::desc("Enable relocating counters at runtime."), 785ffd83dbSDimitry Andric cl::init(false)); 795ffd83dbSDimitry Andric 800b57cec5SDimitry Andric cl::opt<bool> ValueProfileStaticAlloc( 810b57cec5SDimitry Andric "vp-static-alloc", 820b57cec5SDimitry Andric cl::desc("Do static counter allocation for value profiler"), 830b57cec5SDimitry Andric cl::init(true)); 840b57cec5SDimitry Andric 850b57cec5SDimitry Andric cl::opt<double> NumCountersPerValueSite( 860b57cec5SDimitry Andric "vp-counters-per-site", 870b57cec5SDimitry Andric cl::desc("The average number of profile counters allocated " 880b57cec5SDimitry Andric "per value profiling site."), 890b57cec5SDimitry Andric // This is set to a very small value because in real programs, only 900b57cec5SDimitry Andric // a very small percentage of value sites have non-zero targets, e.g, 1/30. 910b57cec5SDimitry Andric // For those sites with non-zero profile, the average number of targets 920b57cec5SDimitry Andric // is usually smaller than 2. 930b57cec5SDimitry Andric cl::init(1.0)); 940b57cec5SDimitry Andric 950b57cec5SDimitry Andric cl::opt<bool> AtomicCounterUpdateAll( 9681ad6265SDimitry Andric "instrprof-atomic-counter-update-all", 970b57cec5SDimitry Andric cl::desc("Make all profile counter updates atomic (for testing only)"), 980b57cec5SDimitry Andric cl::init(false)); 990b57cec5SDimitry Andric 1000b57cec5SDimitry Andric cl::opt<bool> AtomicCounterUpdatePromoted( 10181ad6265SDimitry Andric "atomic-counter-update-promoted", 1020b57cec5SDimitry Andric cl::desc("Do counter update using atomic fetch add " 1030b57cec5SDimitry Andric " for promoted counters only"), 1040b57cec5SDimitry Andric cl::init(false)); 1050b57cec5SDimitry Andric 1065ffd83dbSDimitry Andric cl::opt<bool> AtomicFirstCounter( 10781ad6265SDimitry Andric "atomic-first-counter", 1085ffd83dbSDimitry Andric cl::desc("Use atomic fetch add for first counter in a function (usually " 1095ffd83dbSDimitry Andric "the entry counter)"), 1105ffd83dbSDimitry Andric cl::init(false)); 1115ffd83dbSDimitry Andric 1120b57cec5SDimitry Andric // If the option is not specified, the default behavior about whether 1130b57cec5SDimitry Andric // counter promotion is done depends on how instrumentaiton lowering 1140b57cec5SDimitry Andric // pipeline is setup, i.e., the default value of true of this option 1150b57cec5SDimitry Andric // does not mean the promotion will be done by default. Explicitly 1160b57cec5SDimitry Andric // setting this option can override the default behavior. 11781ad6265SDimitry Andric cl::opt<bool> DoCounterPromotion("do-counter-promotion", 1180b57cec5SDimitry Andric cl::desc("Do counter register promotion"), 1190b57cec5SDimitry Andric cl::init(false)); 1200b57cec5SDimitry Andric cl::opt<unsigned> MaxNumOfPromotionsPerLoop( 12181ad6265SDimitry Andric "max-counter-promotions-per-loop", cl::init(20), 1220b57cec5SDimitry Andric cl::desc("Max number counter promotions per loop to avoid" 1230b57cec5SDimitry Andric " increasing register pressure too much")); 1240b57cec5SDimitry Andric 1250b57cec5SDimitry Andric // A debug option 1260b57cec5SDimitry Andric cl::opt<int> 12781ad6265SDimitry Andric MaxNumOfPromotions("max-counter-promotions", cl::init(-1), 1280b57cec5SDimitry Andric cl::desc("Max number of allowed counter promotions")); 1290b57cec5SDimitry Andric 1300b57cec5SDimitry Andric cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting( 13181ad6265SDimitry Andric "speculative-counter-promotion-max-exiting", cl::init(3), 1320b57cec5SDimitry Andric cl::desc("The max number of exiting blocks of a loop to allow " 1330b57cec5SDimitry Andric " speculative counter promotion")); 1340b57cec5SDimitry Andric 1350b57cec5SDimitry Andric cl::opt<bool> SpeculativeCounterPromotionToLoop( 13681ad6265SDimitry Andric "speculative-counter-promotion-to-loop", 1370b57cec5SDimitry Andric cl::desc("When the option is false, if the target block is in a loop, " 1380b57cec5SDimitry Andric "the promotion will be disallowed unless the promoted counter " 1390b57cec5SDimitry Andric " update can be further/iteratively promoted into an acyclic " 1400b57cec5SDimitry Andric " region.")); 1410b57cec5SDimitry Andric 1420b57cec5SDimitry Andric cl::opt<bool> IterativeCounterPromotion( 14381ad6265SDimitry Andric "iterative-counter-promotion", cl::init(true), 1440b57cec5SDimitry Andric cl::desc("Allow counter promotion across the whole loop nest.")); 1450b57cec5SDimitry Andric 146e8d8bef9SDimitry Andric cl::opt<bool> SkipRetExitBlock( 14781ad6265SDimitry Andric "skip-ret-exit-block", cl::init(true), 148e8d8bef9SDimitry Andric cl::desc("Suppress counter promotion if exit blocks contain ret.")); 149e8d8bef9SDimitry Andric 1500b57cec5SDimitry Andric class InstrProfilingLegacyPass : public ModulePass { 1510b57cec5SDimitry Andric InstrProfiling InstrProf; 1520b57cec5SDimitry Andric 1530b57cec5SDimitry Andric public: 1540b57cec5SDimitry Andric static char ID; 1550b57cec5SDimitry Andric 1560b57cec5SDimitry Andric InstrProfilingLegacyPass() : ModulePass(ID) {} 1570b57cec5SDimitry Andric InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false) 1585ffd83dbSDimitry Andric : ModulePass(ID), InstrProf(Options, IsCS) { 1595ffd83dbSDimitry Andric initializeInstrProfilingLegacyPassPass(*PassRegistry::getPassRegistry()); 1605ffd83dbSDimitry Andric } 1610b57cec5SDimitry Andric 1620b57cec5SDimitry Andric StringRef getPassName() const override { 1630b57cec5SDimitry Andric return "Frontend instrumentation-based coverage lowering"; 1640b57cec5SDimitry Andric } 1650b57cec5SDimitry Andric 1660b57cec5SDimitry Andric bool runOnModule(Module &M) override { 1678bcb0991SDimitry Andric auto GetTLI = [this](Function &F) -> TargetLibraryInfo & { 1688bcb0991SDimitry Andric return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 1698bcb0991SDimitry Andric }; 1708bcb0991SDimitry Andric return InstrProf.run(M, GetTLI); 1710b57cec5SDimitry Andric } 1720b57cec5SDimitry Andric 1730b57cec5SDimitry Andric void getAnalysisUsage(AnalysisUsage &AU) const override { 1740b57cec5SDimitry Andric AU.setPreservesCFG(); 1750b57cec5SDimitry Andric AU.addRequired<TargetLibraryInfoWrapperPass>(); 1760b57cec5SDimitry Andric } 1770b57cec5SDimitry Andric }; 1780b57cec5SDimitry Andric 1790b57cec5SDimitry Andric /// 1800b57cec5SDimitry Andric /// A helper class to promote one counter RMW operation in the loop 1810b57cec5SDimitry Andric /// into register update. 1820b57cec5SDimitry Andric /// 1830b57cec5SDimitry Andric /// RWM update for the counter will be sinked out of the loop after 1840b57cec5SDimitry Andric /// the transformation. 1850b57cec5SDimitry Andric /// 1860b57cec5SDimitry Andric class PGOCounterPromoterHelper : public LoadAndStorePromoter { 1870b57cec5SDimitry Andric public: 1880b57cec5SDimitry Andric PGOCounterPromoterHelper( 1890b57cec5SDimitry Andric Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init, 1900b57cec5SDimitry Andric BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks, 1910b57cec5SDimitry Andric ArrayRef<Instruction *> InsertPts, 1920b57cec5SDimitry Andric DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands, 1930b57cec5SDimitry Andric LoopInfo &LI) 1940b57cec5SDimitry Andric : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks), 1950b57cec5SDimitry Andric InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) { 1960b57cec5SDimitry Andric assert(isa<LoadInst>(L)); 1970b57cec5SDimitry Andric assert(isa<StoreInst>(S)); 1980b57cec5SDimitry Andric SSA.AddAvailableValue(PH, Init); 1990b57cec5SDimitry Andric } 2000b57cec5SDimitry Andric 2010b57cec5SDimitry Andric void doExtraRewritesBeforeFinalDeletion() override { 2020b57cec5SDimitry Andric for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) { 2030b57cec5SDimitry Andric BasicBlock *ExitBlock = ExitBlocks[i]; 2040b57cec5SDimitry Andric Instruction *InsertPos = InsertPts[i]; 2050b57cec5SDimitry Andric // Get LiveIn value into the ExitBlock. If there are multiple 2060b57cec5SDimitry Andric // predecessors, the value is defined by a PHI node in this 2070b57cec5SDimitry Andric // block. 2080b57cec5SDimitry Andric Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock); 2090b57cec5SDimitry Andric Value *Addr = cast<StoreInst>(Store)->getPointerOperand(); 2100b57cec5SDimitry Andric Type *Ty = LiveInValue->getType(); 2110b57cec5SDimitry Andric IRBuilder<> Builder(InsertPos); 21281ad6265SDimitry Andric if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Addr)) { 21381ad6265SDimitry Andric // If isRuntimeCounterRelocationEnabled() is true then the address of 21481ad6265SDimitry Andric // the store instruction is computed with two instructions in 21581ad6265SDimitry Andric // InstrProfiling::getCounterAddress(). We need to copy those 21681ad6265SDimitry Andric // instructions to this block to compute Addr correctly. 21781ad6265SDimitry Andric // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias> 21881ad6265SDimitry Andric // %Addr = inttoptr i64 %BiasAdd to i64* 21981ad6265SDimitry Andric auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0)); 22081ad6265SDimitry Andric assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add); 22181ad6265SDimitry Andric Value *BiasInst = Builder.Insert(OrigBiasInst->clone()); 22281ad6265SDimitry Andric Addr = Builder.CreateIntToPtr(BiasInst, Ty->getPointerTo()); 22381ad6265SDimitry Andric } 2240b57cec5SDimitry Andric if (AtomicCounterUpdatePromoted) 2250b57cec5SDimitry Andric // automic update currently can only be promoted across the current 2260b57cec5SDimitry Andric // loop, not the whole loop nest. 2270b57cec5SDimitry Andric Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue, 228fe6060f1SDimitry Andric MaybeAlign(), 2290b57cec5SDimitry Andric AtomicOrdering::SequentiallyConsistent); 2300b57cec5SDimitry Andric else { 2310b57cec5SDimitry Andric LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted"); 2320b57cec5SDimitry Andric auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue); 2330b57cec5SDimitry Andric auto *NewStore = Builder.CreateStore(NewVal, Addr); 2340b57cec5SDimitry Andric 2350b57cec5SDimitry Andric // Now update the parent loop's candidate list: 2360b57cec5SDimitry Andric if (IterativeCounterPromotion) { 2370b57cec5SDimitry Andric auto *TargetLoop = LI.getLoopFor(ExitBlock); 2380b57cec5SDimitry Andric if (TargetLoop) 2390b57cec5SDimitry Andric LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore); 2400b57cec5SDimitry Andric } 2410b57cec5SDimitry Andric } 2420b57cec5SDimitry Andric } 2430b57cec5SDimitry Andric } 2440b57cec5SDimitry Andric 2450b57cec5SDimitry Andric private: 2460b57cec5SDimitry Andric Instruction *Store; 2470b57cec5SDimitry Andric ArrayRef<BasicBlock *> ExitBlocks; 2480b57cec5SDimitry Andric ArrayRef<Instruction *> InsertPts; 2490b57cec5SDimitry Andric DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates; 2500b57cec5SDimitry Andric LoopInfo &LI; 2510b57cec5SDimitry Andric }; 2520b57cec5SDimitry Andric 2530b57cec5SDimitry Andric /// A helper class to do register promotion for all profile counter 2540b57cec5SDimitry Andric /// updates in a loop. 2550b57cec5SDimitry Andric /// 2560b57cec5SDimitry Andric class PGOCounterPromoter { 2570b57cec5SDimitry Andric public: 2580b57cec5SDimitry Andric PGOCounterPromoter( 2590b57cec5SDimitry Andric DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands, 2600b57cec5SDimitry Andric Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI) 26104eeddc0SDimitry Andric : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) { 2620b57cec5SDimitry Andric 2635ffd83dbSDimitry Andric // Skip collection of ExitBlocks and InsertPts for loops that will not be 2645ffd83dbSDimitry Andric // able to have counters promoted. 2650b57cec5SDimitry Andric SmallVector<BasicBlock *, 8> LoopExitBlocks; 2660b57cec5SDimitry Andric SmallPtrSet<BasicBlock *, 8> BlockSet; 2675ffd83dbSDimitry Andric 2680b57cec5SDimitry Andric L.getExitBlocks(LoopExitBlocks); 2695ffd83dbSDimitry Andric if (!isPromotionPossible(&L, LoopExitBlocks)) 2705ffd83dbSDimitry Andric return; 2710b57cec5SDimitry Andric 2720b57cec5SDimitry Andric for (BasicBlock *ExitBlock : LoopExitBlocks) { 2730b57cec5SDimitry Andric if (BlockSet.insert(ExitBlock).second) { 2740b57cec5SDimitry Andric ExitBlocks.push_back(ExitBlock); 2750b57cec5SDimitry Andric InsertPts.push_back(&*ExitBlock->getFirstInsertionPt()); 2760b57cec5SDimitry Andric } 2770b57cec5SDimitry Andric } 2780b57cec5SDimitry Andric } 2790b57cec5SDimitry Andric 2800b57cec5SDimitry Andric bool run(int64_t *NumPromoted) { 2810b57cec5SDimitry Andric // Skip 'infinite' loops: 2820b57cec5SDimitry Andric if (ExitBlocks.size() == 0) 2830b57cec5SDimitry Andric return false; 284e8d8bef9SDimitry Andric 285e8d8bef9SDimitry Andric // Skip if any of the ExitBlocks contains a ret instruction. 286e8d8bef9SDimitry Andric // This is to prevent dumping of incomplete profile -- if the 287e8d8bef9SDimitry Andric // the loop is a long running loop and dump is called in the middle 288e8d8bef9SDimitry Andric // of the loop, the result profile is incomplete. 289e8d8bef9SDimitry Andric // FIXME: add other heuristics to detect long running loops. 290e8d8bef9SDimitry Andric if (SkipRetExitBlock) { 291e8d8bef9SDimitry Andric for (auto BB : ExitBlocks) 292e8d8bef9SDimitry Andric if (isa<ReturnInst>(BB->getTerminator())) 293e8d8bef9SDimitry Andric return false; 294e8d8bef9SDimitry Andric } 295e8d8bef9SDimitry Andric 2960b57cec5SDimitry Andric unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L); 2970b57cec5SDimitry Andric if (MaxProm == 0) 2980b57cec5SDimitry Andric return false; 2990b57cec5SDimitry Andric 3000b57cec5SDimitry Andric unsigned Promoted = 0; 3010b57cec5SDimitry Andric for (auto &Cand : LoopToCandidates[&L]) { 3020b57cec5SDimitry Andric 3030b57cec5SDimitry Andric SmallVector<PHINode *, 4> NewPHIs; 3040b57cec5SDimitry Andric SSAUpdater SSA(&NewPHIs); 3050b57cec5SDimitry Andric Value *InitVal = ConstantInt::get(Cand.first->getType(), 0); 3060b57cec5SDimitry Andric 3070b57cec5SDimitry Andric // If BFI is set, we will use it to guide the promotions. 3080b57cec5SDimitry Andric if (BFI) { 3090b57cec5SDimitry Andric auto *BB = Cand.first->getParent(); 3100b57cec5SDimitry Andric auto InstrCount = BFI->getBlockProfileCount(BB); 3110b57cec5SDimitry Andric if (!InstrCount) 3120b57cec5SDimitry Andric continue; 3130b57cec5SDimitry Andric auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader()); 3140b57cec5SDimitry Andric // If the average loop trip count is not greater than 1.5, we skip 3150b57cec5SDimitry Andric // promotion. 31681ad6265SDimitry Andric if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2)) 3170b57cec5SDimitry Andric continue; 3180b57cec5SDimitry Andric } 3190b57cec5SDimitry Andric 3200b57cec5SDimitry Andric PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal, 3210b57cec5SDimitry Andric L.getLoopPreheader(), ExitBlocks, 3220b57cec5SDimitry Andric InsertPts, LoopToCandidates, LI); 3230b57cec5SDimitry Andric Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second})); 3240b57cec5SDimitry Andric Promoted++; 3250b57cec5SDimitry Andric if (Promoted >= MaxProm) 3260b57cec5SDimitry Andric break; 3270b57cec5SDimitry Andric 3280b57cec5SDimitry Andric (*NumPromoted)++; 3290b57cec5SDimitry Andric if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions) 3300b57cec5SDimitry Andric break; 3310b57cec5SDimitry Andric } 3320b57cec5SDimitry Andric 3330b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth=" 3340b57cec5SDimitry Andric << L.getLoopDepth() << ")\n"); 3350b57cec5SDimitry Andric return Promoted != 0; 3360b57cec5SDimitry Andric } 3370b57cec5SDimitry Andric 3380b57cec5SDimitry Andric private: 3390b57cec5SDimitry Andric bool allowSpeculativeCounterPromotion(Loop *LP) { 3400b57cec5SDimitry Andric SmallVector<BasicBlock *, 8> ExitingBlocks; 3410b57cec5SDimitry Andric L.getExitingBlocks(ExitingBlocks); 3420b57cec5SDimitry Andric // Not considierered speculative. 3430b57cec5SDimitry Andric if (ExitingBlocks.size() == 1) 3440b57cec5SDimitry Andric return true; 3450b57cec5SDimitry Andric if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting) 3460b57cec5SDimitry Andric return false; 3470b57cec5SDimitry Andric return true; 3480b57cec5SDimitry Andric } 3490b57cec5SDimitry Andric 3505ffd83dbSDimitry Andric // Check whether the loop satisfies the basic conditions needed to perform 3515ffd83dbSDimitry Andric // Counter Promotions. 3524824e7fdSDimitry Andric bool 3534824e7fdSDimitry Andric isPromotionPossible(Loop *LP, 3545ffd83dbSDimitry Andric const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) { 3550b57cec5SDimitry Andric // We can't insert into a catchswitch. 3560b57cec5SDimitry Andric if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) { 3570b57cec5SDimitry Andric return isa<CatchSwitchInst>(Exit->getTerminator()); 3580b57cec5SDimitry Andric })) 3595ffd83dbSDimitry Andric return false; 3600b57cec5SDimitry Andric 3610b57cec5SDimitry Andric if (!LP->hasDedicatedExits()) 3625ffd83dbSDimitry Andric return false; 3630b57cec5SDimitry Andric 3640b57cec5SDimitry Andric BasicBlock *PH = LP->getLoopPreheader(); 3650b57cec5SDimitry Andric if (!PH) 3665ffd83dbSDimitry Andric return false; 3675ffd83dbSDimitry Andric 3685ffd83dbSDimitry Andric return true; 3695ffd83dbSDimitry Andric } 3705ffd83dbSDimitry Andric 3715ffd83dbSDimitry Andric // Returns the max number of Counter Promotions for LP. 3725ffd83dbSDimitry Andric unsigned getMaxNumOfPromotionsInLoop(Loop *LP) { 3735ffd83dbSDimitry Andric SmallVector<BasicBlock *, 8> LoopExitBlocks; 3745ffd83dbSDimitry Andric LP->getExitBlocks(LoopExitBlocks); 3755ffd83dbSDimitry Andric if (!isPromotionPossible(LP, LoopExitBlocks)) 3760b57cec5SDimitry Andric return 0; 3770b57cec5SDimitry Andric 3780b57cec5SDimitry Andric SmallVector<BasicBlock *, 8> ExitingBlocks; 3790b57cec5SDimitry Andric LP->getExitingBlocks(ExitingBlocks); 3800b57cec5SDimitry Andric 3810b57cec5SDimitry Andric // If BFI is set, we do more aggressive promotions based on BFI. 3820b57cec5SDimitry Andric if (BFI) 3830b57cec5SDimitry Andric return (unsigned)-1; 3840b57cec5SDimitry Andric 3850b57cec5SDimitry Andric // Not considierered speculative. 3860b57cec5SDimitry Andric if (ExitingBlocks.size() == 1) 3870b57cec5SDimitry Andric return MaxNumOfPromotionsPerLoop; 3880b57cec5SDimitry Andric 3890b57cec5SDimitry Andric if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting) 3900b57cec5SDimitry Andric return 0; 3910b57cec5SDimitry Andric 3920b57cec5SDimitry Andric // Whether the target block is in a loop does not matter: 3930b57cec5SDimitry Andric if (SpeculativeCounterPromotionToLoop) 3940b57cec5SDimitry Andric return MaxNumOfPromotionsPerLoop; 3950b57cec5SDimitry Andric 3960b57cec5SDimitry Andric // Now check the target block: 3970b57cec5SDimitry Andric unsigned MaxProm = MaxNumOfPromotionsPerLoop; 3980b57cec5SDimitry Andric for (auto *TargetBlock : LoopExitBlocks) { 3990b57cec5SDimitry Andric auto *TargetLoop = LI.getLoopFor(TargetBlock); 4000b57cec5SDimitry Andric if (!TargetLoop) 4010b57cec5SDimitry Andric continue; 4020b57cec5SDimitry Andric unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop); 4030b57cec5SDimitry Andric unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size(); 4040b57cec5SDimitry Andric MaxProm = 4050b57cec5SDimitry Andric std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) - 4060b57cec5SDimitry Andric PendingCandsInTarget); 4070b57cec5SDimitry Andric } 4080b57cec5SDimitry Andric return MaxProm; 4090b57cec5SDimitry Andric } 4100b57cec5SDimitry Andric 4110b57cec5SDimitry Andric DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates; 4120b57cec5SDimitry Andric SmallVector<BasicBlock *, 8> ExitBlocks; 4130b57cec5SDimitry Andric SmallVector<Instruction *, 8> InsertPts; 4140b57cec5SDimitry Andric Loop &L; 4150b57cec5SDimitry Andric LoopInfo &LI; 4160b57cec5SDimitry Andric BlockFrequencyInfo *BFI; 4170b57cec5SDimitry Andric }; 4180b57cec5SDimitry Andric 419e8d8bef9SDimitry Andric enum class ValueProfilingCallType { 420e8d8bef9SDimitry Andric // Individual values are tracked. Currently used for indiret call target 421e8d8bef9SDimitry Andric // profiling. 422e8d8bef9SDimitry Andric Default, 423e8d8bef9SDimitry Andric 424e8d8bef9SDimitry Andric // MemOp: the memop size value profiling. 425e8d8bef9SDimitry Andric MemOp 426e8d8bef9SDimitry Andric }; 427e8d8bef9SDimitry Andric 4280b57cec5SDimitry Andric } // end anonymous namespace 4290b57cec5SDimitry Andric 4300b57cec5SDimitry Andric PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) { 4318bcb0991SDimitry Andric FunctionAnalysisManager &FAM = 4328bcb0991SDimitry Andric AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 4338bcb0991SDimitry Andric auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & { 4348bcb0991SDimitry Andric return FAM.getResult<TargetLibraryAnalysis>(F); 4358bcb0991SDimitry Andric }; 4368bcb0991SDimitry Andric if (!run(M, GetTLI)) 4370b57cec5SDimitry Andric return PreservedAnalyses::all(); 4380b57cec5SDimitry Andric 4390b57cec5SDimitry Andric return PreservedAnalyses::none(); 4400b57cec5SDimitry Andric } 4410b57cec5SDimitry Andric 4420b57cec5SDimitry Andric char InstrProfilingLegacyPass::ID = 0; 4434824e7fdSDimitry Andric INITIALIZE_PASS_BEGIN(InstrProfilingLegacyPass, "instrprof", 4444824e7fdSDimitry Andric "Frontend instrumentation-based coverage lowering.", 4454824e7fdSDimitry Andric false, false) 4460b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 4474824e7fdSDimitry Andric INITIALIZE_PASS_END(InstrProfilingLegacyPass, "instrprof", 4484824e7fdSDimitry Andric "Frontend instrumentation-based coverage lowering.", false, 4494824e7fdSDimitry Andric false) 4500b57cec5SDimitry Andric 4510b57cec5SDimitry Andric ModulePass * 4520b57cec5SDimitry Andric llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options, 4530b57cec5SDimitry Andric bool IsCS) { 4540b57cec5SDimitry Andric return new InstrProfilingLegacyPass(Options, IsCS); 4550b57cec5SDimitry Andric } 4560b57cec5SDimitry Andric 4570b57cec5SDimitry Andric bool InstrProfiling::lowerIntrinsics(Function *F) { 4580b57cec5SDimitry Andric bool MadeChange = false; 4590b57cec5SDimitry Andric PromotionCandidates.clear(); 4600b57cec5SDimitry Andric for (BasicBlock &BB : *F) { 461349cc55cSDimitry Andric for (Instruction &Instr : llvm::make_early_inc_range(BB)) { 46204eeddc0SDimitry Andric if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) { 46304eeddc0SDimitry Andric lowerIncrement(IPIS); 4640b57cec5SDimitry Andric MadeChange = true; 46504eeddc0SDimitry Andric } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) { 46604eeddc0SDimitry Andric lowerIncrement(IPI); 46704eeddc0SDimitry Andric MadeChange = true; 4681fd87a68SDimitry Andric } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) { 4691fd87a68SDimitry Andric lowerCover(IPC); 4701fd87a68SDimitry Andric MadeChange = true; 47104eeddc0SDimitry Andric } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) { 47204eeddc0SDimitry Andric lowerValueProfileInst(IPVP); 4730b57cec5SDimitry Andric MadeChange = true; 4740b57cec5SDimitry Andric } 4750b57cec5SDimitry Andric } 4760b57cec5SDimitry Andric } 4770b57cec5SDimitry Andric 4780b57cec5SDimitry Andric if (!MadeChange) 4790b57cec5SDimitry Andric return false; 4800b57cec5SDimitry Andric 4810b57cec5SDimitry Andric promoteCounterLoadStores(F); 4820b57cec5SDimitry Andric return true; 4830b57cec5SDimitry Andric } 4840b57cec5SDimitry Andric 4855ffd83dbSDimitry Andric bool InstrProfiling::isRuntimeCounterRelocationEnabled() const { 486fe6060f1SDimitry Andric // Mach-O don't support weak external references. 487fe6060f1SDimitry Andric if (TT.isOSBinFormatMachO()) 488fe6060f1SDimitry Andric return false; 489fe6060f1SDimitry Andric 4905ffd83dbSDimitry Andric if (RuntimeCounterRelocation.getNumOccurrences() > 0) 4915ffd83dbSDimitry Andric return RuntimeCounterRelocation; 4925ffd83dbSDimitry Andric 493fe6060f1SDimitry Andric // Fuchsia uses runtime counter relocation by default. 4945ffd83dbSDimitry Andric return TT.isOSFuchsia(); 4955ffd83dbSDimitry Andric } 4965ffd83dbSDimitry Andric 4970b57cec5SDimitry Andric bool InstrProfiling::isCounterPromotionEnabled() const { 4980b57cec5SDimitry Andric if (DoCounterPromotion.getNumOccurrences() > 0) 4990b57cec5SDimitry Andric return DoCounterPromotion; 5000b57cec5SDimitry Andric 5010b57cec5SDimitry Andric return Options.DoCounterPromotion; 5020b57cec5SDimitry Andric } 5030b57cec5SDimitry Andric 5040b57cec5SDimitry Andric void InstrProfiling::promoteCounterLoadStores(Function *F) { 5050b57cec5SDimitry Andric if (!isCounterPromotionEnabled()) 5060b57cec5SDimitry Andric return; 5070b57cec5SDimitry Andric 5080b57cec5SDimitry Andric DominatorTree DT(*F); 5090b57cec5SDimitry Andric LoopInfo LI(DT); 5100b57cec5SDimitry Andric DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates; 5110b57cec5SDimitry Andric 5120b57cec5SDimitry Andric std::unique_ptr<BlockFrequencyInfo> BFI; 5130b57cec5SDimitry Andric if (Options.UseBFIInPromotion) { 5140b57cec5SDimitry Andric std::unique_ptr<BranchProbabilityInfo> BPI; 5158bcb0991SDimitry Andric BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F))); 5160b57cec5SDimitry Andric BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI)); 5170b57cec5SDimitry Andric } 5180b57cec5SDimitry Andric 5190b57cec5SDimitry Andric for (const auto &LoadStore : PromotionCandidates) { 5200b57cec5SDimitry Andric auto *CounterLoad = LoadStore.first; 5210b57cec5SDimitry Andric auto *CounterStore = LoadStore.second; 5220b57cec5SDimitry Andric BasicBlock *BB = CounterLoad->getParent(); 5230b57cec5SDimitry Andric Loop *ParentLoop = LI.getLoopFor(BB); 5240b57cec5SDimitry Andric if (!ParentLoop) 5250b57cec5SDimitry Andric continue; 5260b57cec5SDimitry Andric LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore); 5270b57cec5SDimitry Andric } 5280b57cec5SDimitry Andric 5290b57cec5SDimitry Andric SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder(); 5300b57cec5SDimitry Andric 5310b57cec5SDimitry Andric // Do a post-order traversal of the loops so that counter updates can be 5320b57cec5SDimitry Andric // iteratively hoisted outside the loop nest. 5330b57cec5SDimitry Andric for (auto *Loop : llvm::reverse(Loops)) { 5340b57cec5SDimitry Andric PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get()); 5350b57cec5SDimitry Andric Promoter.run(&TotalCountersPromoted); 5360b57cec5SDimitry Andric } 5370b57cec5SDimitry Andric } 5380b57cec5SDimitry Andric 539349cc55cSDimitry Andric static bool needsRuntimeHookUnconditionally(const Triple &TT) { 540349cc55cSDimitry Andric // On Fuchsia, we only need runtime hook if any counters are present. 541349cc55cSDimitry Andric if (TT.isOSFuchsia()) 542349cc55cSDimitry Andric return false; 543349cc55cSDimitry Andric 544349cc55cSDimitry Andric return true; 545349cc55cSDimitry Andric } 546349cc55cSDimitry Andric 5470b57cec5SDimitry Andric /// Check if the module contains uses of any profiling intrinsics. 5480b57cec5SDimitry Andric static bool containsProfilingIntrinsics(Module &M) { 54904eeddc0SDimitry Andric auto containsIntrinsic = [&](int ID) { 55004eeddc0SDimitry Andric if (auto *F = M.getFunction(Intrinsic::getName(ID))) 55104eeddc0SDimitry Andric return !F->use_empty(); 5520b57cec5SDimitry Andric return false; 55304eeddc0SDimitry Andric }; 5541fd87a68SDimitry Andric return containsIntrinsic(llvm::Intrinsic::instrprof_cover) || 5551fd87a68SDimitry Andric containsIntrinsic(llvm::Intrinsic::instrprof_increment) || 55604eeddc0SDimitry Andric containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) || 55704eeddc0SDimitry Andric containsIntrinsic(llvm::Intrinsic::instrprof_value_profile); 5580b57cec5SDimitry Andric } 5590b57cec5SDimitry Andric 5608bcb0991SDimitry Andric bool InstrProfiling::run( 5618bcb0991SDimitry Andric Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) { 5620b57cec5SDimitry Andric this->M = &M; 5638bcb0991SDimitry Andric this->GetTLI = std::move(GetTLI); 5640b57cec5SDimitry Andric NamesVar = nullptr; 5650b57cec5SDimitry Andric NamesSize = 0; 5660b57cec5SDimitry Andric ProfileDataMap.clear(); 567fe6060f1SDimitry Andric CompilerUsedVars.clear(); 5680b57cec5SDimitry Andric UsedVars.clear(); 5690b57cec5SDimitry Andric TT = Triple(M.getTargetTriple()); 5700b57cec5SDimitry Andric 571349cc55cSDimitry Andric bool MadeChange = false; 572349cc55cSDimitry Andric 5730b57cec5SDimitry Andric // Emit the runtime hook even if no counters are present. 574349cc55cSDimitry Andric if (needsRuntimeHookUnconditionally(TT)) 575349cc55cSDimitry Andric MadeChange = emitRuntimeHook(); 5760b57cec5SDimitry Andric 5770b57cec5SDimitry Andric // Improve compile time by avoiding linear scans when there is no work. 5780b57cec5SDimitry Andric GlobalVariable *CoverageNamesVar = 5790b57cec5SDimitry Andric M.getNamedGlobal(getCoverageUnusedNamesVarName()); 5800b57cec5SDimitry Andric if (!containsProfilingIntrinsics(M) && !CoverageNamesVar) 5810b57cec5SDimitry Andric return MadeChange; 5820b57cec5SDimitry Andric 5830b57cec5SDimitry Andric // We did not know how many value sites there would be inside 5840b57cec5SDimitry Andric // the instrumented function. This is counting the number of instrumented 5850b57cec5SDimitry Andric // target value sites to enter it as field in the profile data variable. 5860b57cec5SDimitry Andric for (Function &F : M) { 5870b57cec5SDimitry Andric InstrProfIncrementInst *FirstProfIncInst = nullptr; 5880b57cec5SDimitry Andric for (BasicBlock &BB : F) 5890b57cec5SDimitry Andric for (auto I = BB.begin(), E = BB.end(); I != E; I++) 5900b57cec5SDimitry Andric if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I)) 5910b57cec5SDimitry Andric computeNumValueSiteCounts(Ind); 5920b57cec5SDimitry Andric else if (FirstProfIncInst == nullptr) 5930b57cec5SDimitry Andric FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I); 5940b57cec5SDimitry Andric 5950b57cec5SDimitry Andric // Value profiling intrinsic lowering requires per-function profile data 5960b57cec5SDimitry Andric // variable to be created first. 5970b57cec5SDimitry Andric if (FirstProfIncInst != nullptr) 5980b57cec5SDimitry Andric static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst)); 5990b57cec5SDimitry Andric } 6000b57cec5SDimitry Andric 6010b57cec5SDimitry Andric for (Function &F : M) 6020b57cec5SDimitry Andric MadeChange |= lowerIntrinsics(&F); 6030b57cec5SDimitry Andric 6040b57cec5SDimitry Andric if (CoverageNamesVar) { 6050b57cec5SDimitry Andric lowerCoverageData(CoverageNamesVar); 6060b57cec5SDimitry Andric MadeChange = true; 6070b57cec5SDimitry Andric } 6080b57cec5SDimitry Andric 6090b57cec5SDimitry Andric if (!MadeChange) 6100b57cec5SDimitry Andric return false; 6110b57cec5SDimitry Andric 6120b57cec5SDimitry Andric emitVNodes(); 6130b57cec5SDimitry Andric emitNameData(); 614349cc55cSDimitry Andric emitRuntimeHook(); 6150b57cec5SDimitry Andric emitRegistration(); 6160b57cec5SDimitry Andric emitUses(); 6170b57cec5SDimitry Andric emitInitialization(); 6180b57cec5SDimitry Andric return true; 6190b57cec5SDimitry Andric } 6200b57cec5SDimitry Andric 621e8d8bef9SDimitry Andric static FunctionCallee getOrInsertValueProfilingCall( 622e8d8bef9SDimitry Andric Module &M, const TargetLibraryInfo &TLI, 623e8d8bef9SDimitry Andric ValueProfilingCallType CallType = ValueProfilingCallType::Default) { 6240b57cec5SDimitry Andric LLVMContext &Ctx = M.getContext(); 6250b57cec5SDimitry Andric auto *ReturnTy = Type::getVoidTy(M.getContext()); 6260b57cec5SDimitry Andric 6270b57cec5SDimitry Andric AttributeList AL; 6280b57cec5SDimitry Andric if (auto AK = TLI.getExtAttrForI32Param(false)) 6290b57cec5SDimitry Andric AL = AL.addParamAttribute(M.getContext(), 2, AK); 6300b57cec5SDimitry Andric 631e8d8bef9SDimitry Andric assert((CallType == ValueProfilingCallType::Default || 632e8d8bef9SDimitry Andric CallType == ValueProfilingCallType::MemOp) && 633e8d8bef9SDimitry Andric "Must be Default or MemOp"); 6340b57cec5SDimitry Andric Type *ParamTypes[] = { 6350b57cec5SDimitry Andric #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType 6360b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfData.inc" 6370b57cec5SDimitry Andric }; 6380b57cec5SDimitry Andric auto *ValueProfilingCallTy = 6390b57cec5SDimitry Andric FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false); 640e8d8bef9SDimitry Andric StringRef FuncName = CallType == ValueProfilingCallType::Default 641e8d8bef9SDimitry Andric ? getInstrProfValueProfFuncName() 642e8d8bef9SDimitry Andric : getInstrProfValueProfMemOpFuncName(); 643e8d8bef9SDimitry Andric return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL); 6440b57cec5SDimitry Andric } 6450b57cec5SDimitry Andric 6460b57cec5SDimitry Andric void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) { 6470b57cec5SDimitry Andric GlobalVariable *Name = Ind->getName(); 6480b57cec5SDimitry Andric uint64_t ValueKind = Ind->getValueKind()->getZExtValue(); 6490b57cec5SDimitry Andric uint64_t Index = Ind->getIndex()->getZExtValue(); 6504824e7fdSDimitry Andric auto &PD = ProfileDataMap[Name]; 6514824e7fdSDimitry Andric PD.NumValueSites[ValueKind] = 6524824e7fdSDimitry Andric std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1)); 6530b57cec5SDimitry Andric } 6540b57cec5SDimitry Andric 6550b57cec5SDimitry Andric void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { 6560eae32dcSDimitry Andric // TODO: Value profiling heavily depends on the data section which is omitted 6570eae32dcSDimitry Andric // in lightweight mode. We need to move the value profile pointer to the 6580eae32dcSDimitry Andric // Counter struct to get this working. 6590eae32dcSDimitry Andric assert( 6600eae32dcSDimitry Andric !DebugInfoCorrelate && 6610eae32dcSDimitry Andric "Value profiling is not yet supported with lightweight instrumentation"); 6620b57cec5SDimitry Andric GlobalVariable *Name = Ind->getName(); 6630b57cec5SDimitry Andric auto It = ProfileDataMap.find(Name); 6640b57cec5SDimitry Andric assert(It != ProfileDataMap.end() && It->second.DataVar && 6650b57cec5SDimitry Andric "value profiling detected in function with no counter incerement"); 6660b57cec5SDimitry Andric 6670b57cec5SDimitry Andric GlobalVariable *DataVar = It->second.DataVar; 6680b57cec5SDimitry Andric uint64_t ValueKind = Ind->getValueKind()->getZExtValue(); 6690b57cec5SDimitry Andric uint64_t Index = Ind->getIndex()->getZExtValue(); 6700b57cec5SDimitry Andric for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind) 6710b57cec5SDimitry Andric Index += It->second.NumValueSites[Kind]; 6720b57cec5SDimitry Andric 6730b57cec5SDimitry Andric IRBuilder<> Builder(Ind); 674e8d8bef9SDimitry Andric bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() == 6750b57cec5SDimitry Andric llvm::InstrProfValueKind::IPVK_MemOPSize); 6760b57cec5SDimitry Andric CallInst *Call = nullptr; 6778bcb0991SDimitry Andric auto *TLI = &GetTLI(*Ind->getFunction()); 6785ffd83dbSDimitry Andric 6795ffd83dbSDimitry Andric // To support value profiling calls within Windows exception handlers, funclet 6805ffd83dbSDimitry Andric // information contained within operand bundles needs to be copied over to 6815ffd83dbSDimitry Andric // the library call. This is required for the IR to be processed by the 6825ffd83dbSDimitry Andric // WinEHPrepare pass. 6835ffd83dbSDimitry Andric SmallVector<OperandBundleDef, 1> OpBundles; 6845ffd83dbSDimitry Andric Ind->getOperandBundlesAsDefs(OpBundles); 685e8d8bef9SDimitry Andric if (!IsMemOpSize) { 6860b57cec5SDimitry Andric Value *Args[3] = {Ind->getTargetValue(), 6870b57cec5SDimitry Andric Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()), 6880b57cec5SDimitry Andric Builder.getInt32(Index)}; 6895ffd83dbSDimitry Andric Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args, 6905ffd83dbSDimitry Andric OpBundles); 6910b57cec5SDimitry Andric } else { 692e8d8bef9SDimitry Andric Value *Args[3] = {Ind->getTargetValue(), 6930b57cec5SDimitry Andric Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()), 694e8d8bef9SDimitry Andric Builder.getInt32(Index)}; 695e8d8bef9SDimitry Andric Call = Builder.CreateCall( 696e8d8bef9SDimitry Andric getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp), 6975ffd83dbSDimitry Andric Args, OpBundles); 6980b57cec5SDimitry Andric } 6990b57cec5SDimitry Andric if (auto AK = TLI->getExtAttrForI32Param(false)) 7000b57cec5SDimitry Andric Call->addParamAttr(2, AK); 7010b57cec5SDimitry Andric Ind->replaceAllUsesWith(Call); 7020b57cec5SDimitry Andric Ind->eraseFromParent(); 7030b57cec5SDimitry Andric } 7040b57cec5SDimitry Andric 7051fd87a68SDimitry Andric Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) { 7061fd87a68SDimitry Andric auto *Counters = getOrCreateRegionCounters(I); 7071fd87a68SDimitry Andric IRBuilder<> Builder(I); 7080b57cec5SDimitry Andric 7091fd87a68SDimitry Andric auto *Addr = Builder.CreateConstInBoundsGEP2_32( 7101fd87a68SDimitry Andric Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue()); 7110b57cec5SDimitry Andric 7121fd87a68SDimitry Andric if (!isRuntimeCounterRelocationEnabled()) 7131fd87a68SDimitry Andric return Addr; 7141fd87a68SDimitry Andric 7155ffd83dbSDimitry Andric Type *Int64Ty = Type::getInt64Ty(M->getContext()); 7161fd87a68SDimitry Andric Function *Fn = I->getParent()->getParent(); 71781ad6265SDimitry Andric LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn]; 71881ad6265SDimitry Andric if (!BiasLI) { 71981ad6265SDimitry Andric IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front()); 7201fd87a68SDimitry Andric auto *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName()); 7215ffd83dbSDimitry Andric if (!Bias) { 722fe6060f1SDimitry Andric // Compiler must define this variable when runtime counter relocation 723fe6060f1SDimitry Andric // is being used. Runtime has a weak external reference that is used 724fe6060f1SDimitry Andric // to check whether that's the case or not. 7254824e7fdSDimitry Andric Bias = new GlobalVariable( 7264824e7fdSDimitry Andric *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage, 7274824e7fdSDimitry Andric Constant::getNullValue(Int64Ty), getInstrProfCounterBiasVarName()); 7285ffd83dbSDimitry Andric Bias->setVisibility(GlobalVariable::HiddenVisibility); 729fe6060f1SDimitry Andric // A definition that's weak (linkonce_odr) without being in a COMDAT 730fe6060f1SDimitry Andric // section wouldn't lead to link errors, but it would lead to a dead 731fe6060f1SDimitry Andric // data word from every TU but one. Putting it in COMDAT ensures there 732fe6060f1SDimitry Andric // will be exactly one data slot in the link. 733fe6060f1SDimitry Andric if (TT.supportsCOMDAT()) 734fe6060f1SDimitry Andric Bias->setComdat(M->getOrInsertComdat(Bias->getName())); 7355ffd83dbSDimitry Andric } 73681ad6265SDimitry Andric BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias); 7375ffd83dbSDimitry Andric } 73881ad6265SDimitry Andric auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI); 7391fd87a68SDimitry Andric return Builder.CreateIntToPtr(Add, Addr->getType()); 7405ffd83dbSDimitry Andric } 7415ffd83dbSDimitry Andric 7421fd87a68SDimitry Andric void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) { 7431fd87a68SDimitry Andric auto *Addr = getCounterAddress(CoverInstruction); 7441fd87a68SDimitry Andric IRBuilder<> Builder(CoverInstruction); 7451fd87a68SDimitry Andric // We store zero to represent that this block is covered. 7461fd87a68SDimitry Andric Builder.CreateStore(Builder.getInt8(0), Addr); 7471fd87a68SDimitry Andric CoverInstruction->eraseFromParent(); 7481fd87a68SDimitry Andric } 7491fd87a68SDimitry Andric 7501fd87a68SDimitry Andric void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) { 7511fd87a68SDimitry Andric auto *Addr = getCounterAddress(Inc); 7521fd87a68SDimitry Andric 7531fd87a68SDimitry Andric IRBuilder<> Builder(Inc); 7545ffd83dbSDimitry Andric if (Options.Atomic || AtomicCounterUpdateAll || 7551fd87a68SDimitry Andric (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) { 7560b57cec5SDimitry Andric Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(), 757fe6060f1SDimitry Andric MaybeAlign(), AtomicOrdering::Monotonic); 7580b57cec5SDimitry Andric } else { 7590b57cec5SDimitry Andric Value *IncStep = Inc->getStep(); 7600b57cec5SDimitry Andric Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount"); 7610b57cec5SDimitry Andric auto *Count = Builder.CreateAdd(Load, Inc->getStep()); 7620b57cec5SDimitry Andric auto *Store = Builder.CreateStore(Count, Addr); 7630b57cec5SDimitry Andric if (isCounterPromotionEnabled()) 7640b57cec5SDimitry Andric PromotionCandidates.emplace_back(cast<Instruction>(Load), Store); 7650b57cec5SDimitry Andric } 7660b57cec5SDimitry Andric Inc->eraseFromParent(); 7670b57cec5SDimitry Andric } 7680b57cec5SDimitry Andric 7690b57cec5SDimitry Andric void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) { 7700b57cec5SDimitry Andric ConstantArray *Names = 7710b57cec5SDimitry Andric cast<ConstantArray>(CoverageNamesVar->getInitializer()); 7720b57cec5SDimitry Andric for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) { 7730b57cec5SDimitry Andric Constant *NC = Names->getOperand(I); 7740b57cec5SDimitry Andric Value *V = NC->stripPointerCasts(); 7750b57cec5SDimitry Andric assert(isa<GlobalVariable>(V) && "Missing reference to function name"); 7760b57cec5SDimitry Andric GlobalVariable *Name = cast<GlobalVariable>(V); 7770b57cec5SDimitry Andric 7780b57cec5SDimitry Andric Name->setLinkage(GlobalValue::PrivateLinkage); 7790b57cec5SDimitry Andric ReferencedNames.push_back(Name); 78081ad6265SDimitry Andric if (isa<ConstantExpr>(NC)) 7810b57cec5SDimitry Andric NC->dropAllReferences(); 7820b57cec5SDimitry Andric } 7830b57cec5SDimitry Andric CoverageNamesVar->eraseFromParent(); 7840b57cec5SDimitry Andric } 7850b57cec5SDimitry Andric 7860b57cec5SDimitry Andric /// Get the name of a profiling variable for a particular function. 78704eeddc0SDimitry Andric static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, 788349cc55cSDimitry Andric bool &Renamed) { 7890b57cec5SDimitry Andric StringRef NamePrefix = getInstrProfNameVarPrefix(); 7900b57cec5SDimitry Andric StringRef Name = Inc->getName()->getName().substr(NamePrefix.size()); 7910b57cec5SDimitry Andric Function *F = Inc->getParent()->getParent(); 7920b57cec5SDimitry Andric Module *M = F->getParent(); 7930b57cec5SDimitry Andric if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) || 794349cc55cSDimitry Andric !canRenameComdatFunc(*F)) { 795349cc55cSDimitry Andric Renamed = false; 7960b57cec5SDimitry Andric return (Prefix + Name).str(); 797349cc55cSDimitry Andric } 798349cc55cSDimitry Andric Renamed = true; 7990b57cec5SDimitry Andric uint64_t FuncHash = Inc->getHash()->getZExtValue(); 8000b57cec5SDimitry Andric SmallVector<char, 24> HashPostfix; 8010b57cec5SDimitry Andric if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix))) 8020b57cec5SDimitry Andric return (Prefix + Name).str(); 8030b57cec5SDimitry Andric return (Prefix + Name + "." + Twine(FuncHash)).str(); 8040b57cec5SDimitry Andric } 8050b57cec5SDimitry Andric 806fe6060f1SDimitry Andric static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) { 807fe6060f1SDimitry Andric auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag)); 808fe6060f1SDimitry Andric if (!MD) 809fe6060f1SDimitry Andric return 0; 810fe6060f1SDimitry Andric 811fe6060f1SDimitry Andric // If the flag is a ConstantAsMetadata, it should be an integer representable 812fe6060f1SDimitry Andric // in 64-bits. 813fe6060f1SDimitry Andric return cast<ConstantInt>(MD->getValue())->getZExtValue(); 814fe6060f1SDimitry Andric } 815fe6060f1SDimitry Andric 816fe6060f1SDimitry Andric static bool enablesValueProfiling(const Module &M) { 817fe6060f1SDimitry Andric return isIRPGOFlagSet(&M) || 818fe6060f1SDimitry Andric getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0; 819fe6060f1SDimitry Andric } 820fe6060f1SDimitry Andric 821fe6060f1SDimitry Andric // Conservatively returns true if data variables may be referenced by code. 822fe6060f1SDimitry Andric static bool profDataReferencedByCode(const Module &M) { 823fe6060f1SDimitry Andric return enablesValueProfiling(M); 824fe6060f1SDimitry Andric } 825fe6060f1SDimitry Andric 8260b57cec5SDimitry Andric static inline bool shouldRecordFunctionAddr(Function *F) { 827fe6060f1SDimitry Andric // Only record function addresses if IR PGO is enabled or if clang value 828fe6060f1SDimitry Andric // profiling is enabled. Recording function addresses greatly increases object 829fe6060f1SDimitry Andric // file size, because it prevents the inliner from deleting functions that 830fe6060f1SDimitry Andric // have been inlined everywhere. 831fe6060f1SDimitry Andric if (!profDataReferencedByCode(*F->getParent())) 832fe6060f1SDimitry Andric return false; 833fe6060f1SDimitry Andric 8340b57cec5SDimitry Andric // Check the linkage 8350b57cec5SDimitry Andric bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage(); 8360b57cec5SDimitry Andric if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() && 8370b57cec5SDimitry Andric !HasAvailableExternallyLinkage) 8380b57cec5SDimitry Andric return true; 8390b57cec5SDimitry Andric 8400b57cec5SDimitry Andric // A function marked 'alwaysinline' with available_externally linkage can't 8410b57cec5SDimitry Andric // have its address taken. Doing so would create an undefined external ref to 8420b57cec5SDimitry Andric // the function, which would fail to link. 8430b57cec5SDimitry Andric if (HasAvailableExternallyLinkage && 8440b57cec5SDimitry Andric F->hasFnAttribute(Attribute::AlwaysInline)) 8450b57cec5SDimitry Andric return false; 8460b57cec5SDimitry Andric 8470b57cec5SDimitry Andric // Prohibit function address recording if the function is both internal and 8480b57cec5SDimitry Andric // COMDAT. This avoids the profile data variable referencing internal symbols 8490b57cec5SDimitry Andric // in COMDAT. 8500b57cec5SDimitry Andric if (F->hasLocalLinkage() && F->hasComdat()) 8510b57cec5SDimitry Andric return false; 8520b57cec5SDimitry Andric 8530b57cec5SDimitry Andric // Check uses of this function for other than direct calls or invokes to it. 8540b57cec5SDimitry Andric // Inline virtual functions have linkeOnceODR linkage. When a key method 8550b57cec5SDimitry Andric // exists, the vtable will only be emitted in the TU where the key method 8560b57cec5SDimitry Andric // is defined. In a TU where vtable is not available, the function won't 8570b57cec5SDimitry Andric // be 'addresstaken'. If its address is not recorded here, the profile data 8580b57cec5SDimitry Andric // with missing address may be picked by the linker leading to missing 8590b57cec5SDimitry Andric // indirect call target info. 8600b57cec5SDimitry Andric return F->hasAddressTaken() || F->hasLinkOnceLinkage(); 8610b57cec5SDimitry Andric } 8620b57cec5SDimitry Andric 8630b57cec5SDimitry Andric static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { 8640b57cec5SDimitry Andric // Don't do this for Darwin. compiler-rt uses linker magic. 8650b57cec5SDimitry Andric if (TT.isOSDarwin()) 8660b57cec5SDimitry Andric return false; 8670b57cec5SDimitry Andric // Use linker script magic to get data/cnts/name start/end. 86881ad6265SDimitry Andric if (TT.isOSAIX() || TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() || 86981ad6265SDimitry Andric TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS() || TT.isOSWindows()) 8700b57cec5SDimitry Andric return false; 8710b57cec5SDimitry Andric 8720b57cec5SDimitry Andric return true; 8730b57cec5SDimitry Andric } 8740b57cec5SDimitry Andric 8750b57cec5SDimitry Andric GlobalVariable * 8761fd87a68SDimitry Andric InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name, 8771fd87a68SDimitry Andric GlobalValue::LinkageTypes Linkage) { 8781fd87a68SDimitry Andric uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); 8791fd87a68SDimitry Andric auto &Ctx = M->getContext(); 8801fd87a68SDimitry Andric GlobalVariable *GV; 8811fd87a68SDimitry Andric if (isa<InstrProfCoverInst>(Inc)) { 8821fd87a68SDimitry Andric auto *CounterTy = Type::getInt8Ty(Ctx); 8831fd87a68SDimitry Andric auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters); 8841fd87a68SDimitry Andric // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. 8851fd87a68SDimitry Andric std::vector<Constant *> InitialValues(NumCounters, 8861fd87a68SDimitry Andric Constant::getAllOnesValue(CounterTy)); 8871fd87a68SDimitry Andric GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, 8881fd87a68SDimitry Andric ConstantArray::get(CounterArrTy, InitialValues), 8891fd87a68SDimitry Andric Name); 8901fd87a68SDimitry Andric GV->setAlignment(Align(1)); 8911fd87a68SDimitry Andric } else { 8921fd87a68SDimitry Andric auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); 8931fd87a68SDimitry Andric GV = new GlobalVariable(*M, CounterTy, false, Linkage, 8941fd87a68SDimitry Andric Constant::getNullValue(CounterTy), Name); 8951fd87a68SDimitry Andric GV->setAlignment(Align(8)); 8961fd87a68SDimitry Andric } 8971fd87a68SDimitry Andric return GV; 8981fd87a68SDimitry Andric } 8991fd87a68SDimitry Andric 9001fd87a68SDimitry Andric GlobalVariable * 90104eeddc0SDimitry Andric InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { 9020b57cec5SDimitry Andric GlobalVariable *NamePtr = Inc->getName(); 9034824e7fdSDimitry Andric auto &PD = ProfileDataMap[NamePtr]; 9044824e7fdSDimitry Andric if (PD.RegionCounters) 9054824e7fdSDimitry Andric return PD.RegionCounters; 9060b57cec5SDimitry Andric 907fe6060f1SDimitry Andric // Match the linkage and visibility of the name global. 9080b57cec5SDimitry Andric Function *Fn = Inc->getParent()->getParent(); 9090b57cec5SDimitry Andric GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); 9100b57cec5SDimitry Andric GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility(); 9110b57cec5SDimitry Andric 9120eae32dcSDimitry Andric // Use internal rather than private linkage so the counter variable shows up 9130eae32dcSDimitry Andric // in the symbol table when using debug info for correlation. 9140eae32dcSDimitry Andric if (DebugInfoCorrelate && TT.isOSBinFormatMachO() && 9150eae32dcSDimitry Andric Linkage == GlobalValue::PrivateLinkage) 9160eae32dcSDimitry Andric Linkage = GlobalValue::InternalLinkage; 9170eae32dcSDimitry Andric 918349cc55cSDimitry Andric // Due to the limitation of binder as of 2021/09/28, the duplicate weak 919349cc55cSDimitry Andric // symbols in the same csect won't be discarded. When there are duplicate weak 920349cc55cSDimitry Andric // symbols, we can NOT guarantee that the relocations get resolved to the 921349cc55cSDimitry Andric // intended weak symbol, so we can not ensure the correctness of the relative 922349cc55cSDimitry Andric // CounterPtr, so we have to use private linkage for counter and data symbols. 923349cc55cSDimitry Andric if (TT.isOSBinFormatXCOFF()) { 924349cc55cSDimitry Andric Linkage = GlobalValue::PrivateLinkage; 925349cc55cSDimitry Andric Visibility = GlobalValue::DefaultVisibility; 926349cc55cSDimitry Andric } 9270b57cec5SDimitry Andric // Move the name variable to the right section. Place them in a COMDAT group 9280b57cec5SDimitry Andric // if the associated function is a COMDAT. This will make sure that only one 9290b57cec5SDimitry Andric // copy of counters of the COMDAT function will be emitted after linking. Keep 9300b57cec5SDimitry Andric // in mind that this pass may run before the inliner, so we need to create a 9310b57cec5SDimitry Andric // new comdat group for the counters and profiling data. If we use the comdat 9320b57cec5SDimitry Andric // of the parent function, that will result in relocations against discarded 9330b57cec5SDimitry Andric // sections. 934fe6060f1SDimitry Andric // 935fe6060f1SDimitry Andric // If the data variable is referenced by code, counters and data have to be 936fe6060f1SDimitry Andric // in different comdats for COFF because the Visual C++ linker will report 937fe6060f1SDimitry Andric // duplicate symbol errors if there are multiple external symbols with the 938fe6060f1SDimitry Andric // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE. 939fe6060f1SDimitry Andric // 940fe6060f1SDimitry Andric // For ELF, when not using COMDAT, put counters, data and values into a 941fe6060f1SDimitry Andric // nodeduplicate COMDAT which is lowered to a zero-flag section group. This 942fe6060f1SDimitry Andric // allows -z start-stop-gc to discard the entire group when the function is 943fe6060f1SDimitry Andric // discarded. 944fe6060f1SDimitry Andric bool DataReferencedByCode = profDataReferencedByCode(*M); 945c14a5a88SDimitry Andric bool NeedComdat = needsComdatForCounter(*Fn, *M); 946349cc55cSDimitry Andric bool Renamed; 947349cc55cSDimitry Andric std::string CntsVarName = 948349cc55cSDimitry Andric getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed); 949349cc55cSDimitry Andric std::string DataVarName = 950349cc55cSDimitry Andric getVarName(Inc, getInstrProfDataVarPrefix(), Renamed); 951fe6060f1SDimitry Andric auto MaybeSetComdat = [&](GlobalVariable *GV) { 952fe6060f1SDimitry Andric bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); 953fe6060f1SDimitry Andric if (UseComdat) { 954fe6060f1SDimitry Andric StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode 955fe6060f1SDimitry Andric ? GV->getName() 956fe6060f1SDimitry Andric : CntsVarName; 957fe6060f1SDimitry Andric Comdat *C = M->getOrInsertComdat(GroupName); 958fe6060f1SDimitry Andric if (!NeedComdat) 959fe6060f1SDimitry Andric C->setSelectionKind(Comdat::NoDeduplicate); 960fe6060f1SDimitry Andric GV->setComdat(C); 961fe6060f1SDimitry Andric } 962c14a5a88SDimitry Andric }; 9630b57cec5SDimitry Andric 9640b57cec5SDimitry Andric uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); 9650b57cec5SDimitry Andric LLVMContext &Ctx = M->getContext(); 9660b57cec5SDimitry Andric 9671fd87a68SDimitry Andric auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage); 9680b57cec5SDimitry Andric CounterPtr->setVisibility(Visibility); 9690b57cec5SDimitry Andric CounterPtr->setSection( 9700b57cec5SDimitry Andric getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat())); 971c14a5a88SDimitry Andric MaybeSetComdat(CounterPtr); 972c14a5a88SDimitry Andric CounterPtr->setLinkage(Linkage); 9734824e7fdSDimitry Andric PD.RegionCounters = CounterPtr; 9740eae32dcSDimitry Andric if (DebugInfoCorrelate) { 9750eae32dcSDimitry Andric if (auto *SP = Fn->getSubprogram()) { 9760eae32dcSDimitry Andric DIBuilder DB(*M, true, SP->getUnit()); 9770eae32dcSDimitry Andric Metadata *FunctionNameAnnotation[] = { 9780eae32dcSDimitry Andric MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName), 9790eae32dcSDimitry Andric MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)), 9800eae32dcSDimitry Andric }; 9810eae32dcSDimitry Andric Metadata *CFGHashAnnotation[] = { 9820eae32dcSDimitry Andric MDString::get(Ctx, InstrProfCorrelator::CFGHashAttributeName), 9830eae32dcSDimitry Andric ConstantAsMetadata::get(Inc->getHash()), 9840eae32dcSDimitry Andric }; 9850eae32dcSDimitry Andric Metadata *NumCountersAnnotation[] = { 9860eae32dcSDimitry Andric MDString::get(Ctx, InstrProfCorrelator::NumCountersAttributeName), 9870eae32dcSDimitry Andric ConstantAsMetadata::get(Inc->getNumCounters()), 9880eae32dcSDimitry Andric }; 9890eae32dcSDimitry Andric auto Annotations = DB.getOrCreateArray({ 9900eae32dcSDimitry Andric MDNode::get(Ctx, FunctionNameAnnotation), 9910eae32dcSDimitry Andric MDNode::get(Ctx, CFGHashAnnotation), 9920eae32dcSDimitry Andric MDNode::get(Ctx, NumCountersAnnotation), 9930eae32dcSDimitry Andric }); 9940eae32dcSDimitry Andric auto *DICounter = DB.createGlobalVariableExpression( 9950eae32dcSDimitry Andric SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(), 9960eae32dcSDimitry Andric /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"), 9970eae32dcSDimitry Andric CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr, 9980eae32dcSDimitry Andric /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0, 9990eae32dcSDimitry Andric Annotations); 10000eae32dcSDimitry Andric CounterPtr->addDebugInfo(DICounter); 10010eae32dcSDimitry Andric DB.finalize(); 10020eae32dcSDimitry Andric } else { 10030eae32dcSDimitry Andric std::string Msg = ("Missing debug info for function " + Fn->getName() + 10040eae32dcSDimitry Andric "; required for profile correlation.") 10050eae32dcSDimitry Andric .str(); 10060eae32dcSDimitry Andric Ctx.diagnose( 10070eae32dcSDimitry Andric DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); 10080eae32dcSDimitry Andric } 10090eae32dcSDimitry Andric } 10100b57cec5SDimitry Andric 10110b57cec5SDimitry Andric auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); 10120b57cec5SDimitry Andric // Allocate statically the array of pointers to value profile nodes for 10130b57cec5SDimitry Andric // the current function. 10140b57cec5SDimitry Andric Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy); 10150b57cec5SDimitry Andric uint64_t NS = 0; 10160b57cec5SDimitry Andric for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 10170b57cec5SDimitry Andric NS += PD.NumValueSites[Kind]; 1018fe6060f1SDimitry Andric if (NS > 0 && ValueProfileStaticAlloc && 1019fe6060f1SDimitry Andric !needsRuntimeRegistrationOfSectionRange(TT)) { 10200b57cec5SDimitry Andric ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS); 1021fe6060f1SDimitry Andric auto *ValuesVar = new GlobalVariable( 1022fe6060f1SDimitry Andric *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy), 1023349cc55cSDimitry Andric getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed)); 10240b57cec5SDimitry Andric ValuesVar->setVisibility(Visibility); 10250b57cec5SDimitry Andric ValuesVar->setSection( 10260b57cec5SDimitry Andric getInstrProfSectionName(IPSK_vals, TT.getObjectFormat())); 10278bcb0991SDimitry Andric ValuesVar->setAlignment(Align(8)); 1028c14a5a88SDimitry Andric MaybeSetComdat(ValuesVar); 10290b57cec5SDimitry Andric ValuesPtrExpr = 10300b57cec5SDimitry Andric ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx)); 10310b57cec5SDimitry Andric } 10320b57cec5SDimitry Andric 103304eeddc0SDimitry Andric if (DebugInfoCorrelate) { 103404eeddc0SDimitry Andric // Mark the counter variable as used so that it isn't optimized out. 103504eeddc0SDimitry Andric CompilerUsedVars.push_back(PD.RegionCounters); 10360eae32dcSDimitry Andric return PD.RegionCounters; 103704eeddc0SDimitry Andric } 10380eae32dcSDimitry Andric 10390b57cec5SDimitry Andric // Create data variable. 1040349cc55cSDimitry Andric auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext()); 10410b57cec5SDimitry Andric auto *Int16Ty = Type::getInt16Ty(Ctx); 10420b57cec5SDimitry Andric auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1); 10430b57cec5SDimitry Andric Type *DataTypes[] = { 10440b57cec5SDimitry Andric #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType, 10450b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfData.inc" 10460b57cec5SDimitry Andric }; 10470b57cec5SDimitry Andric auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes)); 10480b57cec5SDimitry Andric 10490b57cec5SDimitry Andric Constant *FunctionAddr = shouldRecordFunctionAddr(Fn) 10500b57cec5SDimitry Andric ? ConstantExpr::getBitCast(Fn, Int8PtrTy) 10510b57cec5SDimitry Andric : ConstantPointerNull::get(Int8PtrTy); 10520b57cec5SDimitry Andric 10530b57cec5SDimitry Andric Constant *Int16ArrayVals[IPVK_Last + 1]; 10540b57cec5SDimitry Andric for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 10550b57cec5SDimitry Andric Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]); 10560b57cec5SDimitry Andric 1057fe6060f1SDimitry Andric // If the data variable is not referenced by code (if we don't emit 1058fe6060f1SDimitry Andric // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the 1059fe6060f1SDimitry Andric // data variable live under linker GC, the data variable can be private. This 1060fe6060f1SDimitry Andric // optimization applies to ELF. 1061fe6060f1SDimitry Andric // 1062fe6060f1SDimitry Andric // On COFF, a comdat leader cannot be local so we require DataReferencedByCode 1063fe6060f1SDimitry Andric // to be false. 1064349cc55cSDimitry Andric // 1065349cc55cSDimitry Andric // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees 1066349cc55cSDimitry Andric // that other copies must have the same CFG and cannot have value profiling. 1067349cc55cSDimitry Andric // If no hash suffix, other profd copies may be referenced by code. 1068349cc55cSDimitry Andric if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) && 1069349cc55cSDimitry Andric (TT.isOSBinFormatELF() || 1070fe6060f1SDimitry Andric (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) { 1071fe6060f1SDimitry Andric Linkage = GlobalValue::PrivateLinkage; 1072fe6060f1SDimitry Andric Visibility = GlobalValue::DefaultVisibility; 1073fe6060f1SDimitry Andric } 1074e8d8bef9SDimitry Andric auto *Data = 1075349cc55cSDimitry Andric new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName); 1076349cc55cSDimitry Andric // Reference the counter variable with a label difference (link-time 1077349cc55cSDimitry Andric // constant). 1078349cc55cSDimitry Andric auto *RelativeCounterPtr = 1079349cc55cSDimitry Andric ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), 1080349cc55cSDimitry Andric ConstantExpr::getPtrToInt(Data, IntPtrTy)); 1081349cc55cSDimitry Andric 1082349cc55cSDimitry Andric Constant *DataVals[] = { 1083349cc55cSDimitry Andric #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init, 1084349cc55cSDimitry Andric #include "llvm/ProfileData/InstrProfData.inc" 1085349cc55cSDimitry Andric }; 1086349cc55cSDimitry Andric Data->setInitializer(ConstantStruct::get(DataTy, DataVals)); 1087349cc55cSDimitry Andric 10880b57cec5SDimitry Andric Data->setVisibility(Visibility); 10890b57cec5SDimitry Andric Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat())); 10908bcb0991SDimitry Andric Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT)); 1091c14a5a88SDimitry Andric MaybeSetComdat(Data); 1092c14a5a88SDimitry Andric Data->setLinkage(Linkage); 10930b57cec5SDimitry Andric 10940b57cec5SDimitry Andric PD.DataVar = Data; 10950b57cec5SDimitry Andric 10960b57cec5SDimitry Andric // Mark the data variable as used so that it isn't stripped out. 1097fe6060f1SDimitry Andric CompilerUsedVars.push_back(Data); 10980b57cec5SDimitry Andric // Now that the linkage set by the FE has been passed to the data and counter 10990b57cec5SDimitry Andric // variables, reset Name variable's linkage and visibility to private so that 11000b57cec5SDimitry Andric // it can be removed later by the compiler. 11010b57cec5SDimitry Andric NamePtr->setLinkage(GlobalValue::PrivateLinkage); 11020b57cec5SDimitry Andric // Collect the referenced names to be used by emitNameData. 11030b57cec5SDimitry Andric ReferencedNames.push_back(NamePtr); 11040b57cec5SDimitry Andric 11054824e7fdSDimitry Andric return PD.RegionCounters; 11060b57cec5SDimitry Andric } 11070b57cec5SDimitry Andric 11080b57cec5SDimitry Andric void InstrProfiling::emitVNodes() { 11090b57cec5SDimitry Andric if (!ValueProfileStaticAlloc) 11100b57cec5SDimitry Andric return; 11110b57cec5SDimitry Andric 11120b57cec5SDimitry Andric // For now only support this on platforms that do 11130b57cec5SDimitry Andric // not require runtime registration to discover 11140b57cec5SDimitry Andric // named section start/end. 11150b57cec5SDimitry Andric if (needsRuntimeRegistrationOfSectionRange(TT)) 11160b57cec5SDimitry Andric return; 11170b57cec5SDimitry Andric 11180b57cec5SDimitry Andric size_t TotalNS = 0; 11190b57cec5SDimitry Andric for (auto &PD : ProfileDataMap) { 11200b57cec5SDimitry Andric for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 11210b57cec5SDimitry Andric TotalNS += PD.second.NumValueSites[Kind]; 11220b57cec5SDimitry Andric } 11230b57cec5SDimitry Andric 11240b57cec5SDimitry Andric if (!TotalNS) 11250b57cec5SDimitry Andric return; 11260b57cec5SDimitry Andric 11270b57cec5SDimitry Andric uint64_t NumCounters = TotalNS * NumCountersPerValueSite; 11280b57cec5SDimitry Andric // Heuristic for small programs with very few total value sites. 11290b57cec5SDimitry Andric // The default value of vp-counters-per-site is chosen based on 11300b57cec5SDimitry Andric // the observation that large apps usually have a low percentage 11310b57cec5SDimitry Andric // of value sites that actually have any profile data, and thus 11320b57cec5SDimitry Andric // the average number of counters per site is low. For small 11330b57cec5SDimitry Andric // apps with very few sites, this may not be true. Bump up the 11340b57cec5SDimitry Andric // number of counters in this case. 11350b57cec5SDimitry Andric #define INSTR_PROF_MIN_VAL_COUNTS 10 11360b57cec5SDimitry Andric if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS) 11370b57cec5SDimitry Andric NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2); 11380b57cec5SDimitry Andric 11390b57cec5SDimitry Andric auto &Ctx = M->getContext(); 11400b57cec5SDimitry Andric Type *VNodeTypes[] = { 11410b57cec5SDimitry Andric #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType, 11420b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfData.inc" 11430b57cec5SDimitry Andric }; 11440b57cec5SDimitry Andric auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes)); 11450b57cec5SDimitry Andric 11460b57cec5SDimitry Andric ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters); 11470b57cec5SDimitry Andric auto *VNodesVar = new GlobalVariable( 11480b57cec5SDimitry Andric *M, VNodesTy, false, GlobalValue::PrivateLinkage, 11490b57cec5SDimitry Andric Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName()); 11500b57cec5SDimitry Andric VNodesVar->setSection( 11510b57cec5SDimitry Andric getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat())); 1152fe6060f1SDimitry Andric // VNodesVar is used by runtime but not referenced via relocation by other 1153fe6060f1SDimitry Andric // sections. Conservatively make it linker retained. 11540b57cec5SDimitry Andric UsedVars.push_back(VNodesVar); 11550b57cec5SDimitry Andric } 11560b57cec5SDimitry Andric 11570b57cec5SDimitry Andric void InstrProfiling::emitNameData() { 11580b57cec5SDimitry Andric std::string UncompressedData; 11590b57cec5SDimitry Andric 11600b57cec5SDimitry Andric if (ReferencedNames.empty()) 11610b57cec5SDimitry Andric return; 11620b57cec5SDimitry Andric 11630b57cec5SDimitry Andric std::string CompressedNameStr; 11640b57cec5SDimitry Andric if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr, 11655ffd83dbSDimitry Andric DoInstrProfNameCompression)) { 1166349cc55cSDimitry Andric report_fatal_error(Twine(toString(std::move(E))), false); 11670b57cec5SDimitry Andric } 11680b57cec5SDimitry Andric 11690b57cec5SDimitry Andric auto &Ctx = M->getContext(); 11704824e7fdSDimitry Andric auto *NamesVal = 11714824e7fdSDimitry Andric ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false); 11720b57cec5SDimitry Andric NamesVar = new GlobalVariable(*M, NamesVal->getType(), true, 11730b57cec5SDimitry Andric GlobalValue::PrivateLinkage, NamesVal, 11740b57cec5SDimitry Andric getInstrProfNamesVarName()); 11750b57cec5SDimitry Andric NamesSize = CompressedNameStr.size(); 11760b57cec5SDimitry Andric NamesVar->setSection( 11770b57cec5SDimitry Andric getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); 11780b57cec5SDimitry Andric // On COFF, it's important to reduce the alignment down to 1 to prevent the 11790b57cec5SDimitry Andric // linker from inserting padding before the start of the names section or 11800b57cec5SDimitry Andric // between names entries. 11815ffd83dbSDimitry Andric NamesVar->setAlignment(Align(1)); 1182fe6060f1SDimitry Andric // NamesVar is used by runtime but not referenced via relocation by other 1183fe6060f1SDimitry Andric // sections. Conservatively make it linker retained. 11840b57cec5SDimitry Andric UsedVars.push_back(NamesVar); 11850b57cec5SDimitry Andric 11860b57cec5SDimitry Andric for (auto *NamePtr : ReferencedNames) 11870b57cec5SDimitry Andric NamePtr->eraseFromParent(); 11880b57cec5SDimitry Andric } 11890b57cec5SDimitry Andric 11900b57cec5SDimitry Andric void InstrProfiling::emitRegistration() { 11910b57cec5SDimitry Andric if (!needsRuntimeRegistrationOfSectionRange(TT)) 11920b57cec5SDimitry Andric return; 11930b57cec5SDimitry Andric 11940b57cec5SDimitry Andric // Construct the function. 11950b57cec5SDimitry Andric auto *VoidTy = Type::getVoidTy(M->getContext()); 11960b57cec5SDimitry Andric auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext()); 11970b57cec5SDimitry Andric auto *Int64Ty = Type::getInt64Ty(M->getContext()); 11980b57cec5SDimitry Andric auto *RegisterFTy = FunctionType::get(VoidTy, false); 11990b57cec5SDimitry Andric auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage, 12000b57cec5SDimitry Andric getInstrProfRegFuncsName(), M); 12010b57cec5SDimitry Andric RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); 12020b57cec5SDimitry Andric if (Options.NoRedZone) 12030b57cec5SDimitry Andric RegisterF->addFnAttr(Attribute::NoRedZone); 12040b57cec5SDimitry Andric 12050b57cec5SDimitry Andric auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false); 12060b57cec5SDimitry Andric auto *RuntimeRegisterF = 12070b57cec5SDimitry Andric Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage, 12080b57cec5SDimitry Andric getInstrProfRegFuncName(), M); 12090b57cec5SDimitry Andric 12100b57cec5SDimitry Andric IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF)); 1211fe6060f1SDimitry Andric for (Value *Data : CompilerUsedVars) 1212fe6060f1SDimitry Andric if (!isa<Function>(Data)) 1213fe6060f1SDimitry Andric IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy)); 12140b57cec5SDimitry Andric for (Value *Data : UsedVars) 12150b57cec5SDimitry Andric if (Data != NamesVar && !isa<Function>(Data)) 12160b57cec5SDimitry Andric IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy)); 12170b57cec5SDimitry Andric 12180b57cec5SDimitry Andric if (NamesVar) { 12190b57cec5SDimitry Andric Type *ParamTypes[] = {VoidPtrTy, Int64Ty}; 12200b57cec5SDimitry Andric auto *NamesRegisterTy = 12210b57cec5SDimitry Andric FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false); 12220b57cec5SDimitry Andric auto *NamesRegisterF = 12230b57cec5SDimitry Andric Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage, 12240b57cec5SDimitry Andric getInstrProfNamesRegFuncName(), M); 12250b57cec5SDimitry Andric IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy), 12260b57cec5SDimitry Andric IRB.getInt64(NamesSize)}); 12270b57cec5SDimitry Andric } 12280b57cec5SDimitry Andric 12290b57cec5SDimitry Andric IRB.CreateRetVoid(); 12300b57cec5SDimitry Andric } 12310b57cec5SDimitry Andric 12320b57cec5SDimitry Andric bool InstrProfiling::emitRuntimeHook() { 1233349cc55cSDimitry Andric // We expect the linker to be invoked with -u<hook_var> flag for Linux 1234349cc55cSDimitry Andric // in which case there is no need to emit the external variable. 1235349cc55cSDimitry Andric if (TT.isOSLinux()) 12360b57cec5SDimitry Andric return false; 12370b57cec5SDimitry Andric 12380b57cec5SDimitry Andric // If the module's provided its own runtime, we don't need to do anything. 12390b57cec5SDimitry Andric if (M->getGlobalVariable(getInstrProfRuntimeHookVarName())) 12400b57cec5SDimitry Andric return false; 12410b57cec5SDimitry Andric 12420b57cec5SDimitry Andric // Declare an external variable that will pull in the runtime initialization. 12430b57cec5SDimitry Andric auto *Int32Ty = Type::getInt32Ty(M->getContext()); 12440b57cec5SDimitry Andric auto *Var = 12450b57cec5SDimitry Andric new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage, 12460b57cec5SDimitry Andric nullptr, getInstrProfRuntimeHookVarName()); 1247753f127fSDimitry Andric Var->setVisibility(GlobalValue::HiddenVisibility); 12480b57cec5SDimitry Andric 124981ad6265SDimitry Andric if (TT.isOSBinFormatELF() && !TT.isPS()) { 1250349cc55cSDimitry Andric // Mark the user variable as used so that it isn't stripped out. 1251349cc55cSDimitry Andric CompilerUsedVars.push_back(Var); 1252349cc55cSDimitry Andric } else { 12530b57cec5SDimitry Andric // Make a function that uses it. 12540b57cec5SDimitry Andric auto *User = Function::Create(FunctionType::get(Int32Ty, false), 12550b57cec5SDimitry Andric GlobalValue::LinkOnceODRLinkage, 12560b57cec5SDimitry Andric getInstrProfRuntimeHookVarUseFuncName(), M); 12570b57cec5SDimitry Andric User->addFnAttr(Attribute::NoInline); 12580b57cec5SDimitry Andric if (Options.NoRedZone) 12590b57cec5SDimitry Andric User->addFnAttr(Attribute::NoRedZone); 12600b57cec5SDimitry Andric User->setVisibility(GlobalValue::HiddenVisibility); 12610b57cec5SDimitry Andric if (TT.supportsCOMDAT()) 12620b57cec5SDimitry Andric User->setComdat(M->getOrInsertComdat(User->getName())); 12630b57cec5SDimitry Andric 12640b57cec5SDimitry Andric IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User)); 12650b57cec5SDimitry Andric auto *Load = IRB.CreateLoad(Int32Ty, Var); 12660b57cec5SDimitry Andric IRB.CreateRet(Load); 12670b57cec5SDimitry Andric 1268349cc55cSDimitry Andric // Mark the function as used so that it isn't stripped out. 1269fe6060f1SDimitry Andric CompilerUsedVars.push_back(User); 1270349cc55cSDimitry Andric } 12710b57cec5SDimitry Andric return true; 12720b57cec5SDimitry Andric } 12730b57cec5SDimitry Andric 12740b57cec5SDimitry Andric void InstrProfiling::emitUses() { 1275fe6060f1SDimitry Andric // The metadata sections are parallel arrays. Optimizers (e.g. 1276fe6060f1SDimitry Andric // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so 1277fe6060f1SDimitry Andric // we conservatively retain all unconditionally in the compiler. 1278fe6060f1SDimitry Andric // 1279349cc55cSDimitry Andric // On ELF and Mach-O, the linker can guarantee the associated sections will be 1280349cc55cSDimitry Andric // retained or discarded as a unit, so llvm.compiler.used is sufficient. 1281349cc55cSDimitry Andric // Similarly on COFF, if prof data is not referenced by code we use one comdat 1282349cc55cSDimitry Andric // and ensure this GC property as well. Otherwise, we have to conservatively 1283349cc55cSDimitry Andric // make all of the sections retained by the linker. 1284349cc55cSDimitry Andric if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() || 1285fe6060f1SDimitry Andric (TT.isOSBinFormatCOFF() && !profDataReferencedByCode(*M))) 1286fe6060f1SDimitry Andric appendToCompilerUsed(*M, CompilerUsedVars); 1287fe6060f1SDimitry Andric else 1288fe6060f1SDimitry Andric appendToUsed(*M, CompilerUsedVars); 1289fe6060f1SDimitry Andric 1290fe6060f1SDimitry Andric // We do not add proper references from used metadata sections to NamesVar and 1291fe6060f1SDimitry Andric // VNodesVar, so we have to be conservative and place them in llvm.used 1292fe6060f1SDimitry Andric // regardless of the target, 12930b57cec5SDimitry Andric appendToUsed(*M, UsedVars); 12940b57cec5SDimitry Andric } 12950b57cec5SDimitry Andric 12960b57cec5SDimitry Andric void InstrProfiling::emitInitialization() { 12970b57cec5SDimitry Andric // Create ProfileFileName variable. Don't don't this for the 12980b57cec5SDimitry Andric // context-sensitive instrumentation lowering: This lowering is after 12990b57cec5SDimitry Andric // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should 13000b57cec5SDimitry Andric // have already create the variable before LTO/ThinLTO linking. 13010b57cec5SDimitry Andric if (!IsCS) 13020b57cec5SDimitry Andric createProfileFileNameVar(*M, Options.InstrProfileOutput); 13030b57cec5SDimitry Andric Function *RegisterF = M->getFunction(getInstrProfRegFuncsName()); 13040b57cec5SDimitry Andric if (!RegisterF) 13050b57cec5SDimitry Andric return; 13060b57cec5SDimitry Andric 13070b57cec5SDimitry Andric // Create the initialization function. 13080b57cec5SDimitry Andric auto *VoidTy = Type::getVoidTy(M->getContext()); 13090b57cec5SDimitry Andric auto *F = Function::Create(FunctionType::get(VoidTy, false), 13100b57cec5SDimitry Andric GlobalValue::InternalLinkage, 13110b57cec5SDimitry Andric getInstrProfInitFuncName(), M); 13120b57cec5SDimitry Andric F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); 13130b57cec5SDimitry Andric F->addFnAttr(Attribute::NoInline); 13140b57cec5SDimitry Andric if (Options.NoRedZone) 13150b57cec5SDimitry Andric F->addFnAttr(Attribute::NoRedZone); 13160b57cec5SDimitry Andric 13170b57cec5SDimitry Andric // Add the basic block and the necessary calls. 13180b57cec5SDimitry Andric IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F)); 13190b57cec5SDimitry Andric IRB.CreateCall(RegisterF, {}); 13200b57cec5SDimitry Andric IRB.CreateRetVoid(); 13210b57cec5SDimitry Andric 13220b57cec5SDimitry Andric appendToGlobalCtors(*M, F, 0); 13230b57cec5SDimitry Andric } 1324