10b57cec5SDimitry Andric //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
95f757f3fSDimitry Andric // This pass lowers instrprof_* intrinsics emitted by an instrumentor.
100b57cec5SDimitry Andric // It also builds the data structures and initialization code needed for
110b57cec5SDimitry Andric // updating execution counts and emitting the profile at runtime.
120b57cec5SDimitry Andric //
130b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
140b57cec5SDimitry Andric
150b57cec5SDimitry Andric #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
160b57cec5SDimitry Andric #include "llvm/ADT/ArrayRef.h"
175f757f3fSDimitry Andric #include "llvm/ADT/STLExtras.h"
180b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h"
190b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
200b57cec5SDimitry Andric #include "llvm/ADT/Twine.h"
210b57cec5SDimitry Andric #include "llvm/Analysis/BlockFrequencyInfo.h"
220b57cec5SDimitry Andric #include "llvm/Analysis/BranchProbabilityInfo.h"
230b57cec5SDimitry Andric #include "llvm/Analysis/LoopInfo.h"
240b57cec5SDimitry Andric #include "llvm/Analysis/TargetLibraryInfo.h"
250b57cec5SDimitry Andric #include "llvm/IR/Attributes.h"
260b57cec5SDimitry Andric #include "llvm/IR/BasicBlock.h"
275f757f3fSDimitry Andric #include "llvm/IR/CFG.h"
280b57cec5SDimitry Andric #include "llvm/IR/Constant.h"
290b57cec5SDimitry Andric #include "llvm/IR/Constants.h"
300eae32dcSDimitry Andric #include "llvm/IR/DIBuilder.h"
310b57cec5SDimitry Andric #include "llvm/IR/DerivedTypes.h"
320eae32dcSDimitry Andric #include "llvm/IR/DiagnosticInfo.h"
330b57cec5SDimitry Andric #include "llvm/IR/Dominators.h"
340b57cec5SDimitry Andric #include "llvm/IR/Function.h"
350b57cec5SDimitry Andric #include "llvm/IR/GlobalValue.h"
360b57cec5SDimitry Andric #include "llvm/IR/GlobalVariable.h"
370b57cec5SDimitry Andric #include "llvm/IR/IRBuilder.h"
380b57cec5SDimitry Andric #include "llvm/IR/Instruction.h"
390b57cec5SDimitry Andric #include "llvm/IR/Instructions.h"
400b57cec5SDimitry Andric #include "llvm/IR/IntrinsicInst.h"
410b57cec5SDimitry Andric #include "llvm/IR/Module.h"
420b57cec5SDimitry Andric #include "llvm/IR/Type.h"
43480093f4SDimitry Andric #include "llvm/InitializePasses.h"
440b57cec5SDimitry Andric #include "llvm/Pass.h"
450b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProf.h"
460eae32dcSDimitry Andric #include "llvm/ProfileData/InstrProfCorrelator.h"
470b57cec5SDimitry Andric #include "llvm/Support/Casting.h"
480b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h"
490b57cec5SDimitry Andric #include "llvm/Support/Error.h"
500b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h"
5106c3fb27SDimitry Andric #include "llvm/TargetParser/Triple.h"
525f757f3fSDimitry Andric #include "llvm/Transforms/Instrumentation.h"
535f757f3fSDimitry Andric #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
545f757f3fSDimitry Andric #include "llvm/Transforms/Utils/BasicBlockUtils.h"
550b57cec5SDimitry Andric #include "llvm/Transforms/Utils/ModuleUtils.h"
560b57cec5SDimitry Andric #include "llvm/Transforms/Utils/SSAUpdater.h"
570b57cec5SDimitry Andric #include <algorithm>
580b57cec5SDimitry Andric #include <cassert>
590b57cec5SDimitry Andric #include <cstdint>
600b57cec5SDimitry Andric #include <string>
610b57cec5SDimitry Andric
620b57cec5SDimitry Andric using namespace llvm;
630b57cec5SDimitry Andric
640b57cec5SDimitry Andric #define DEBUG_TYPE "instrprof"
650b57cec5SDimitry Andric
660eae32dcSDimitry Andric namespace llvm {
675f757f3fSDimitry Andric // TODO: Remove -debug-info-correlate in next LLVM release, in favor of
685f757f3fSDimitry Andric // -profile-correlate=debug-info.
695f757f3fSDimitry Andric cl::opt<bool> DebugInfoCorrelate(
705f757f3fSDimitry Andric "debug-info-correlate",
715f757f3fSDimitry Andric cl::desc("Use debug info to correlate profiles. (Deprecated, use "
725f757f3fSDimitry Andric "-profile-correlate=debug-info)"),
730eae32dcSDimitry Andric cl::init(false));
745f757f3fSDimitry Andric
755f757f3fSDimitry Andric cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate(
765f757f3fSDimitry Andric "profile-correlate",
775f757f3fSDimitry Andric cl::desc("Use debug info or binary file to correlate profiles."),
785f757f3fSDimitry Andric cl::init(InstrProfCorrelator::NONE),
795f757f3fSDimitry Andric cl::values(clEnumValN(InstrProfCorrelator::NONE, "",
805f757f3fSDimitry Andric "No profile correlation"),
815f757f3fSDimitry Andric clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info",
825f757f3fSDimitry Andric "Use debug info to correlate"),
835f757f3fSDimitry Andric clEnumValN(InstrProfCorrelator::BINARY, "binary",
845f757f3fSDimitry Andric "Use binary to correlate")));
850eae32dcSDimitry Andric } // namespace llvm
860eae32dcSDimitry Andric
870b57cec5SDimitry Andric namespace {
880b57cec5SDimitry Andric
890b57cec5SDimitry Andric cl::opt<bool> DoHashBasedCounterSplit(
900b57cec5SDimitry Andric "hash-based-counter-split",
910b57cec5SDimitry Andric cl::desc("Rename counter variable of a comdat function based on cfg hash"),
920b57cec5SDimitry Andric cl::init(true));
930b57cec5SDimitry Andric
944824e7fdSDimitry Andric cl::opt<bool>
954824e7fdSDimitry Andric RuntimeCounterRelocation("runtime-counter-relocation",
965ffd83dbSDimitry Andric cl::desc("Enable relocating counters at runtime."),
975ffd83dbSDimitry Andric cl::init(false));
985ffd83dbSDimitry Andric
990b57cec5SDimitry Andric cl::opt<bool> ValueProfileStaticAlloc(
1000b57cec5SDimitry Andric "vp-static-alloc",
1010b57cec5SDimitry Andric cl::desc("Do static counter allocation for value profiler"),
1020b57cec5SDimitry Andric cl::init(true));
1030b57cec5SDimitry Andric
1040b57cec5SDimitry Andric cl::opt<double> NumCountersPerValueSite(
1050b57cec5SDimitry Andric "vp-counters-per-site",
1060b57cec5SDimitry Andric cl::desc("The average number of profile counters allocated "
1070b57cec5SDimitry Andric "per value profiling site."),
1080b57cec5SDimitry Andric // This is set to a very small value because in real programs, only
1090b57cec5SDimitry Andric // a very small percentage of value sites have non-zero targets, e.g, 1/30.
1100b57cec5SDimitry Andric // For those sites with non-zero profile, the average number of targets
1110b57cec5SDimitry Andric // is usually smaller than 2.
1120b57cec5SDimitry Andric cl::init(1.0));
1130b57cec5SDimitry Andric
1140b57cec5SDimitry Andric cl::opt<bool> AtomicCounterUpdateAll(
11581ad6265SDimitry Andric "instrprof-atomic-counter-update-all",
1160b57cec5SDimitry Andric cl::desc("Make all profile counter updates atomic (for testing only)"),
1170b57cec5SDimitry Andric cl::init(false));
1180b57cec5SDimitry Andric
1190b57cec5SDimitry Andric cl::opt<bool> AtomicCounterUpdatePromoted(
12081ad6265SDimitry Andric "atomic-counter-update-promoted",
1210b57cec5SDimitry Andric cl::desc("Do counter update using atomic fetch add "
1220b57cec5SDimitry Andric " for promoted counters only"),
1230b57cec5SDimitry Andric cl::init(false));
1240b57cec5SDimitry Andric
1255ffd83dbSDimitry Andric cl::opt<bool> AtomicFirstCounter(
12681ad6265SDimitry Andric "atomic-first-counter",
1275ffd83dbSDimitry Andric cl::desc("Use atomic fetch add for first counter in a function (usually "
1285ffd83dbSDimitry Andric "the entry counter)"),
1295ffd83dbSDimitry Andric cl::init(false));
1305ffd83dbSDimitry Andric
1310b57cec5SDimitry Andric // If the option is not specified, the default behavior about whether
1320b57cec5SDimitry Andric // counter promotion is done depends on how instrumentaiton lowering
1330b57cec5SDimitry Andric // pipeline is setup, i.e., the default value of true of this option
1340b57cec5SDimitry Andric // does not mean the promotion will be done by default. Explicitly
1350b57cec5SDimitry Andric // setting this option can override the default behavior.
13681ad6265SDimitry Andric cl::opt<bool> DoCounterPromotion("do-counter-promotion",
1370b57cec5SDimitry Andric cl::desc("Do counter register promotion"),
1380b57cec5SDimitry Andric cl::init(false));
1390b57cec5SDimitry Andric cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
14081ad6265SDimitry Andric "max-counter-promotions-per-loop", cl::init(20),
1410b57cec5SDimitry Andric cl::desc("Max number counter promotions per loop to avoid"
1420b57cec5SDimitry Andric " increasing register pressure too much"));
1430b57cec5SDimitry Andric
1440b57cec5SDimitry Andric // A debug option
1450b57cec5SDimitry Andric cl::opt<int>
14681ad6265SDimitry Andric MaxNumOfPromotions("max-counter-promotions", cl::init(-1),
1470b57cec5SDimitry Andric cl::desc("Max number of allowed counter promotions"));
1480b57cec5SDimitry Andric
1490b57cec5SDimitry Andric cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
15081ad6265SDimitry Andric "speculative-counter-promotion-max-exiting", cl::init(3),
1510b57cec5SDimitry Andric cl::desc("The max number of exiting blocks of a loop to allow "
1520b57cec5SDimitry Andric " speculative counter promotion"));
1530b57cec5SDimitry Andric
1540b57cec5SDimitry Andric cl::opt<bool> SpeculativeCounterPromotionToLoop(
15581ad6265SDimitry Andric "speculative-counter-promotion-to-loop",
1560b57cec5SDimitry Andric cl::desc("When the option is false, if the target block is in a loop, "
1570b57cec5SDimitry Andric "the promotion will be disallowed unless the promoted counter "
1580b57cec5SDimitry Andric " update can be further/iteratively promoted into an acyclic "
1590b57cec5SDimitry Andric " region."));
1600b57cec5SDimitry Andric
1610b57cec5SDimitry Andric cl::opt<bool> IterativeCounterPromotion(
16281ad6265SDimitry Andric "iterative-counter-promotion", cl::init(true),
1630b57cec5SDimitry Andric cl::desc("Allow counter promotion across the whole loop nest."));
1640b57cec5SDimitry Andric
165e8d8bef9SDimitry Andric cl::opt<bool> SkipRetExitBlock(
16681ad6265SDimitry Andric "skip-ret-exit-block", cl::init(true),
167e8d8bef9SDimitry Andric cl::desc("Suppress counter promotion if exit blocks contain ret."));
168e8d8bef9SDimitry Andric
1695f757f3fSDimitry Andric using LoadStorePair = std::pair<Instruction *, Instruction *>;
1705f757f3fSDimitry Andric
1715f757f3fSDimitry Andric class InstrLowerer final {
1725f757f3fSDimitry Andric public:
InstrLowerer(Module & M,const InstrProfOptions & Options,std::function<const TargetLibraryInfo & (Function & F)> GetTLI,bool IsCS)1735f757f3fSDimitry Andric InstrLowerer(Module &M, const InstrProfOptions &Options,
1745f757f3fSDimitry Andric std::function<const TargetLibraryInfo &(Function &F)> GetTLI,
1755f757f3fSDimitry Andric bool IsCS)
1765f757f3fSDimitry Andric : M(M), Options(Options), TT(Triple(M.getTargetTriple())), IsCS(IsCS),
1775f757f3fSDimitry Andric GetTLI(GetTLI) {}
1785f757f3fSDimitry Andric
1795f757f3fSDimitry Andric bool lower();
1805f757f3fSDimitry Andric
1815f757f3fSDimitry Andric private:
1825f757f3fSDimitry Andric Module &M;
1835f757f3fSDimitry Andric const InstrProfOptions Options;
1845f757f3fSDimitry Andric const Triple TT;
1855f757f3fSDimitry Andric // Is this lowering for the context-sensitive instrumentation.
1865f757f3fSDimitry Andric const bool IsCS;
1875f757f3fSDimitry Andric
1885f757f3fSDimitry Andric std::function<const TargetLibraryInfo &(Function &F)> GetTLI;
1895f757f3fSDimitry Andric struct PerFunctionProfileData {
1905f757f3fSDimitry Andric uint32_t NumValueSites[IPVK_Last + 1] = {};
1915f757f3fSDimitry Andric GlobalVariable *RegionCounters = nullptr;
1925f757f3fSDimitry Andric GlobalVariable *DataVar = nullptr;
1935f757f3fSDimitry Andric GlobalVariable *RegionBitmaps = nullptr;
1945f757f3fSDimitry Andric uint32_t NumBitmapBytes = 0;
1955f757f3fSDimitry Andric
1965f757f3fSDimitry Andric PerFunctionProfileData() = default;
1975f757f3fSDimitry Andric };
1985f757f3fSDimitry Andric DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
1995f757f3fSDimitry Andric /// If runtime relocation is enabled, this maps functions to the load
2005f757f3fSDimitry Andric /// instruction that produces the profile relocation bias.
2015f757f3fSDimitry Andric DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
2025f757f3fSDimitry Andric std::vector<GlobalValue *> CompilerUsedVars;
2035f757f3fSDimitry Andric std::vector<GlobalValue *> UsedVars;
2045f757f3fSDimitry Andric std::vector<GlobalVariable *> ReferencedNames;
2055f757f3fSDimitry Andric GlobalVariable *NamesVar = nullptr;
2065f757f3fSDimitry Andric size_t NamesSize = 0;
2075f757f3fSDimitry Andric
2085f757f3fSDimitry Andric // vector of counter load/store pairs to be register promoted.
2095f757f3fSDimitry Andric std::vector<LoadStorePair> PromotionCandidates;
2105f757f3fSDimitry Andric
2115f757f3fSDimitry Andric int64_t TotalCountersPromoted = 0;
2125f757f3fSDimitry Andric
2135f757f3fSDimitry Andric /// Lower instrumentation intrinsics in the function. Returns true if there
2145f757f3fSDimitry Andric /// any lowering.
2155f757f3fSDimitry Andric bool lowerIntrinsics(Function *F);
2165f757f3fSDimitry Andric
2175f757f3fSDimitry Andric /// Register-promote counter loads and stores in loops.
2185f757f3fSDimitry Andric void promoteCounterLoadStores(Function *F);
2195f757f3fSDimitry Andric
2205f757f3fSDimitry Andric /// Returns true if relocating counters at runtime is enabled.
2215f757f3fSDimitry Andric bool isRuntimeCounterRelocationEnabled() const;
2225f757f3fSDimitry Andric
2235f757f3fSDimitry Andric /// Returns true if profile counter update register promotion is enabled.
2245f757f3fSDimitry Andric bool isCounterPromotionEnabled() const;
2255f757f3fSDimitry Andric
2265f757f3fSDimitry Andric /// Count the number of instrumented value sites for the function.
2275f757f3fSDimitry Andric void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
2285f757f3fSDimitry Andric
2295f757f3fSDimitry Andric /// Replace instrprof.value.profile with a call to runtime library.
2305f757f3fSDimitry Andric void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
2315f757f3fSDimitry Andric
2325f757f3fSDimitry Andric /// Replace instrprof.cover with a store instruction to the coverage byte.
2335f757f3fSDimitry Andric void lowerCover(InstrProfCoverInst *Inc);
2345f757f3fSDimitry Andric
2355f757f3fSDimitry Andric /// Replace instrprof.timestamp with a call to
2365f757f3fSDimitry Andric /// INSTR_PROF_PROFILE_SET_TIMESTAMP.
2375f757f3fSDimitry Andric void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction);
2385f757f3fSDimitry Andric
2395f757f3fSDimitry Andric /// Replace instrprof.increment with an increment of the appropriate value.
2405f757f3fSDimitry Andric void lowerIncrement(InstrProfIncrementInst *Inc);
2415f757f3fSDimitry Andric
2425f757f3fSDimitry Andric /// Force emitting of name vars for unused functions.
2435f757f3fSDimitry Andric void lowerCoverageData(GlobalVariable *CoverageNamesVar);
2445f757f3fSDimitry Andric
2455f757f3fSDimitry Andric /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction
2465f757f3fSDimitry Andric /// using the index represented by the a temp value into a bitmap.
2475f757f3fSDimitry Andric void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins);
2485f757f3fSDimitry Andric
2495f757f3fSDimitry Andric /// Replace instrprof.mcdc.temp.update with a shift and or instruction using
2505f757f3fSDimitry Andric /// the corresponding condition ID.
2515f757f3fSDimitry Andric void lowerMCDCCondBitmapUpdate(InstrProfMCDCCondBitmapUpdate *Ins);
2525f757f3fSDimitry Andric
2535f757f3fSDimitry Andric /// Compute the address of the counter value that this profiling instruction
2545f757f3fSDimitry Andric /// acts on.
2555f757f3fSDimitry Andric Value *getCounterAddress(InstrProfCntrInstBase *I);
2565f757f3fSDimitry Andric
2575f757f3fSDimitry Andric /// Get the region counters for an increment, creating them if necessary.
2585f757f3fSDimitry Andric ///
2595f757f3fSDimitry Andric /// If the counter array doesn't yet exist, the profile data variables
2605f757f3fSDimitry Andric /// referring to them will also be created.
2615f757f3fSDimitry Andric GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc);
2625f757f3fSDimitry Andric
2635f757f3fSDimitry Andric /// Create the region counters.
2645f757f3fSDimitry Andric GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc,
2655f757f3fSDimitry Andric StringRef Name,
2665f757f3fSDimitry Andric GlobalValue::LinkageTypes Linkage);
2675f757f3fSDimitry Andric
2685f757f3fSDimitry Andric /// Compute the address of the test vector bitmap that this profiling
2695f757f3fSDimitry Andric /// instruction acts on.
2705f757f3fSDimitry Andric Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);
2715f757f3fSDimitry Andric
2725f757f3fSDimitry Andric /// Get the region bitmaps for an increment, creating them if necessary.
2735f757f3fSDimitry Andric ///
2745f757f3fSDimitry Andric /// If the bitmap array doesn't yet exist, the profile data variables
2755f757f3fSDimitry Andric /// referring to them will also be created.
2765f757f3fSDimitry Andric GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc);
2775f757f3fSDimitry Andric
2785f757f3fSDimitry Andric /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with
2795f757f3fSDimitry Andric /// an MC/DC Decision region. The number of bytes required is indicated by
2805f757f3fSDimitry Andric /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called
2815f757f3fSDimitry Andric /// as part of setupProfileSection() and is conceptually very similar to
2825f757f3fSDimitry Andric /// what is done for profile data counters in createRegionCounters().
2835f757f3fSDimitry Andric GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
2845f757f3fSDimitry Andric StringRef Name,
2855f757f3fSDimitry Andric GlobalValue::LinkageTypes Linkage);
2865f757f3fSDimitry Andric
2875f757f3fSDimitry Andric /// Set Comdat property of GV, if required.
2885f757f3fSDimitry Andric void maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef VarName);
2895f757f3fSDimitry Andric
2905f757f3fSDimitry Andric /// Setup the sections into which counters and bitmaps are allocated.
2915f757f3fSDimitry Andric GlobalVariable *setupProfileSection(InstrProfInstBase *Inc,
2925f757f3fSDimitry Andric InstrProfSectKind IPSK);
2935f757f3fSDimitry Andric
2945f757f3fSDimitry Andric /// Create INSTR_PROF_DATA variable for counters and bitmaps.
2955f757f3fSDimitry Andric void createDataVariable(InstrProfCntrInstBase *Inc);
2965f757f3fSDimitry Andric
2975f757f3fSDimitry Andric /// Emit the section with compressed function names.
2985f757f3fSDimitry Andric void emitNameData();
2995f757f3fSDimitry Andric
3005f757f3fSDimitry Andric /// Emit value nodes section for value profiling.
3015f757f3fSDimitry Andric void emitVNodes();
3025f757f3fSDimitry Andric
3035f757f3fSDimitry Andric /// Emit runtime registration functions for each profile data variable.
3045f757f3fSDimitry Andric void emitRegistration();
3055f757f3fSDimitry Andric
3065f757f3fSDimitry Andric /// Emit the necessary plumbing to pull in the runtime initialization.
3075f757f3fSDimitry Andric /// Returns true if a change was made.
3085f757f3fSDimitry Andric bool emitRuntimeHook();
3095f757f3fSDimitry Andric
3105f757f3fSDimitry Andric /// Add uses of our data variables and runtime hook.
3115f757f3fSDimitry Andric void emitUses();
3125f757f3fSDimitry Andric
3135f757f3fSDimitry Andric /// Create a static initializer for our data, on platforms that need it,
3145f757f3fSDimitry Andric /// and for any profile output file that was specified.
3155f757f3fSDimitry Andric void emitInitialization();
3165f757f3fSDimitry Andric };
3175f757f3fSDimitry Andric
3180b57cec5SDimitry Andric ///
3190b57cec5SDimitry Andric /// A helper class to promote one counter RMW operation in the loop
3200b57cec5SDimitry Andric /// into register update.
3210b57cec5SDimitry Andric ///
3220b57cec5SDimitry Andric /// RWM update for the counter will be sinked out of the loop after
3230b57cec5SDimitry Andric /// the transformation.
3240b57cec5SDimitry Andric ///
3250b57cec5SDimitry Andric class PGOCounterPromoterHelper : public LoadAndStorePromoter {
3260b57cec5SDimitry Andric public:
PGOCounterPromoterHelper(Instruction * L,Instruction * S,SSAUpdater & SSA,Value * Init,BasicBlock * PH,ArrayRef<BasicBlock * > ExitBlocks,ArrayRef<Instruction * > InsertPts,DenseMap<Loop *,SmallVector<LoadStorePair,8>> & LoopToCands,LoopInfo & LI)3270b57cec5SDimitry Andric PGOCounterPromoterHelper(
3280b57cec5SDimitry Andric Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,
3290b57cec5SDimitry Andric BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
3300b57cec5SDimitry Andric ArrayRef<Instruction *> InsertPts,
3310b57cec5SDimitry Andric DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
3320b57cec5SDimitry Andric LoopInfo &LI)
3330b57cec5SDimitry Andric : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
3340b57cec5SDimitry Andric InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
3350b57cec5SDimitry Andric assert(isa<LoadInst>(L));
3360b57cec5SDimitry Andric assert(isa<StoreInst>(S));
3370b57cec5SDimitry Andric SSA.AddAvailableValue(PH, Init);
3380b57cec5SDimitry Andric }
3390b57cec5SDimitry Andric
doExtraRewritesBeforeFinalDeletion()3400b57cec5SDimitry Andric void doExtraRewritesBeforeFinalDeletion() override {
3410b57cec5SDimitry Andric for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
3420b57cec5SDimitry Andric BasicBlock *ExitBlock = ExitBlocks[i];
3430b57cec5SDimitry Andric Instruction *InsertPos = InsertPts[i];
3440b57cec5SDimitry Andric // Get LiveIn value into the ExitBlock. If there are multiple
3450b57cec5SDimitry Andric // predecessors, the value is defined by a PHI node in this
3460b57cec5SDimitry Andric // block.
3470b57cec5SDimitry Andric Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
3480b57cec5SDimitry Andric Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
3490b57cec5SDimitry Andric Type *Ty = LiveInValue->getType();
3500b57cec5SDimitry Andric IRBuilder<> Builder(InsertPos);
35181ad6265SDimitry Andric if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Addr)) {
35281ad6265SDimitry Andric // If isRuntimeCounterRelocationEnabled() is true then the address of
35381ad6265SDimitry Andric // the store instruction is computed with two instructions in
35481ad6265SDimitry Andric // InstrProfiling::getCounterAddress(). We need to copy those
35581ad6265SDimitry Andric // instructions to this block to compute Addr correctly.
35681ad6265SDimitry Andric // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
35781ad6265SDimitry Andric // %Addr = inttoptr i64 %BiasAdd to i64*
35881ad6265SDimitry Andric auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
35981ad6265SDimitry Andric assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
36081ad6265SDimitry Andric Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
3615f757f3fSDimitry Andric Addr = Builder.CreateIntToPtr(BiasInst,
3625f757f3fSDimitry Andric PointerType::getUnqual(Ty->getContext()));
36381ad6265SDimitry Andric }
3640b57cec5SDimitry Andric if (AtomicCounterUpdatePromoted)
3650b57cec5SDimitry Andric // automic update currently can only be promoted across the current
3660b57cec5SDimitry Andric // loop, not the whole loop nest.
3670b57cec5SDimitry Andric Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
368fe6060f1SDimitry Andric MaybeAlign(),
3690b57cec5SDimitry Andric AtomicOrdering::SequentiallyConsistent);
3700b57cec5SDimitry Andric else {
3710b57cec5SDimitry Andric LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
3720b57cec5SDimitry Andric auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
3730b57cec5SDimitry Andric auto *NewStore = Builder.CreateStore(NewVal, Addr);
3740b57cec5SDimitry Andric
3750b57cec5SDimitry Andric // Now update the parent loop's candidate list:
3760b57cec5SDimitry Andric if (IterativeCounterPromotion) {
3770b57cec5SDimitry Andric auto *TargetLoop = LI.getLoopFor(ExitBlock);
3780b57cec5SDimitry Andric if (TargetLoop)
3790b57cec5SDimitry Andric LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
3800b57cec5SDimitry Andric }
3810b57cec5SDimitry Andric }
3820b57cec5SDimitry Andric }
3830b57cec5SDimitry Andric }
3840b57cec5SDimitry Andric
3850b57cec5SDimitry Andric private:
3860b57cec5SDimitry Andric Instruction *Store;
3870b57cec5SDimitry Andric ArrayRef<BasicBlock *> ExitBlocks;
3880b57cec5SDimitry Andric ArrayRef<Instruction *> InsertPts;
3890b57cec5SDimitry Andric DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
3900b57cec5SDimitry Andric LoopInfo &LI;
3910b57cec5SDimitry Andric };
3920b57cec5SDimitry Andric
3930b57cec5SDimitry Andric /// A helper class to do register promotion for all profile counter
3940b57cec5SDimitry Andric /// updates in a loop.
3950b57cec5SDimitry Andric ///
3960b57cec5SDimitry Andric class PGOCounterPromoter {
3970b57cec5SDimitry Andric public:
PGOCounterPromoter(DenseMap<Loop *,SmallVector<LoadStorePair,8>> & LoopToCands,Loop & CurLoop,LoopInfo & LI,BlockFrequencyInfo * BFI)3980b57cec5SDimitry Andric PGOCounterPromoter(
3990b57cec5SDimitry Andric DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
4000b57cec5SDimitry Andric Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
40104eeddc0SDimitry Andric : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
4020b57cec5SDimitry Andric
4035ffd83dbSDimitry Andric // Skip collection of ExitBlocks and InsertPts for loops that will not be
4045ffd83dbSDimitry Andric // able to have counters promoted.
4050b57cec5SDimitry Andric SmallVector<BasicBlock *, 8> LoopExitBlocks;
4060b57cec5SDimitry Andric SmallPtrSet<BasicBlock *, 8> BlockSet;
4075ffd83dbSDimitry Andric
4080b57cec5SDimitry Andric L.getExitBlocks(LoopExitBlocks);
4095ffd83dbSDimitry Andric if (!isPromotionPossible(&L, LoopExitBlocks))
4105ffd83dbSDimitry Andric return;
4110b57cec5SDimitry Andric
4120b57cec5SDimitry Andric for (BasicBlock *ExitBlock : LoopExitBlocks) {
4135f757f3fSDimitry Andric if (BlockSet.insert(ExitBlock).second &&
4145f757f3fSDimitry Andric llvm::none_of(predecessors(ExitBlock), [&](const BasicBlock *Pred) {
4155f757f3fSDimitry Andric return llvm::isPresplitCoroSuspendExitEdge(*Pred, *ExitBlock);
4165f757f3fSDimitry Andric })) {
4170b57cec5SDimitry Andric ExitBlocks.push_back(ExitBlock);
4180b57cec5SDimitry Andric InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
4190b57cec5SDimitry Andric }
4200b57cec5SDimitry Andric }
4210b57cec5SDimitry Andric }
4220b57cec5SDimitry Andric
run(int64_t * NumPromoted)4230b57cec5SDimitry Andric bool run(int64_t *NumPromoted) {
4240b57cec5SDimitry Andric // Skip 'infinite' loops:
4250b57cec5SDimitry Andric if (ExitBlocks.size() == 0)
4260b57cec5SDimitry Andric return false;
427e8d8bef9SDimitry Andric
428e8d8bef9SDimitry Andric // Skip if any of the ExitBlocks contains a ret instruction.
429e8d8bef9SDimitry Andric // This is to prevent dumping of incomplete profile -- if the
430e8d8bef9SDimitry Andric // the loop is a long running loop and dump is called in the middle
431e8d8bef9SDimitry Andric // of the loop, the result profile is incomplete.
432e8d8bef9SDimitry Andric // FIXME: add other heuristics to detect long running loops.
433e8d8bef9SDimitry Andric if (SkipRetExitBlock) {
434bdd1243dSDimitry Andric for (auto *BB : ExitBlocks)
435e8d8bef9SDimitry Andric if (isa<ReturnInst>(BB->getTerminator()))
436e8d8bef9SDimitry Andric return false;
437e8d8bef9SDimitry Andric }
438e8d8bef9SDimitry Andric
4390b57cec5SDimitry Andric unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
4400b57cec5SDimitry Andric if (MaxProm == 0)
4410b57cec5SDimitry Andric return false;
4420b57cec5SDimitry Andric
4430b57cec5SDimitry Andric unsigned Promoted = 0;
4440b57cec5SDimitry Andric for (auto &Cand : LoopToCandidates[&L]) {
4450b57cec5SDimitry Andric
4460b57cec5SDimitry Andric SmallVector<PHINode *, 4> NewPHIs;
4470b57cec5SDimitry Andric SSAUpdater SSA(&NewPHIs);
4480b57cec5SDimitry Andric Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
4490b57cec5SDimitry Andric
4500b57cec5SDimitry Andric // If BFI is set, we will use it to guide the promotions.
4510b57cec5SDimitry Andric if (BFI) {
4520b57cec5SDimitry Andric auto *BB = Cand.first->getParent();
4530b57cec5SDimitry Andric auto InstrCount = BFI->getBlockProfileCount(BB);
4540b57cec5SDimitry Andric if (!InstrCount)
4550b57cec5SDimitry Andric continue;
4560b57cec5SDimitry Andric auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
4570b57cec5SDimitry Andric // If the average loop trip count is not greater than 1.5, we skip
4580b57cec5SDimitry Andric // promotion.
45981ad6265SDimitry Andric if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
4600b57cec5SDimitry Andric continue;
4610b57cec5SDimitry Andric }
4620b57cec5SDimitry Andric
4630b57cec5SDimitry Andric PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
4640b57cec5SDimitry Andric L.getLoopPreheader(), ExitBlocks,
4650b57cec5SDimitry Andric InsertPts, LoopToCandidates, LI);
4660b57cec5SDimitry Andric Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
4670b57cec5SDimitry Andric Promoted++;
4680b57cec5SDimitry Andric if (Promoted >= MaxProm)
4690b57cec5SDimitry Andric break;
4700b57cec5SDimitry Andric
4710b57cec5SDimitry Andric (*NumPromoted)++;
4720b57cec5SDimitry Andric if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
4730b57cec5SDimitry Andric break;
4740b57cec5SDimitry Andric }
4750b57cec5SDimitry Andric
4760b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
4770b57cec5SDimitry Andric << L.getLoopDepth() << ")\n");
4780b57cec5SDimitry Andric return Promoted != 0;
4790b57cec5SDimitry Andric }
4800b57cec5SDimitry Andric
4810b57cec5SDimitry Andric private:
allowSpeculativeCounterPromotion(Loop * LP)4820b57cec5SDimitry Andric bool allowSpeculativeCounterPromotion(Loop *LP) {
4830b57cec5SDimitry Andric SmallVector<BasicBlock *, 8> ExitingBlocks;
4840b57cec5SDimitry Andric L.getExitingBlocks(ExitingBlocks);
4850b57cec5SDimitry Andric // Not considierered speculative.
4860b57cec5SDimitry Andric if (ExitingBlocks.size() == 1)
4870b57cec5SDimitry Andric return true;
4880b57cec5SDimitry Andric if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
4890b57cec5SDimitry Andric return false;
4900b57cec5SDimitry Andric return true;
4910b57cec5SDimitry Andric }
4920b57cec5SDimitry Andric
4935ffd83dbSDimitry Andric // Check whether the loop satisfies the basic conditions needed to perform
4945ffd83dbSDimitry Andric // Counter Promotions.
4954824e7fdSDimitry Andric bool
isPromotionPossible(Loop * LP,const SmallVectorImpl<BasicBlock * > & LoopExitBlocks)4964824e7fdSDimitry Andric isPromotionPossible(Loop *LP,
4975ffd83dbSDimitry Andric const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
4980b57cec5SDimitry Andric // We can't insert into a catchswitch.
4990b57cec5SDimitry Andric if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
5000b57cec5SDimitry Andric return isa<CatchSwitchInst>(Exit->getTerminator());
5010b57cec5SDimitry Andric }))
5025ffd83dbSDimitry Andric return false;
5030b57cec5SDimitry Andric
5040b57cec5SDimitry Andric if (!LP->hasDedicatedExits())
5055ffd83dbSDimitry Andric return false;
5060b57cec5SDimitry Andric
5070b57cec5SDimitry Andric BasicBlock *PH = LP->getLoopPreheader();
5080b57cec5SDimitry Andric if (!PH)
5095ffd83dbSDimitry Andric return false;
5105ffd83dbSDimitry Andric
5115ffd83dbSDimitry Andric return true;
5125ffd83dbSDimitry Andric }
5135ffd83dbSDimitry Andric
5145ffd83dbSDimitry Andric // Returns the max number of Counter Promotions for LP.
getMaxNumOfPromotionsInLoop(Loop * LP)5155ffd83dbSDimitry Andric unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
5165ffd83dbSDimitry Andric SmallVector<BasicBlock *, 8> LoopExitBlocks;
5175ffd83dbSDimitry Andric LP->getExitBlocks(LoopExitBlocks);
5185ffd83dbSDimitry Andric if (!isPromotionPossible(LP, LoopExitBlocks))
5190b57cec5SDimitry Andric return 0;
5200b57cec5SDimitry Andric
5210b57cec5SDimitry Andric SmallVector<BasicBlock *, 8> ExitingBlocks;
5220b57cec5SDimitry Andric LP->getExitingBlocks(ExitingBlocks);
5230b57cec5SDimitry Andric
5240b57cec5SDimitry Andric // If BFI is set, we do more aggressive promotions based on BFI.
5250b57cec5SDimitry Andric if (BFI)
5260b57cec5SDimitry Andric return (unsigned)-1;
5270b57cec5SDimitry Andric
5280b57cec5SDimitry Andric // Not considierered speculative.
5290b57cec5SDimitry Andric if (ExitingBlocks.size() == 1)
5300b57cec5SDimitry Andric return MaxNumOfPromotionsPerLoop;
5310b57cec5SDimitry Andric
5320b57cec5SDimitry Andric if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
5330b57cec5SDimitry Andric return 0;
5340b57cec5SDimitry Andric
5350b57cec5SDimitry Andric // Whether the target block is in a loop does not matter:
5360b57cec5SDimitry Andric if (SpeculativeCounterPromotionToLoop)
5370b57cec5SDimitry Andric return MaxNumOfPromotionsPerLoop;
5380b57cec5SDimitry Andric
5390b57cec5SDimitry Andric // Now check the target block:
5400b57cec5SDimitry Andric unsigned MaxProm = MaxNumOfPromotionsPerLoop;
5410b57cec5SDimitry Andric for (auto *TargetBlock : LoopExitBlocks) {
5420b57cec5SDimitry Andric auto *TargetLoop = LI.getLoopFor(TargetBlock);
5430b57cec5SDimitry Andric if (!TargetLoop)
5440b57cec5SDimitry Andric continue;
5450b57cec5SDimitry Andric unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
5460b57cec5SDimitry Andric unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
5470b57cec5SDimitry Andric MaxProm =
5480b57cec5SDimitry Andric std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
5490b57cec5SDimitry Andric PendingCandsInTarget);
5500b57cec5SDimitry Andric }
5510b57cec5SDimitry Andric return MaxProm;
5520b57cec5SDimitry Andric }
5530b57cec5SDimitry Andric
5540b57cec5SDimitry Andric DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
5550b57cec5SDimitry Andric SmallVector<BasicBlock *, 8> ExitBlocks;
5560b57cec5SDimitry Andric SmallVector<Instruction *, 8> InsertPts;
5570b57cec5SDimitry Andric Loop &L;
5580b57cec5SDimitry Andric LoopInfo &LI;
5590b57cec5SDimitry Andric BlockFrequencyInfo *BFI;
5600b57cec5SDimitry Andric };
5610b57cec5SDimitry Andric
562e8d8bef9SDimitry Andric enum class ValueProfilingCallType {
563e8d8bef9SDimitry Andric // Individual values are tracked. Currently used for indiret call target
564e8d8bef9SDimitry Andric // profiling.
565e8d8bef9SDimitry Andric Default,
566e8d8bef9SDimitry Andric
567e8d8bef9SDimitry Andric // MemOp: the memop size value profiling.
568e8d8bef9SDimitry Andric MemOp
569e8d8bef9SDimitry Andric };
570e8d8bef9SDimitry Andric
5710b57cec5SDimitry Andric } // end anonymous namespace
5720b57cec5SDimitry Andric
run(Module & M,ModuleAnalysisManager & AM)5735f757f3fSDimitry Andric PreservedAnalyses InstrProfilingLoweringPass::run(Module &M,
5745f757f3fSDimitry Andric ModuleAnalysisManager &AM) {
5758bcb0991SDimitry Andric FunctionAnalysisManager &FAM =
5768bcb0991SDimitry Andric AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
5778bcb0991SDimitry Andric auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
5788bcb0991SDimitry Andric return FAM.getResult<TargetLibraryAnalysis>(F);
5798bcb0991SDimitry Andric };
5805f757f3fSDimitry Andric InstrLowerer Lowerer(M, Options, GetTLI, IsCS);
5815f757f3fSDimitry Andric if (!Lowerer.lower())
5820b57cec5SDimitry Andric return PreservedAnalyses::all();
5830b57cec5SDimitry Andric
5840b57cec5SDimitry Andric return PreservedAnalyses::none();
5850b57cec5SDimitry Andric }
5860b57cec5SDimitry Andric
lowerIntrinsics(Function * F)5875f757f3fSDimitry Andric bool InstrLowerer::lowerIntrinsics(Function *F) {
5880b57cec5SDimitry Andric bool MadeChange = false;
5890b57cec5SDimitry Andric PromotionCandidates.clear();
5900b57cec5SDimitry Andric for (BasicBlock &BB : *F) {
591349cc55cSDimitry Andric for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
59204eeddc0SDimitry Andric if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
59304eeddc0SDimitry Andric lowerIncrement(IPIS);
5940b57cec5SDimitry Andric MadeChange = true;
59504eeddc0SDimitry Andric } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
59604eeddc0SDimitry Andric lowerIncrement(IPI);
59704eeddc0SDimitry Andric MadeChange = true;
59806c3fb27SDimitry Andric } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(&Instr)) {
59906c3fb27SDimitry Andric lowerTimestamp(IPC);
60006c3fb27SDimitry Andric MadeChange = true;
6011fd87a68SDimitry Andric } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
6021fd87a68SDimitry Andric lowerCover(IPC);
6031fd87a68SDimitry Andric MadeChange = true;
60404eeddc0SDimitry Andric } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
60504eeddc0SDimitry Andric lowerValueProfileInst(IPVP);
6060b57cec5SDimitry Andric MadeChange = true;
6075f757f3fSDimitry Andric } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(&Instr)) {
6085f757f3fSDimitry Andric IPMP->eraseFromParent();
6095f757f3fSDimitry Andric MadeChange = true;
6105f757f3fSDimitry Andric } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(&Instr)) {
6115f757f3fSDimitry Andric lowerMCDCTestVectorBitmapUpdate(IPBU);
6125f757f3fSDimitry Andric MadeChange = true;
6135f757f3fSDimitry Andric } else if (auto *IPTU = dyn_cast<InstrProfMCDCCondBitmapUpdate>(&Instr)) {
6145f757f3fSDimitry Andric lowerMCDCCondBitmapUpdate(IPTU);
6155f757f3fSDimitry Andric MadeChange = true;
6160b57cec5SDimitry Andric }
6170b57cec5SDimitry Andric }
6180b57cec5SDimitry Andric }
6190b57cec5SDimitry Andric
6200b57cec5SDimitry Andric if (!MadeChange)
6210b57cec5SDimitry Andric return false;
6220b57cec5SDimitry Andric
6230b57cec5SDimitry Andric promoteCounterLoadStores(F);
6240b57cec5SDimitry Andric return true;
6250b57cec5SDimitry Andric }
6260b57cec5SDimitry Andric
isRuntimeCounterRelocationEnabled() const6275f757f3fSDimitry Andric bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {
628fe6060f1SDimitry Andric // Mach-O don't support weak external references.
629fe6060f1SDimitry Andric if (TT.isOSBinFormatMachO())
630fe6060f1SDimitry Andric return false;
631fe6060f1SDimitry Andric
6325ffd83dbSDimitry Andric if (RuntimeCounterRelocation.getNumOccurrences() > 0)
6335ffd83dbSDimitry Andric return RuntimeCounterRelocation;
6345ffd83dbSDimitry Andric
635fe6060f1SDimitry Andric // Fuchsia uses runtime counter relocation by default.
6365ffd83dbSDimitry Andric return TT.isOSFuchsia();
6375ffd83dbSDimitry Andric }
6385ffd83dbSDimitry Andric
isCounterPromotionEnabled() const6395f757f3fSDimitry Andric bool InstrLowerer::isCounterPromotionEnabled() const {
6400b57cec5SDimitry Andric if (DoCounterPromotion.getNumOccurrences() > 0)
6410b57cec5SDimitry Andric return DoCounterPromotion;
6420b57cec5SDimitry Andric
6430b57cec5SDimitry Andric return Options.DoCounterPromotion;
6440b57cec5SDimitry Andric }
6450b57cec5SDimitry Andric
promoteCounterLoadStores(Function * F)6465f757f3fSDimitry Andric void InstrLowerer::promoteCounterLoadStores(Function *F) {
6470b57cec5SDimitry Andric if (!isCounterPromotionEnabled())
6480b57cec5SDimitry Andric return;
6490b57cec5SDimitry Andric
6500b57cec5SDimitry Andric DominatorTree DT(*F);
6510b57cec5SDimitry Andric LoopInfo LI(DT);
6520b57cec5SDimitry Andric DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
6530b57cec5SDimitry Andric
6540b57cec5SDimitry Andric std::unique_ptr<BlockFrequencyInfo> BFI;
6550b57cec5SDimitry Andric if (Options.UseBFIInPromotion) {
6560b57cec5SDimitry Andric std::unique_ptr<BranchProbabilityInfo> BPI;
6578bcb0991SDimitry Andric BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
6580b57cec5SDimitry Andric BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
6590b57cec5SDimitry Andric }
6600b57cec5SDimitry Andric
6610b57cec5SDimitry Andric for (const auto &LoadStore : PromotionCandidates) {
6620b57cec5SDimitry Andric auto *CounterLoad = LoadStore.first;
6630b57cec5SDimitry Andric auto *CounterStore = LoadStore.second;
6640b57cec5SDimitry Andric BasicBlock *BB = CounterLoad->getParent();
6650b57cec5SDimitry Andric Loop *ParentLoop = LI.getLoopFor(BB);
6660b57cec5SDimitry Andric if (!ParentLoop)
6670b57cec5SDimitry Andric continue;
6680b57cec5SDimitry Andric LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
6690b57cec5SDimitry Andric }
6700b57cec5SDimitry Andric
6710b57cec5SDimitry Andric SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
6720b57cec5SDimitry Andric
6730b57cec5SDimitry Andric // Do a post-order traversal of the loops so that counter updates can be
6740b57cec5SDimitry Andric // iteratively hoisted outside the loop nest.
6750b57cec5SDimitry Andric for (auto *Loop : llvm::reverse(Loops)) {
6760b57cec5SDimitry Andric PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
6770b57cec5SDimitry Andric Promoter.run(&TotalCountersPromoted);
6780b57cec5SDimitry Andric }
6790b57cec5SDimitry Andric }
6800b57cec5SDimitry Andric
needsRuntimeHookUnconditionally(const Triple & TT)681349cc55cSDimitry Andric static bool needsRuntimeHookUnconditionally(const Triple &TT) {
682349cc55cSDimitry Andric // On Fuchsia, we only need runtime hook if any counters are present.
683349cc55cSDimitry Andric if (TT.isOSFuchsia())
684349cc55cSDimitry Andric return false;
685349cc55cSDimitry Andric
686349cc55cSDimitry Andric return true;
687349cc55cSDimitry Andric }
688349cc55cSDimitry Andric
6890b57cec5SDimitry Andric /// Check if the module contains uses of any profiling intrinsics.
containsProfilingIntrinsics(Module & M)6900b57cec5SDimitry Andric static bool containsProfilingIntrinsics(Module &M) {
69104eeddc0SDimitry Andric auto containsIntrinsic = [&](int ID) {
69204eeddc0SDimitry Andric if (auto *F = M.getFunction(Intrinsic::getName(ID)))
69304eeddc0SDimitry Andric return !F->use_empty();
6940b57cec5SDimitry Andric return false;
69504eeddc0SDimitry Andric };
6961fd87a68SDimitry Andric return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
6971fd87a68SDimitry Andric containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
69804eeddc0SDimitry Andric containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
69906c3fb27SDimitry Andric containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) ||
70004eeddc0SDimitry Andric containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
7010b57cec5SDimitry Andric }
7020b57cec5SDimitry Andric
lower()7035f757f3fSDimitry Andric bool InstrLowerer::lower() {
704349cc55cSDimitry Andric bool MadeChange = false;
705bdd1243dSDimitry Andric bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
706bdd1243dSDimitry Andric if (NeedsRuntimeHook)
707349cc55cSDimitry Andric MadeChange = emitRuntimeHook();
7080b57cec5SDimitry Andric
709bdd1243dSDimitry Andric bool ContainsProfiling = containsProfilingIntrinsics(M);
7100b57cec5SDimitry Andric GlobalVariable *CoverageNamesVar =
7110b57cec5SDimitry Andric M.getNamedGlobal(getCoverageUnusedNamesVarName());
712bdd1243dSDimitry Andric // Improve compile time by avoiding linear scans when there is no work.
713bdd1243dSDimitry Andric if (!ContainsProfiling && !CoverageNamesVar)
7140b57cec5SDimitry Andric return MadeChange;
7150b57cec5SDimitry Andric
7160b57cec5SDimitry Andric // We did not know how many value sites there would be inside
7170b57cec5SDimitry Andric // the instrumented function. This is counting the number of instrumented
7180b57cec5SDimitry Andric // target value sites to enter it as field in the profile data variable.
7190b57cec5SDimitry Andric for (Function &F : M) {
7205f757f3fSDimitry Andric InstrProfCntrInstBase *FirstProfInst = nullptr;
7215f757f3fSDimitry Andric for (BasicBlock &BB : F) {
7225f757f3fSDimitry Andric for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
7230b57cec5SDimitry Andric if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
7240b57cec5SDimitry Andric computeNumValueSiteCounts(Ind);
7255f757f3fSDimitry Andric else {
7265f757f3fSDimitry Andric if (FirstProfInst == nullptr &&
72706c3fb27SDimitry Andric (isa<InstrProfIncrementInst>(I) || isa<InstrProfCoverInst>(I)))
7285f757f3fSDimitry Andric FirstProfInst = dyn_cast<InstrProfCntrInstBase>(I);
7295f757f3fSDimitry Andric // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.
7305f757f3fSDimitry Andric if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(I))
7315f757f3fSDimitry Andric static_cast<void>(getOrCreateRegionBitmaps(Params));
7325f757f3fSDimitry Andric }
7335f757f3fSDimitry Andric }
7345f757f3fSDimitry Andric }
7350b57cec5SDimitry Andric
7365f757f3fSDimitry Andric // Use a profile intrinsic to create the region counters and data variable.
7375f757f3fSDimitry Andric // Also create the data variable based on the MCDCParams.
7385f757f3fSDimitry Andric if (FirstProfInst != nullptr) {
73906c3fb27SDimitry Andric static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
7400b57cec5SDimitry Andric }
7415f757f3fSDimitry Andric }
7420b57cec5SDimitry Andric
7430b57cec5SDimitry Andric for (Function &F : M)
7440b57cec5SDimitry Andric MadeChange |= lowerIntrinsics(&F);
7450b57cec5SDimitry Andric
7460b57cec5SDimitry Andric if (CoverageNamesVar) {
7470b57cec5SDimitry Andric lowerCoverageData(CoverageNamesVar);
7480b57cec5SDimitry Andric MadeChange = true;
7490b57cec5SDimitry Andric }
7500b57cec5SDimitry Andric
7510b57cec5SDimitry Andric if (!MadeChange)
7520b57cec5SDimitry Andric return false;
7530b57cec5SDimitry Andric
7540b57cec5SDimitry Andric emitVNodes();
7550b57cec5SDimitry Andric emitNameData();
756bdd1243dSDimitry Andric
757bdd1243dSDimitry Andric // Emit runtime hook for the cases where the target does not unconditionally
758bdd1243dSDimitry Andric // require pulling in profile runtime, and coverage is enabled on code that is
759bdd1243dSDimitry Andric // not eliminated by the front-end, e.g. unused functions with internal
760bdd1243dSDimitry Andric // linkage.
761bdd1243dSDimitry Andric if (!NeedsRuntimeHook && ContainsProfiling)
762349cc55cSDimitry Andric emitRuntimeHook();
763bdd1243dSDimitry Andric
7640b57cec5SDimitry Andric emitRegistration();
7650b57cec5SDimitry Andric emitUses();
7660b57cec5SDimitry Andric emitInitialization();
7670b57cec5SDimitry Andric return true;
7680b57cec5SDimitry Andric }
7690b57cec5SDimitry Andric
getOrInsertValueProfilingCall(Module & M,const TargetLibraryInfo & TLI,ValueProfilingCallType CallType=ValueProfilingCallType::Default)770e8d8bef9SDimitry Andric static FunctionCallee getOrInsertValueProfilingCall(
771e8d8bef9SDimitry Andric Module &M, const TargetLibraryInfo &TLI,
772e8d8bef9SDimitry Andric ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
7730b57cec5SDimitry Andric LLVMContext &Ctx = M.getContext();
7740b57cec5SDimitry Andric auto *ReturnTy = Type::getVoidTy(M.getContext());
7750b57cec5SDimitry Andric
7760b57cec5SDimitry Andric AttributeList AL;
7770b57cec5SDimitry Andric if (auto AK = TLI.getExtAttrForI32Param(false))
7780b57cec5SDimitry Andric AL = AL.addParamAttribute(M.getContext(), 2, AK);
7790b57cec5SDimitry Andric
780e8d8bef9SDimitry Andric assert((CallType == ValueProfilingCallType::Default ||
781e8d8bef9SDimitry Andric CallType == ValueProfilingCallType::MemOp) &&
782e8d8bef9SDimitry Andric "Must be Default or MemOp");
7830b57cec5SDimitry Andric Type *ParamTypes[] = {
7840b57cec5SDimitry Andric #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
7850b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfData.inc"
7860b57cec5SDimitry Andric };
7870b57cec5SDimitry Andric auto *ValueProfilingCallTy =
788bdd1243dSDimitry Andric FunctionType::get(ReturnTy, ArrayRef(ParamTypes), false);
789e8d8bef9SDimitry Andric StringRef FuncName = CallType == ValueProfilingCallType::Default
790e8d8bef9SDimitry Andric ? getInstrProfValueProfFuncName()
791e8d8bef9SDimitry Andric : getInstrProfValueProfMemOpFuncName();
792e8d8bef9SDimitry Andric return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
7930b57cec5SDimitry Andric }
7940b57cec5SDimitry Andric
computeNumValueSiteCounts(InstrProfValueProfileInst * Ind)7955f757f3fSDimitry Andric void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
7960b57cec5SDimitry Andric GlobalVariable *Name = Ind->getName();
7970b57cec5SDimitry Andric uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
7980b57cec5SDimitry Andric uint64_t Index = Ind->getIndex()->getZExtValue();
7994824e7fdSDimitry Andric auto &PD = ProfileDataMap[Name];
8004824e7fdSDimitry Andric PD.NumValueSites[ValueKind] =
8014824e7fdSDimitry Andric std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));
8020b57cec5SDimitry Andric }
8030b57cec5SDimitry Andric
lowerValueProfileInst(InstrProfValueProfileInst * Ind)8045f757f3fSDimitry Andric void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
8050eae32dcSDimitry Andric // TODO: Value profiling heavily depends on the data section which is omitted
8060eae32dcSDimitry Andric // in lightweight mode. We need to move the value profile pointer to the
8070eae32dcSDimitry Andric // Counter struct to get this working.
8080eae32dcSDimitry Andric assert(
8095f757f3fSDimitry Andric !DebugInfoCorrelate && ProfileCorrelate == InstrProfCorrelator::NONE &&
8100eae32dcSDimitry Andric "Value profiling is not yet supported with lightweight instrumentation");
8110b57cec5SDimitry Andric GlobalVariable *Name = Ind->getName();
8120b57cec5SDimitry Andric auto It = ProfileDataMap.find(Name);
8130b57cec5SDimitry Andric assert(It != ProfileDataMap.end() && It->second.DataVar &&
8140b57cec5SDimitry Andric "value profiling detected in function with no counter incerement");
8150b57cec5SDimitry Andric
8160b57cec5SDimitry Andric GlobalVariable *DataVar = It->second.DataVar;
8170b57cec5SDimitry Andric uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
8180b57cec5SDimitry Andric uint64_t Index = Ind->getIndex()->getZExtValue();
8190b57cec5SDimitry Andric for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
8200b57cec5SDimitry Andric Index += It->second.NumValueSites[Kind];
8210b57cec5SDimitry Andric
8220b57cec5SDimitry Andric IRBuilder<> Builder(Ind);
823e8d8bef9SDimitry Andric bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
8240b57cec5SDimitry Andric llvm::InstrProfValueKind::IPVK_MemOPSize);
8250b57cec5SDimitry Andric CallInst *Call = nullptr;
8268bcb0991SDimitry Andric auto *TLI = &GetTLI(*Ind->getFunction());
8275ffd83dbSDimitry Andric
8285ffd83dbSDimitry Andric // To support value profiling calls within Windows exception handlers, funclet
8295ffd83dbSDimitry Andric // information contained within operand bundles needs to be copied over to
8305ffd83dbSDimitry Andric // the library call. This is required for the IR to be processed by the
8315ffd83dbSDimitry Andric // WinEHPrepare pass.
8325ffd83dbSDimitry Andric SmallVector<OperandBundleDef, 1> OpBundles;
8335ffd83dbSDimitry Andric Ind->getOperandBundlesAsDefs(OpBundles);
834e8d8bef9SDimitry Andric if (!IsMemOpSize) {
8355f757f3fSDimitry Andric Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)};
8365f757f3fSDimitry Andric Call = Builder.CreateCall(getOrInsertValueProfilingCall(M, *TLI), Args,
8375ffd83dbSDimitry Andric OpBundles);
8380b57cec5SDimitry Andric } else {
8395f757f3fSDimitry Andric Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)};
840e8d8bef9SDimitry Andric Call = Builder.CreateCall(
8415f757f3fSDimitry Andric getOrInsertValueProfilingCall(M, *TLI, ValueProfilingCallType::MemOp),
8425ffd83dbSDimitry Andric Args, OpBundles);
8430b57cec5SDimitry Andric }
8440b57cec5SDimitry Andric if (auto AK = TLI->getExtAttrForI32Param(false))
8450b57cec5SDimitry Andric Call->addParamAttr(2, AK);
8460b57cec5SDimitry Andric Ind->replaceAllUsesWith(Call);
8470b57cec5SDimitry Andric Ind->eraseFromParent();
8480b57cec5SDimitry Andric }
8490b57cec5SDimitry Andric
getCounterAddress(InstrProfCntrInstBase * I)8505f757f3fSDimitry Andric Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
8511fd87a68SDimitry Andric auto *Counters = getOrCreateRegionCounters(I);
8521fd87a68SDimitry Andric IRBuilder<> Builder(I);
8530b57cec5SDimitry Andric
85406c3fb27SDimitry Andric if (isa<InstrProfTimestampInst>(I))
85506c3fb27SDimitry Andric Counters->setAlignment(Align(8));
85606c3fb27SDimitry Andric
8571fd87a68SDimitry Andric auto *Addr = Builder.CreateConstInBoundsGEP2_32(
8581fd87a68SDimitry Andric Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());
8590b57cec5SDimitry Andric
8601fd87a68SDimitry Andric if (!isRuntimeCounterRelocationEnabled())
8611fd87a68SDimitry Andric return Addr;
8621fd87a68SDimitry Andric
8635f757f3fSDimitry Andric Type *Int64Ty = Type::getInt64Ty(M.getContext());
8641fd87a68SDimitry Andric Function *Fn = I->getParent()->getParent();
86581ad6265SDimitry Andric LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
86681ad6265SDimitry Andric if (!BiasLI) {
86781ad6265SDimitry Andric IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
8685f757f3fSDimitry Andric auto *Bias = M.getGlobalVariable(getInstrProfCounterBiasVarName());
8695ffd83dbSDimitry Andric if (!Bias) {
870fe6060f1SDimitry Andric // Compiler must define this variable when runtime counter relocation
871fe6060f1SDimitry Andric // is being used. Runtime has a weak external reference that is used
872fe6060f1SDimitry Andric // to check whether that's the case or not.
8734824e7fdSDimitry Andric Bias = new GlobalVariable(
8745f757f3fSDimitry Andric M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
8754824e7fdSDimitry Andric Constant::getNullValue(Int64Ty), getInstrProfCounterBiasVarName());
8765ffd83dbSDimitry Andric Bias->setVisibility(GlobalVariable::HiddenVisibility);
877fe6060f1SDimitry Andric // A definition that's weak (linkonce_odr) without being in a COMDAT
878fe6060f1SDimitry Andric // section wouldn't lead to link errors, but it would lead to a dead
879fe6060f1SDimitry Andric // data word from every TU but one. Putting it in COMDAT ensures there
880fe6060f1SDimitry Andric // will be exactly one data slot in the link.
881fe6060f1SDimitry Andric if (TT.supportsCOMDAT())
8825f757f3fSDimitry Andric Bias->setComdat(M.getOrInsertComdat(Bias->getName()));
8835ffd83dbSDimitry Andric }
88481ad6265SDimitry Andric BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
8855ffd83dbSDimitry Andric }
88681ad6265SDimitry Andric auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
8871fd87a68SDimitry Andric return Builder.CreateIntToPtr(Add, Addr->getType());
8885ffd83dbSDimitry Andric }
8895ffd83dbSDimitry Andric
getBitmapAddress(InstrProfMCDCTVBitmapUpdate * I)8905f757f3fSDimitry Andric Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
8915f757f3fSDimitry Andric auto *Bitmaps = getOrCreateRegionBitmaps(I);
8925f757f3fSDimitry Andric IRBuilder<> Builder(I);
8935f757f3fSDimitry Andric
8945f757f3fSDimitry Andric auto *Addr = Builder.CreateConstInBoundsGEP2_32(
8955f757f3fSDimitry Andric Bitmaps->getValueType(), Bitmaps, 0, I->getBitmapIndex()->getZExtValue());
8965f757f3fSDimitry Andric
8975f757f3fSDimitry Andric if (isRuntimeCounterRelocationEnabled()) {
8985f757f3fSDimitry Andric LLVMContext &Ctx = M.getContext();
8995f757f3fSDimitry Andric Ctx.diagnose(DiagnosticInfoPGOProfile(
9005f757f3fSDimitry Andric M.getName().data(),
9015f757f3fSDimitry Andric Twine("Runtime counter relocation is presently not supported for MC/DC "
9025f757f3fSDimitry Andric "bitmaps."),
9035f757f3fSDimitry Andric DS_Warning));
9045f757f3fSDimitry Andric }
9055f757f3fSDimitry Andric
9065f757f3fSDimitry Andric return Addr;
9075f757f3fSDimitry Andric }
9085f757f3fSDimitry Andric
lowerCover(InstrProfCoverInst * CoverInstruction)9095f757f3fSDimitry Andric void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
9101fd87a68SDimitry Andric auto *Addr = getCounterAddress(CoverInstruction);
9111fd87a68SDimitry Andric IRBuilder<> Builder(CoverInstruction);
9121fd87a68SDimitry Andric // We store zero to represent that this block is covered.
9131fd87a68SDimitry Andric Builder.CreateStore(Builder.getInt8(0), Addr);
9141fd87a68SDimitry Andric CoverInstruction->eraseFromParent();
9151fd87a68SDimitry Andric }
9161fd87a68SDimitry Andric
lowerTimestamp(InstrProfTimestampInst * TimestampInstruction)9175f757f3fSDimitry Andric void InstrLowerer::lowerTimestamp(
91806c3fb27SDimitry Andric InstrProfTimestampInst *TimestampInstruction) {
91906c3fb27SDimitry Andric assert(TimestampInstruction->getIndex()->isZeroValue() &&
92006c3fb27SDimitry Andric "timestamp probes are always the first probe for a function");
9215f757f3fSDimitry Andric auto &Ctx = M.getContext();
92206c3fb27SDimitry Andric auto *TimestampAddr = getCounterAddress(TimestampInstruction);
92306c3fb27SDimitry Andric IRBuilder<> Builder(TimestampInstruction);
92406c3fb27SDimitry Andric auto *CalleeTy =
92506c3fb27SDimitry Andric FunctionType::get(Type::getVoidTy(Ctx), TimestampAddr->getType(), false);
9265f757f3fSDimitry Andric auto Callee = M.getOrInsertFunction(
92706c3fb27SDimitry Andric INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy);
92806c3fb27SDimitry Andric Builder.CreateCall(Callee, {TimestampAddr});
92906c3fb27SDimitry Andric TimestampInstruction->eraseFromParent();
93006c3fb27SDimitry Andric }
93106c3fb27SDimitry Andric
lowerIncrement(InstrProfIncrementInst * Inc)9325f757f3fSDimitry Andric void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
9331fd87a68SDimitry Andric auto *Addr = getCounterAddress(Inc);
9341fd87a68SDimitry Andric
9351fd87a68SDimitry Andric IRBuilder<> Builder(Inc);
9365ffd83dbSDimitry Andric if (Options.Atomic || AtomicCounterUpdateAll ||
9371fd87a68SDimitry Andric (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
9380b57cec5SDimitry Andric Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
939fe6060f1SDimitry Andric MaybeAlign(), AtomicOrdering::Monotonic);
9400b57cec5SDimitry Andric } else {
9410b57cec5SDimitry Andric Value *IncStep = Inc->getStep();
9420b57cec5SDimitry Andric Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
9430b57cec5SDimitry Andric auto *Count = Builder.CreateAdd(Load, Inc->getStep());
9440b57cec5SDimitry Andric auto *Store = Builder.CreateStore(Count, Addr);
9450b57cec5SDimitry Andric if (isCounterPromotionEnabled())
9460b57cec5SDimitry Andric PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
9470b57cec5SDimitry Andric }
9480b57cec5SDimitry Andric Inc->eraseFromParent();
9490b57cec5SDimitry Andric }
9500b57cec5SDimitry Andric
lowerCoverageData(GlobalVariable * CoverageNamesVar)9515f757f3fSDimitry Andric void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
9520b57cec5SDimitry Andric ConstantArray *Names =
9530b57cec5SDimitry Andric cast<ConstantArray>(CoverageNamesVar->getInitializer());
9540b57cec5SDimitry Andric for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
9550b57cec5SDimitry Andric Constant *NC = Names->getOperand(I);
9560b57cec5SDimitry Andric Value *V = NC->stripPointerCasts();
9570b57cec5SDimitry Andric assert(isa<GlobalVariable>(V) && "Missing reference to function name");
9580b57cec5SDimitry Andric GlobalVariable *Name = cast<GlobalVariable>(V);
9590b57cec5SDimitry Andric
9600b57cec5SDimitry Andric Name->setLinkage(GlobalValue::PrivateLinkage);
9610b57cec5SDimitry Andric ReferencedNames.push_back(Name);
96281ad6265SDimitry Andric if (isa<ConstantExpr>(NC))
9630b57cec5SDimitry Andric NC->dropAllReferences();
9640b57cec5SDimitry Andric }
9650b57cec5SDimitry Andric CoverageNamesVar->eraseFromParent();
9660b57cec5SDimitry Andric }
9670b57cec5SDimitry Andric
lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate * Update)9685f757f3fSDimitry Andric void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
9695f757f3fSDimitry Andric InstrProfMCDCTVBitmapUpdate *Update) {
9705f757f3fSDimitry Andric IRBuilder<> Builder(Update);
9715f757f3fSDimitry Andric auto *Int8Ty = Type::getInt8Ty(M.getContext());
9725f757f3fSDimitry Andric auto *Int8PtrTy = PointerType::getUnqual(M.getContext());
9735f757f3fSDimitry Andric auto *Int32Ty = Type::getInt32Ty(M.getContext());
9745f757f3fSDimitry Andric auto *Int64Ty = Type::getInt64Ty(M.getContext());
9755f757f3fSDimitry Andric auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
9765f757f3fSDimitry Andric auto *BitmapAddr = getBitmapAddress(Update);
9775f757f3fSDimitry Andric
9785f757f3fSDimitry Andric // Load Temp Val.
9795f757f3fSDimitry Andric // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
9805f757f3fSDimitry Andric auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp");
9815f757f3fSDimitry Andric
9825f757f3fSDimitry Andric // Calculate byte offset using div8.
9835f757f3fSDimitry Andric // %1 = lshr i32 %mcdc.temp, 3
9845f757f3fSDimitry Andric auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);
9855f757f3fSDimitry Andric
9865f757f3fSDimitry Andric // Add byte offset to section base byte address.
9875f757f3fSDimitry Andric // %2 = zext i32 %1 to i64
9885f757f3fSDimitry Andric // %3 = add i64 ptrtoint (ptr @__profbm_test to i64), %2
9895f757f3fSDimitry Andric auto *BitmapByteAddr =
9905f757f3fSDimitry Andric Builder.CreateAdd(Builder.CreatePtrToInt(BitmapAddr, Int64Ty),
9915f757f3fSDimitry Andric Builder.CreateZExtOrBitCast(BitmapByteOffset, Int64Ty));
9925f757f3fSDimitry Andric
9935f757f3fSDimitry Andric // Convert to a pointer.
9945f757f3fSDimitry Andric // %4 = inttoptr i32 %3 to ptr
9955f757f3fSDimitry Andric BitmapByteAddr = Builder.CreateIntToPtr(BitmapByteAddr, Int8PtrTy);
9965f757f3fSDimitry Andric
9975f757f3fSDimitry Andric // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)
9985f757f3fSDimitry Andric // %5 = and i32 %mcdc.temp, 7
9995f757f3fSDimitry Andric // %6 = trunc i32 %5 to i8
10005f757f3fSDimitry Andric auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty);
10015f757f3fSDimitry Andric
10025f757f3fSDimitry Andric // Shift bit offset left to form a bitmap.
10035f757f3fSDimitry Andric // %7 = shl i8 1, %6
10045f757f3fSDimitry Andric auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);
10055f757f3fSDimitry Andric
10065f757f3fSDimitry Andric // Load profile bitmap byte.
10075f757f3fSDimitry Andric // %mcdc.bits = load i8, ptr %4, align 1
10085f757f3fSDimitry Andric auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits");
10095f757f3fSDimitry Andric
10105f757f3fSDimitry Andric // Perform logical OR of profile bitmap byte and shifted bit offset.
10115f757f3fSDimitry Andric // %8 = or i8 %mcdc.bits, %7
10125f757f3fSDimitry Andric auto *Result = Builder.CreateOr(Bitmap, ShiftedVal);
10135f757f3fSDimitry Andric
10145f757f3fSDimitry Andric // Store the updated profile bitmap byte.
10155f757f3fSDimitry Andric // store i8 %8, ptr %3, align 1
10165f757f3fSDimitry Andric Builder.CreateStore(Result, BitmapByteAddr);
10175f757f3fSDimitry Andric Update->eraseFromParent();
10185f757f3fSDimitry Andric }
10195f757f3fSDimitry Andric
lowerMCDCCondBitmapUpdate(InstrProfMCDCCondBitmapUpdate * Update)10205f757f3fSDimitry Andric void InstrLowerer::lowerMCDCCondBitmapUpdate(
10215f757f3fSDimitry Andric InstrProfMCDCCondBitmapUpdate *Update) {
10225f757f3fSDimitry Andric IRBuilder<> Builder(Update);
10235f757f3fSDimitry Andric auto *Int32Ty = Type::getInt32Ty(M.getContext());
10245f757f3fSDimitry Andric auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
10255f757f3fSDimitry Andric
10265f757f3fSDimitry Andric // Load the MCDC temporary value from the stack.
10275f757f3fSDimitry Andric // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
10285f757f3fSDimitry Andric auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp");
10295f757f3fSDimitry Andric
10305f757f3fSDimitry Andric // Zero-extend the evaluated condition boolean value (0 or 1) by 32bits.
10315f757f3fSDimitry Andric // %1 = zext i1 %tobool to i32
10325f757f3fSDimitry Andric auto *CondV_32 = Builder.CreateZExt(Update->getCondBool(), Int32Ty);
10335f757f3fSDimitry Andric
10345f757f3fSDimitry Andric // Shift the boolean value left (by the condition's ID) to form a bitmap.
10355f757f3fSDimitry Andric // %2 = shl i32 %1, <Update->getCondID()>
10365f757f3fSDimitry Andric auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->getCondID());
10375f757f3fSDimitry Andric
10385f757f3fSDimitry Andric // Perform logical OR of the bitmap against the loaded MCDC temporary value.
10395f757f3fSDimitry Andric // %3 = or i32 %mcdc.temp, %2
10405f757f3fSDimitry Andric auto *Result = Builder.CreateOr(Temp, ShiftedVal);
10415f757f3fSDimitry Andric
10425f757f3fSDimitry Andric // Store the updated temporary value back to the stack.
10435f757f3fSDimitry Andric // store i32 %3, ptr %mcdc.addr, align 4
10445f757f3fSDimitry Andric Builder.CreateStore(Result, MCDCCondBitmapAddr);
10455f757f3fSDimitry Andric Update->eraseFromParent();
10465f757f3fSDimitry Andric }
10475f757f3fSDimitry Andric
10480b57cec5SDimitry Andric /// Get the name of a profiling variable for a particular function.
getVarName(InstrProfInstBase * Inc,StringRef Prefix,bool & Renamed)104904eeddc0SDimitry Andric static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
1050349cc55cSDimitry Andric bool &Renamed) {
10510b57cec5SDimitry Andric StringRef NamePrefix = getInstrProfNameVarPrefix();
10520b57cec5SDimitry Andric StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
10530b57cec5SDimitry Andric Function *F = Inc->getParent()->getParent();
10540b57cec5SDimitry Andric Module *M = F->getParent();
10550b57cec5SDimitry Andric if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
1056349cc55cSDimitry Andric !canRenameComdatFunc(*F)) {
1057349cc55cSDimitry Andric Renamed = false;
10580b57cec5SDimitry Andric return (Prefix + Name).str();
1059349cc55cSDimitry Andric }
1060349cc55cSDimitry Andric Renamed = true;
10610b57cec5SDimitry Andric uint64_t FuncHash = Inc->getHash()->getZExtValue();
10620b57cec5SDimitry Andric SmallVector<char, 24> HashPostfix;
10635f757f3fSDimitry Andric if (Name.ends_with((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
10640b57cec5SDimitry Andric return (Prefix + Name).str();
10650b57cec5SDimitry Andric return (Prefix + Name + "." + Twine(FuncHash)).str();
10660b57cec5SDimitry Andric }
10670b57cec5SDimitry Andric
getIntModuleFlagOrZero(const Module & M,StringRef Flag)1068fe6060f1SDimitry Andric static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
1069fe6060f1SDimitry Andric auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
1070fe6060f1SDimitry Andric if (!MD)
1071fe6060f1SDimitry Andric return 0;
1072fe6060f1SDimitry Andric
1073fe6060f1SDimitry Andric // If the flag is a ConstantAsMetadata, it should be an integer representable
1074fe6060f1SDimitry Andric // in 64-bits.
1075fe6060f1SDimitry Andric return cast<ConstantInt>(MD->getValue())->getZExtValue();
1076fe6060f1SDimitry Andric }
1077fe6060f1SDimitry Andric
enablesValueProfiling(const Module & M)1078fe6060f1SDimitry Andric static bool enablesValueProfiling(const Module &M) {
1079fe6060f1SDimitry Andric return isIRPGOFlagSet(&M) ||
1080fe6060f1SDimitry Andric getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
1081fe6060f1SDimitry Andric }
1082fe6060f1SDimitry Andric
1083fe6060f1SDimitry Andric // Conservatively returns true if data variables may be referenced by code.
profDataReferencedByCode(const Module & M)1084fe6060f1SDimitry Andric static bool profDataReferencedByCode(const Module &M) {
1085fe6060f1SDimitry Andric return enablesValueProfiling(M);
1086fe6060f1SDimitry Andric }
1087fe6060f1SDimitry Andric
shouldRecordFunctionAddr(Function * F)10880b57cec5SDimitry Andric static inline bool shouldRecordFunctionAddr(Function *F) {
1089fe6060f1SDimitry Andric // Only record function addresses if IR PGO is enabled or if clang value
1090fe6060f1SDimitry Andric // profiling is enabled. Recording function addresses greatly increases object
1091fe6060f1SDimitry Andric // file size, because it prevents the inliner from deleting functions that
1092fe6060f1SDimitry Andric // have been inlined everywhere.
1093fe6060f1SDimitry Andric if (!profDataReferencedByCode(*F->getParent()))
1094fe6060f1SDimitry Andric return false;
1095fe6060f1SDimitry Andric
10960b57cec5SDimitry Andric // Check the linkage
10970b57cec5SDimitry Andric bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
10980b57cec5SDimitry Andric if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
10990b57cec5SDimitry Andric !HasAvailableExternallyLinkage)
11000b57cec5SDimitry Andric return true;
11010b57cec5SDimitry Andric
11020b57cec5SDimitry Andric // A function marked 'alwaysinline' with available_externally linkage can't
11030b57cec5SDimitry Andric // have its address taken. Doing so would create an undefined external ref to
11040b57cec5SDimitry Andric // the function, which would fail to link.
11050b57cec5SDimitry Andric if (HasAvailableExternallyLinkage &&
11060b57cec5SDimitry Andric F->hasFnAttribute(Attribute::AlwaysInline))
11070b57cec5SDimitry Andric return false;
11080b57cec5SDimitry Andric
11090b57cec5SDimitry Andric // Prohibit function address recording if the function is both internal and
11100b57cec5SDimitry Andric // COMDAT. This avoids the profile data variable referencing internal symbols
11110b57cec5SDimitry Andric // in COMDAT.
11120b57cec5SDimitry Andric if (F->hasLocalLinkage() && F->hasComdat())
11130b57cec5SDimitry Andric return false;
11140b57cec5SDimitry Andric
11150b57cec5SDimitry Andric // Check uses of this function for other than direct calls or invokes to it.
11160b57cec5SDimitry Andric // Inline virtual functions have linkeOnceODR linkage. When a key method
11170b57cec5SDimitry Andric // exists, the vtable will only be emitted in the TU where the key method
11180b57cec5SDimitry Andric // is defined. In a TU where vtable is not available, the function won't
11190b57cec5SDimitry Andric // be 'addresstaken'. If its address is not recorded here, the profile data
11200b57cec5SDimitry Andric // with missing address may be picked by the linker leading to missing
11210b57cec5SDimitry Andric // indirect call target info.
11220b57cec5SDimitry Andric return F->hasAddressTaken() || F->hasLinkOnceLinkage();
11230b57cec5SDimitry Andric }
11240b57cec5SDimitry Andric
shouldUsePublicSymbol(Function * Fn)112506c3fb27SDimitry Andric static inline bool shouldUsePublicSymbol(Function *Fn) {
112606c3fb27SDimitry Andric // It isn't legal to make an alias of this function at all
112706c3fb27SDimitry Andric if (Fn->isDeclarationForLinker())
112806c3fb27SDimitry Andric return true;
112906c3fb27SDimitry Andric
113006c3fb27SDimitry Andric // Symbols with local linkage can just use the symbol directly without
113106c3fb27SDimitry Andric // introducing relocations
113206c3fb27SDimitry Andric if (Fn->hasLocalLinkage())
113306c3fb27SDimitry Andric return true;
113406c3fb27SDimitry Andric
113506c3fb27SDimitry Andric // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
113606c3fb27SDimitry Andric // unfavorable interaction between the new alias and the alias renaming done
113706c3fb27SDimitry Andric // in LowerTypeTests under ThinLTO. For comdat functions that would normally
113806c3fb27SDimitry Andric // be deduplicated, but the renaming scheme ends up preventing renaming, since
113906c3fb27SDimitry Andric // it creates unique names for each alias, resulting in duplicated symbols. In
114006c3fb27SDimitry Andric // the future, we should update the CFI related passes to migrate these
114106c3fb27SDimitry Andric // aliases to the same module as the jump-table they refer to will be defined.
114206c3fb27SDimitry Andric if (Fn->hasMetadata(LLVMContext::MD_type))
114306c3fb27SDimitry Andric return true;
114406c3fb27SDimitry Andric
114506c3fb27SDimitry Andric // For comdat functions, an alias would need the same linkage as the original
114606c3fb27SDimitry Andric // function and hidden visibility. There is no point in adding an alias with
114706c3fb27SDimitry Andric // identical linkage an visibility to avoid introducing symbolic relocations.
114806c3fb27SDimitry Andric if (Fn->hasComdat() &&
114906c3fb27SDimitry Andric (Fn->getVisibility() == GlobalValue::VisibilityTypes::HiddenVisibility))
115006c3fb27SDimitry Andric return true;
115106c3fb27SDimitry Andric
115206c3fb27SDimitry Andric // its OK to use an alias
115306c3fb27SDimitry Andric return false;
115406c3fb27SDimitry Andric }
115506c3fb27SDimitry Andric
getFuncAddrForProfData(Function * Fn)115606c3fb27SDimitry Andric static inline Constant *getFuncAddrForProfData(Function *Fn) {
11575f757f3fSDimitry Andric auto *Int8PtrTy = PointerType::getUnqual(Fn->getContext());
115806c3fb27SDimitry Andric // Store a nullptr in __llvm_profd, if we shouldn't use a real address
115906c3fb27SDimitry Andric if (!shouldRecordFunctionAddr(Fn))
116006c3fb27SDimitry Andric return ConstantPointerNull::get(Int8PtrTy);
116106c3fb27SDimitry Andric
116206c3fb27SDimitry Andric // If we can't use an alias, we must use the public symbol, even though this
116306c3fb27SDimitry Andric // may require a symbolic relocation.
116406c3fb27SDimitry Andric if (shouldUsePublicSymbol(Fn))
11655f757f3fSDimitry Andric return Fn;
116606c3fb27SDimitry Andric
116706c3fb27SDimitry Andric // When possible use a private alias to avoid symbolic relocations.
116806c3fb27SDimitry Andric auto *GA = GlobalAlias::create(GlobalValue::LinkageTypes::PrivateLinkage,
116906c3fb27SDimitry Andric Fn->getName() + ".local", Fn);
117006c3fb27SDimitry Andric
117106c3fb27SDimitry Andric // When the instrumented function is a COMDAT function, we cannot use a
117206c3fb27SDimitry Andric // private alias. If we did, we would create reference to a local label in
117306c3fb27SDimitry Andric // this function's section. If this version of the function isn't selected by
117406c3fb27SDimitry Andric // the linker, then the metadata would introduce a reference to a discarded
117506c3fb27SDimitry Andric // section. So, for COMDAT functions, we need to adjust the linkage of the
117606c3fb27SDimitry Andric // alias. Using hidden visibility avoids a dynamic relocation and an entry in
117706c3fb27SDimitry Andric // the dynamic symbol table.
117806c3fb27SDimitry Andric //
117906c3fb27SDimitry Andric // Note that this handles COMDAT functions with visibility other than Hidden,
118006c3fb27SDimitry Andric // since that case is covered in shouldUsePublicSymbol()
118106c3fb27SDimitry Andric if (Fn->hasComdat()) {
118206c3fb27SDimitry Andric GA->setLinkage(Fn->getLinkage());
118306c3fb27SDimitry Andric GA->setVisibility(GlobalValue::VisibilityTypes::HiddenVisibility);
118406c3fb27SDimitry Andric }
118506c3fb27SDimitry Andric
118606c3fb27SDimitry Andric // appendToCompilerUsed(*Fn->getParent(), {GA});
118706c3fb27SDimitry Andric
11885f757f3fSDimitry Andric return GA;
118906c3fb27SDimitry Andric }
119006c3fb27SDimitry Andric
needsRuntimeRegistrationOfSectionRange(const Triple & TT)11910b57cec5SDimitry Andric static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
11921db9f3b2SDimitry Andric // compiler-rt uses linker support to get data/counters/name start/end for
11931db9f3b2SDimitry Andric // ELF, COFF, Mach-O and XCOFF.
11941db9f3b2SDimitry Andric if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
11951db9f3b2SDimitry Andric TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF())
11960b57cec5SDimitry Andric return false;
11970b57cec5SDimitry Andric
11980b57cec5SDimitry Andric return true;
11990b57cec5SDimitry Andric }
12000b57cec5SDimitry Andric
maybeSetComdat(GlobalVariable * GV,Function * Fn,StringRef VarName)12015f757f3fSDimitry Andric void InstrLowerer::maybeSetComdat(GlobalVariable *GV, Function *Fn,
12025f757f3fSDimitry Andric StringRef VarName) {
12035f757f3fSDimitry Andric bool DataReferencedByCode = profDataReferencedByCode(M);
12045f757f3fSDimitry Andric bool NeedComdat = needsComdatForCounter(*Fn, M);
12055f757f3fSDimitry Andric bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
12065f757f3fSDimitry Andric
12075f757f3fSDimitry Andric if (!UseComdat)
12085f757f3fSDimitry Andric return;
12095f757f3fSDimitry Andric
12105f757f3fSDimitry Andric StringRef GroupName =
12115f757f3fSDimitry Andric TT.isOSBinFormatCOFF() && DataReferencedByCode ? GV->getName() : VarName;
12125f757f3fSDimitry Andric Comdat *C = M.getOrInsertComdat(GroupName);
12135f757f3fSDimitry Andric if (!NeedComdat)
12145f757f3fSDimitry Andric C->setSelectionKind(Comdat::NoDeduplicate);
12155f757f3fSDimitry Andric GV->setComdat(C);
12165f757f3fSDimitry Andric // COFF doesn't allow the comdat group leader to have private linkage, so
12175f757f3fSDimitry Andric // upgrade private linkage to internal linkage to produce a symbol table
12185f757f3fSDimitry Andric // entry.
12195f757f3fSDimitry Andric if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
12205f757f3fSDimitry Andric GV->setLinkage(GlobalValue::InternalLinkage);
12211fd87a68SDimitry Andric }
12221fd87a68SDimitry Andric
setupProfileSection(InstrProfInstBase * Inc,InstrProfSectKind IPSK)12235f757f3fSDimitry Andric GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,
12245f757f3fSDimitry Andric InstrProfSectKind IPSK) {
12250b57cec5SDimitry Andric GlobalVariable *NamePtr = Inc->getName();
12260b57cec5SDimitry Andric
1227fe6060f1SDimitry Andric // Match the linkage and visibility of the name global.
12280b57cec5SDimitry Andric Function *Fn = Inc->getParent()->getParent();
12290b57cec5SDimitry Andric GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
12300b57cec5SDimitry Andric GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
12310b57cec5SDimitry Andric
12320eae32dcSDimitry Andric // Use internal rather than private linkage so the counter variable shows up
12330eae32dcSDimitry Andric // in the symbol table when using debug info for correlation.
12345f757f3fSDimitry Andric if ((DebugInfoCorrelate ||
12355f757f3fSDimitry Andric ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) &&
12365f757f3fSDimitry Andric TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage)
12370eae32dcSDimitry Andric Linkage = GlobalValue::InternalLinkage;
12380eae32dcSDimitry Andric
1239349cc55cSDimitry Andric // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1240349cc55cSDimitry Andric // symbols in the same csect won't be discarded. When there are duplicate weak
1241349cc55cSDimitry Andric // symbols, we can NOT guarantee that the relocations get resolved to the
1242349cc55cSDimitry Andric // intended weak symbol, so we can not ensure the correctness of the relative
1243349cc55cSDimitry Andric // CounterPtr, so we have to use private linkage for counter and data symbols.
1244349cc55cSDimitry Andric if (TT.isOSBinFormatXCOFF()) {
1245349cc55cSDimitry Andric Linkage = GlobalValue::PrivateLinkage;
1246349cc55cSDimitry Andric Visibility = GlobalValue::DefaultVisibility;
1247349cc55cSDimitry Andric }
12480b57cec5SDimitry Andric // Move the name variable to the right section. Place them in a COMDAT group
12490b57cec5SDimitry Andric // if the associated function is a COMDAT. This will make sure that only one
12500b57cec5SDimitry Andric // copy of counters of the COMDAT function will be emitted after linking. Keep
12510b57cec5SDimitry Andric // in mind that this pass may run before the inliner, so we need to create a
12520b57cec5SDimitry Andric // new comdat group for the counters and profiling data. If we use the comdat
12530b57cec5SDimitry Andric // of the parent function, that will result in relocations against discarded
12540b57cec5SDimitry Andric // sections.
1255fe6060f1SDimitry Andric //
1256fe6060f1SDimitry Andric // If the data variable is referenced by code, counters and data have to be
1257fe6060f1SDimitry Andric // in different comdats for COFF because the Visual C++ linker will report
1258fe6060f1SDimitry Andric // duplicate symbol errors if there are multiple external symbols with the
1259fe6060f1SDimitry Andric // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
1260fe6060f1SDimitry Andric //
1261fe6060f1SDimitry Andric // For ELF, when not using COMDAT, put counters, data and values into a
1262fe6060f1SDimitry Andric // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
1263fe6060f1SDimitry Andric // allows -z start-stop-gc to discard the entire group when the function is
1264fe6060f1SDimitry Andric // discarded.
1265349cc55cSDimitry Andric bool Renamed;
12665f757f3fSDimitry Andric GlobalVariable *Ptr;
12675f757f3fSDimitry Andric StringRef VarPrefix;
12685f757f3fSDimitry Andric std::string VarName;
12695f757f3fSDimitry Andric if (IPSK == IPSK_cnts) {
12705f757f3fSDimitry Andric VarPrefix = getInstrProfCountersVarPrefix();
12715f757f3fSDimitry Andric VarName = getVarName(Inc, VarPrefix, Renamed);
12725f757f3fSDimitry Andric InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Inc);
12735f757f3fSDimitry Andric Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);
12745f757f3fSDimitry Andric } else if (IPSK == IPSK_bitmap) {
12755f757f3fSDimitry Andric VarPrefix = getInstrProfBitmapVarPrefix();
12765f757f3fSDimitry Andric VarName = getVarName(Inc, VarPrefix, Renamed);
12775f757f3fSDimitry Andric InstrProfMCDCBitmapInstBase *BitmapUpdate =
12785f757f3fSDimitry Andric dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);
12795f757f3fSDimitry Andric Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);
12805f757f3fSDimitry Andric } else {
12815f757f3fSDimitry Andric llvm_unreachable("Profile Section must be for Counters or Bitmaps");
1282fe6060f1SDimitry Andric }
12830b57cec5SDimitry Andric
12845f757f3fSDimitry Andric Ptr->setVisibility(Visibility);
12855f757f3fSDimitry Andric // Put the counters and bitmaps in their own sections so linkers can
12865f757f3fSDimitry Andric // remove unneeded sections.
12875f757f3fSDimitry Andric Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat()));
12885f757f3fSDimitry Andric Ptr->setLinkage(Linkage);
12895f757f3fSDimitry Andric maybeSetComdat(Ptr, Fn, VarName);
12905f757f3fSDimitry Andric return Ptr;
12915f757f3fSDimitry Andric }
12925f757f3fSDimitry Andric
12935f757f3fSDimitry Andric GlobalVariable *
createRegionBitmaps(InstrProfMCDCBitmapInstBase * Inc,StringRef Name,GlobalValue::LinkageTypes Linkage)12945f757f3fSDimitry Andric InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
12955f757f3fSDimitry Andric StringRef Name,
12965f757f3fSDimitry Andric GlobalValue::LinkageTypes Linkage) {
12975f757f3fSDimitry Andric uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue();
12985f757f3fSDimitry Andric auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M.getContext()), NumBytes);
12995f757f3fSDimitry Andric auto GV = new GlobalVariable(M, BitmapTy, false, Linkage,
13005f757f3fSDimitry Andric Constant::getNullValue(BitmapTy), Name);
13015f757f3fSDimitry Andric GV->setAlignment(Align(1));
13025f757f3fSDimitry Andric return GV;
13035f757f3fSDimitry Andric }
13045f757f3fSDimitry Andric
13055f757f3fSDimitry Andric GlobalVariable *
getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase * Inc)13065f757f3fSDimitry Andric InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
13075f757f3fSDimitry Andric GlobalVariable *NamePtr = Inc->getName();
13085f757f3fSDimitry Andric auto &PD = ProfileDataMap[NamePtr];
13095f757f3fSDimitry Andric if (PD.RegionBitmaps)
13105f757f3fSDimitry Andric return PD.RegionBitmaps;
13115f757f3fSDimitry Andric
13125f757f3fSDimitry Andric // If RegionBitmaps doesn't already exist, create it by first setting up
13135f757f3fSDimitry Andric // the corresponding profile section.
13145f757f3fSDimitry Andric auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
13155f757f3fSDimitry Andric PD.RegionBitmaps = BitmapPtr;
13165f757f3fSDimitry Andric PD.NumBitmapBytes = Inc->getNumBitmapBytes()->getZExtValue();
13175f757f3fSDimitry Andric return PD.RegionBitmaps;
13185f757f3fSDimitry Andric }
13195f757f3fSDimitry Andric
13205f757f3fSDimitry Andric GlobalVariable *
createRegionCounters(InstrProfCntrInstBase * Inc,StringRef Name,GlobalValue::LinkageTypes Linkage)13215f757f3fSDimitry Andric InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
13225f757f3fSDimitry Andric GlobalValue::LinkageTypes Linkage) {
13230b57cec5SDimitry Andric uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
13245f757f3fSDimitry Andric auto &Ctx = M.getContext();
13255f757f3fSDimitry Andric GlobalVariable *GV;
13265f757f3fSDimitry Andric if (isa<InstrProfCoverInst>(Inc)) {
13275f757f3fSDimitry Andric auto *CounterTy = Type::getInt8Ty(Ctx);
13285f757f3fSDimitry Andric auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
13295f757f3fSDimitry Andric // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
13305f757f3fSDimitry Andric std::vector<Constant *> InitialValues(NumCounters,
13315f757f3fSDimitry Andric Constant::getAllOnesValue(CounterTy));
13325f757f3fSDimitry Andric GV = new GlobalVariable(M, CounterArrTy, false, Linkage,
13335f757f3fSDimitry Andric ConstantArray::get(CounterArrTy, InitialValues),
13345f757f3fSDimitry Andric Name);
13355f757f3fSDimitry Andric GV->setAlignment(Align(1));
13365f757f3fSDimitry Andric } else {
13375f757f3fSDimitry Andric auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
13385f757f3fSDimitry Andric GV = new GlobalVariable(M, CounterTy, false, Linkage,
13395f757f3fSDimitry Andric Constant::getNullValue(CounterTy), Name);
13405f757f3fSDimitry Andric GV->setAlignment(Align(8));
13415f757f3fSDimitry Andric }
13425f757f3fSDimitry Andric return GV;
13435f757f3fSDimitry Andric }
13440b57cec5SDimitry Andric
13455f757f3fSDimitry Andric GlobalVariable *
getOrCreateRegionCounters(InstrProfCntrInstBase * Inc)13465f757f3fSDimitry Andric InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
13475f757f3fSDimitry Andric GlobalVariable *NamePtr = Inc->getName();
13485f757f3fSDimitry Andric auto &PD = ProfileDataMap[NamePtr];
13495f757f3fSDimitry Andric if (PD.RegionCounters)
13505f757f3fSDimitry Andric return PD.RegionCounters;
13515f757f3fSDimitry Andric
13525f757f3fSDimitry Andric // If RegionCounters doesn't already exist, create it by first setting up
13535f757f3fSDimitry Andric // the corresponding profile section.
13545f757f3fSDimitry Andric auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
13554824e7fdSDimitry Andric PD.RegionCounters = CounterPtr;
13565f757f3fSDimitry Andric
13575f757f3fSDimitry Andric if (DebugInfoCorrelate ||
13585f757f3fSDimitry Andric ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) {
13595f757f3fSDimitry Andric LLVMContext &Ctx = M.getContext();
13605f757f3fSDimitry Andric Function *Fn = Inc->getParent()->getParent();
13610eae32dcSDimitry Andric if (auto *SP = Fn->getSubprogram()) {
13625f757f3fSDimitry Andric DIBuilder DB(M, true, SP->getUnit());
13630eae32dcSDimitry Andric Metadata *FunctionNameAnnotation[] = {
13640eae32dcSDimitry Andric MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName),
13650eae32dcSDimitry Andric MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)),
13660eae32dcSDimitry Andric };
13670eae32dcSDimitry Andric Metadata *CFGHashAnnotation[] = {
13680eae32dcSDimitry Andric MDString::get(Ctx, InstrProfCorrelator::CFGHashAttributeName),
13690eae32dcSDimitry Andric ConstantAsMetadata::get(Inc->getHash()),
13700eae32dcSDimitry Andric };
13710eae32dcSDimitry Andric Metadata *NumCountersAnnotation[] = {
13720eae32dcSDimitry Andric MDString::get(Ctx, InstrProfCorrelator::NumCountersAttributeName),
13730eae32dcSDimitry Andric ConstantAsMetadata::get(Inc->getNumCounters()),
13740eae32dcSDimitry Andric };
13750eae32dcSDimitry Andric auto Annotations = DB.getOrCreateArray({
13760eae32dcSDimitry Andric MDNode::get(Ctx, FunctionNameAnnotation),
13770eae32dcSDimitry Andric MDNode::get(Ctx, CFGHashAnnotation),
13780eae32dcSDimitry Andric MDNode::get(Ctx, NumCountersAnnotation),
13790eae32dcSDimitry Andric });
13800eae32dcSDimitry Andric auto *DICounter = DB.createGlobalVariableExpression(
13810eae32dcSDimitry Andric SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
13820eae32dcSDimitry Andric /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
13830eae32dcSDimitry Andric CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
13840eae32dcSDimitry Andric /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
13850eae32dcSDimitry Andric Annotations);
13860eae32dcSDimitry Andric CounterPtr->addDebugInfo(DICounter);
13870eae32dcSDimitry Andric DB.finalize();
13880eae32dcSDimitry Andric }
13890b57cec5SDimitry Andric
13905f757f3fSDimitry Andric // Mark the counter variable as used so that it isn't optimized out.
13915f757f3fSDimitry Andric CompilerUsedVars.push_back(PD.RegionCounters);
13925f757f3fSDimitry Andric }
13935f757f3fSDimitry Andric
13945f757f3fSDimitry Andric // Create the data variable (if it doesn't already exist).
13955f757f3fSDimitry Andric createDataVariable(Inc);
13965f757f3fSDimitry Andric
13975f757f3fSDimitry Andric return PD.RegionCounters;
13985f757f3fSDimitry Andric }
13995f757f3fSDimitry Andric
createDataVariable(InstrProfCntrInstBase * Inc)14005f757f3fSDimitry Andric void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
14015f757f3fSDimitry Andric // When debug information is correlated to profile data, a data variable
14025f757f3fSDimitry Andric // is not needed.
14035f757f3fSDimitry Andric if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)
14045f757f3fSDimitry Andric return;
14055f757f3fSDimitry Andric
14065f757f3fSDimitry Andric GlobalVariable *NamePtr = Inc->getName();
14075f757f3fSDimitry Andric auto &PD = ProfileDataMap[NamePtr];
14085f757f3fSDimitry Andric
14095f757f3fSDimitry Andric // Return if data variable was already created.
14105f757f3fSDimitry Andric if (PD.DataVar)
14115f757f3fSDimitry Andric return;
14125f757f3fSDimitry Andric
14135f757f3fSDimitry Andric LLVMContext &Ctx = M.getContext();
14145f757f3fSDimitry Andric
14155f757f3fSDimitry Andric Function *Fn = Inc->getParent()->getParent();
14165f757f3fSDimitry Andric GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
14175f757f3fSDimitry Andric GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
14185f757f3fSDimitry Andric
14195f757f3fSDimitry Andric // Due to the limitation of binder as of 2021/09/28, the duplicate weak
14205f757f3fSDimitry Andric // symbols in the same csect won't be discarded. When there are duplicate weak
14215f757f3fSDimitry Andric // symbols, we can NOT guarantee that the relocations get resolved to the
14225f757f3fSDimitry Andric // intended weak symbol, so we can not ensure the correctness of the relative
14235f757f3fSDimitry Andric // CounterPtr, so we have to use private linkage for counter and data symbols.
14245f757f3fSDimitry Andric if (TT.isOSBinFormatXCOFF()) {
14255f757f3fSDimitry Andric Linkage = GlobalValue::PrivateLinkage;
14265f757f3fSDimitry Andric Visibility = GlobalValue::DefaultVisibility;
14275f757f3fSDimitry Andric }
14285f757f3fSDimitry Andric
14295f757f3fSDimitry Andric bool DataReferencedByCode = profDataReferencedByCode(M);
14305f757f3fSDimitry Andric bool NeedComdat = needsComdatForCounter(*Fn, M);
14315f757f3fSDimitry Andric bool Renamed;
14325f757f3fSDimitry Andric
14335f757f3fSDimitry Andric // The Data Variable section is anchored to profile counters.
14345f757f3fSDimitry Andric std::string CntsVarName =
14355f757f3fSDimitry Andric getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);
14365f757f3fSDimitry Andric std::string DataVarName =
14375f757f3fSDimitry Andric getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
14385f757f3fSDimitry Andric
14395f757f3fSDimitry Andric auto *Int8PtrTy = PointerType::getUnqual(Ctx);
14400b57cec5SDimitry Andric // Allocate statically the array of pointers to value profile nodes for
14410b57cec5SDimitry Andric // the current function.
14420b57cec5SDimitry Andric Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
14430b57cec5SDimitry Andric uint64_t NS = 0;
14440b57cec5SDimitry Andric for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
14450b57cec5SDimitry Andric NS += PD.NumValueSites[Kind];
1446fe6060f1SDimitry Andric if (NS > 0 && ValueProfileStaticAlloc &&
1447fe6060f1SDimitry Andric !needsRuntimeRegistrationOfSectionRange(TT)) {
14480b57cec5SDimitry Andric ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
1449fe6060f1SDimitry Andric auto *ValuesVar = new GlobalVariable(
14505f757f3fSDimitry Andric M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
1451349cc55cSDimitry Andric getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
14520b57cec5SDimitry Andric ValuesVar->setVisibility(Visibility);
14535f757f3fSDimitry Andric setGlobalVariableLargeSection(TT, *ValuesVar);
14540b57cec5SDimitry Andric ValuesVar->setSection(
14550b57cec5SDimitry Andric getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
14568bcb0991SDimitry Andric ValuesVar->setAlignment(Align(8));
14575f757f3fSDimitry Andric maybeSetComdat(ValuesVar, Fn, CntsVarName);
14585f757f3fSDimitry Andric ValuesPtrExpr = ValuesVar;
14590b57cec5SDimitry Andric }
14600b57cec5SDimitry Andric
14615f757f3fSDimitry Andric uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
14625f757f3fSDimitry Andric auto *CounterPtr = PD.RegionCounters;
14635f757f3fSDimitry Andric
14645f757f3fSDimitry Andric uint64_t NumBitmapBytes = PD.NumBitmapBytes;
14650eae32dcSDimitry Andric
14660b57cec5SDimitry Andric // Create data variable.
14675f757f3fSDimitry Andric auto *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext());
14680b57cec5SDimitry Andric auto *Int16Ty = Type::getInt16Ty(Ctx);
14690b57cec5SDimitry Andric auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
14700b57cec5SDimitry Andric Type *DataTypes[] = {
14710b57cec5SDimitry Andric #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
14720b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfData.inc"
14730b57cec5SDimitry Andric };
1474bdd1243dSDimitry Andric auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));
14750b57cec5SDimitry Andric
147606c3fb27SDimitry Andric Constant *FunctionAddr = getFuncAddrForProfData(Fn);
14770b57cec5SDimitry Andric
14780b57cec5SDimitry Andric Constant *Int16ArrayVals[IPVK_Last + 1];
14790b57cec5SDimitry Andric for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
14800b57cec5SDimitry Andric Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
14810b57cec5SDimitry Andric
1482fe6060f1SDimitry Andric // If the data variable is not referenced by code (if we don't emit
1483fe6060f1SDimitry Andric // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1484fe6060f1SDimitry Andric // data variable live under linker GC, the data variable can be private. This
1485fe6060f1SDimitry Andric // optimization applies to ELF.
1486fe6060f1SDimitry Andric //
1487fe6060f1SDimitry Andric // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1488fe6060f1SDimitry Andric // to be false.
1489349cc55cSDimitry Andric //
1490349cc55cSDimitry Andric // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1491349cc55cSDimitry Andric // that other copies must have the same CFG and cannot have value profiling.
1492349cc55cSDimitry Andric // If no hash suffix, other profd copies may be referenced by code.
1493349cc55cSDimitry Andric if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1494349cc55cSDimitry Andric (TT.isOSBinFormatELF() ||
1495fe6060f1SDimitry Andric (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1496fe6060f1SDimitry Andric Linkage = GlobalValue::PrivateLinkage;
1497fe6060f1SDimitry Andric Visibility = GlobalValue::DefaultVisibility;
1498fe6060f1SDimitry Andric }
1499e8d8bef9SDimitry Andric auto *Data =
15005f757f3fSDimitry Andric new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName);
15015f757f3fSDimitry Andric Constant *RelativeCounterPtr;
15025f757f3fSDimitry Andric GlobalVariable *BitmapPtr = PD.RegionBitmaps;
15035f757f3fSDimitry Andric Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);
15045f757f3fSDimitry Andric InstrProfSectKind DataSectionKind;
15055f757f3fSDimitry Andric // With binary profile correlation, profile data is not loaded into memory.
15065f757f3fSDimitry Andric // profile data must reference profile counter with an absolute relocation.
15075f757f3fSDimitry Andric if (ProfileCorrelate == InstrProfCorrelator::BINARY) {
15085f757f3fSDimitry Andric DataSectionKind = IPSK_covdata;
15095f757f3fSDimitry Andric RelativeCounterPtr = ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy);
15105f757f3fSDimitry Andric if (BitmapPtr != nullptr)
15115f757f3fSDimitry Andric RelativeBitmapPtr = ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy);
15125f757f3fSDimitry Andric } else {
1513349cc55cSDimitry Andric // Reference the counter variable with a label difference (link-time
1514349cc55cSDimitry Andric // constant).
15155f757f3fSDimitry Andric DataSectionKind = IPSK_data;
15165f757f3fSDimitry Andric RelativeCounterPtr =
1517349cc55cSDimitry Andric ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
1518349cc55cSDimitry Andric ConstantExpr::getPtrToInt(Data, IntPtrTy));
15195f757f3fSDimitry Andric if (BitmapPtr != nullptr)
15205f757f3fSDimitry Andric RelativeBitmapPtr =
15215f757f3fSDimitry Andric ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy),
15225f757f3fSDimitry Andric ConstantExpr::getPtrToInt(Data, IntPtrTy));
15235f757f3fSDimitry Andric }
1524349cc55cSDimitry Andric
1525349cc55cSDimitry Andric Constant *DataVals[] = {
1526349cc55cSDimitry Andric #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1527349cc55cSDimitry Andric #include "llvm/ProfileData/InstrProfData.inc"
1528349cc55cSDimitry Andric };
1529349cc55cSDimitry Andric Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1530349cc55cSDimitry Andric
15310b57cec5SDimitry Andric Data->setVisibility(Visibility);
15325f757f3fSDimitry Andric Data->setSection(
15335f757f3fSDimitry Andric getInstrProfSectionName(DataSectionKind, TT.getObjectFormat()));
15348bcb0991SDimitry Andric Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
15355f757f3fSDimitry Andric maybeSetComdat(Data, Fn, CntsVarName);
15360b57cec5SDimitry Andric
15370b57cec5SDimitry Andric PD.DataVar = Data;
15380b57cec5SDimitry Andric
15390b57cec5SDimitry Andric // Mark the data variable as used so that it isn't stripped out.
1540fe6060f1SDimitry Andric CompilerUsedVars.push_back(Data);
15410b57cec5SDimitry Andric // Now that the linkage set by the FE has been passed to the data and counter
15420b57cec5SDimitry Andric // variables, reset Name variable's linkage and visibility to private so that
15430b57cec5SDimitry Andric // it can be removed later by the compiler.
15440b57cec5SDimitry Andric NamePtr->setLinkage(GlobalValue::PrivateLinkage);
15450b57cec5SDimitry Andric // Collect the referenced names to be used by emitNameData.
15460b57cec5SDimitry Andric ReferencedNames.push_back(NamePtr);
15470b57cec5SDimitry Andric }
15480b57cec5SDimitry Andric
emitVNodes()15495f757f3fSDimitry Andric void InstrLowerer::emitVNodes() {
15500b57cec5SDimitry Andric if (!ValueProfileStaticAlloc)
15510b57cec5SDimitry Andric return;
15520b57cec5SDimitry Andric
15530b57cec5SDimitry Andric // For now only support this on platforms that do
15540b57cec5SDimitry Andric // not require runtime registration to discover
15550b57cec5SDimitry Andric // named section start/end.
15560b57cec5SDimitry Andric if (needsRuntimeRegistrationOfSectionRange(TT))
15570b57cec5SDimitry Andric return;
15580b57cec5SDimitry Andric
15590b57cec5SDimitry Andric size_t TotalNS = 0;
15600b57cec5SDimitry Andric for (auto &PD : ProfileDataMap) {
15610b57cec5SDimitry Andric for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
15620b57cec5SDimitry Andric TotalNS += PD.second.NumValueSites[Kind];
15630b57cec5SDimitry Andric }
15640b57cec5SDimitry Andric
15650b57cec5SDimitry Andric if (!TotalNS)
15660b57cec5SDimitry Andric return;
15670b57cec5SDimitry Andric
15680b57cec5SDimitry Andric uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
15690b57cec5SDimitry Andric // Heuristic for small programs with very few total value sites.
15700b57cec5SDimitry Andric // The default value of vp-counters-per-site is chosen based on
15710b57cec5SDimitry Andric // the observation that large apps usually have a low percentage
15720b57cec5SDimitry Andric // of value sites that actually have any profile data, and thus
15730b57cec5SDimitry Andric // the average number of counters per site is low. For small
15740b57cec5SDimitry Andric // apps with very few sites, this may not be true. Bump up the
15750b57cec5SDimitry Andric // number of counters in this case.
15760b57cec5SDimitry Andric #define INSTR_PROF_MIN_VAL_COUNTS 10
15770b57cec5SDimitry Andric if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
15780b57cec5SDimitry Andric NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
15790b57cec5SDimitry Andric
15805f757f3fSDimitry Andric auto &Ctx = M.getContext();
15810b57cec5SDimitry Andric Type *VNodeTypes[] = {
15820b57cec5SDimitry Andric #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
15830b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfData.inc"
15840b57cec5SDimitry Andric };
1585bdd1243dSDimitry Andric auto *VNodeTy = StructType::get(Ctx, ArrayRef(VNodeTypes));
15860b57cec5SDimitry Andric
15870b57cec5SDimitry Andric ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
15880b57cec5SDimitry Andric auto *VNodesVar = new GlobalVariable(
15895f757f3fSDimitry Andric M, VNodesTy, false, GlobalValue::PrivateLinkage,
15900b57cec5SDimitry Andric Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
15915f757f3fSDimitry Andric setGlobalVariableLargeSection(TT, *VNodesVar);
15920b57cec5SDimitry Andric VNodesVar->setSection(
15930b57cec5SDimitry Andric getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
15945f757f3fSDimitry Andric VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(VNodesTy));
1595fe6060f1SDimitry Andric // VNodesVar is used by runtime but not referenced via relocation by other
1596fe6060f1SDimitry Andric // sections. Conservatively make it linker retained.
15970b57cec5SDimitry Andric UsedVars.push_back(VNodesVar);
15980b57cec5SDimitry Andric }
15990b57cec5SDimitry Andric
emitNameData()16005f757f3fSDimitry Andric void InstrLowerer::emitNameData() {
16010b57cec5SDimitry Andric std::string UncompressedData;
16020b57cec5SDimitry Andric
16030b57cec5SDimitry Andric if (ReferencedNames.empty())
16040b57cec5SDimitry Andric return;
16050b57cec5SDimitry Andric
16060b57cec5SDimitry Andric std::string CompressedNameStr;
16070b57cec5SDimitry Andric if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
16085ffd83dbSDimitry Andric DoInstrProfNameCompression)) {
1609349cc55cSDimitry Andric report_fatal_error(Twine(toString(std::move(E))), false);
16100b57cec5SDimitry Andric }
16110b57cec5SDimitry Andric
16125f757f3fSDimitry Andric auto &Ctx = M.getContext();
16134824e7fdSDimitry Andric auto *NamesVal =
16144824e7fdSDimitry Andric ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);
16155f757f3fSDimitry Andric NamesVar = new GlobalVariable(M, NamesVal->getType(), true,
16160b57cec5SDimitry Andric GlobalValue::PrivateLinkage, NamesVal,
16170b57cec5SDimitry Andric getInstrProfNamesVarName());
16180b57cec5SDimitry Andric NamesSize = CompressedNameStr.size();
16195f757f3fSDimitry Andric setGlobalVariableLargeSection(TT, *NamesVar);
16200b57cec5SDimitry Andric NamesVar->setSection(
16215f757f3fSDimitry Andric ProfileCorrelate == InstrProfCorrelator::BINARY
16225f757f3fSDimitry Andric ? getInstrProfSectionName(IPSK_covname, TT.getObjectFormat())
16235f757f3fSDimitry Andric : getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
16240b57cec5SDimitry Andric // On COFF, it's important to reduce the alignment down to 1 to prevent the
16250b57cec5SDimitry Andric // linker from inserting padding before the start of the names section or
16260b57cec5SDimitry Andric // between names entries.
16275ffd83dbSDimitry Andric NamesVar->setAlignment(Align(1));
1628fe6060f1SDimitry Andric // NamesVar is used by runtime but not referenced via relocation by other
1629fe6060f1SDimitry Andric // sections. Conservatively make it linker retained.
16300b57cec5SDimitry Andric UsedVars.push_back(NamesVar);
16310b57cec5SDimitry Andric
16320b57cec5SDimitry Andric for (auto *NamePtr : ReferencedNames)
16330b57cec5SDimitry Andric NamePtr->eraseFromParent();
16340b57cec5SDimitry Andric }
16350b57cec5SDimitry Andric
emitRegistration()16365f757f3fSDimitry Andric void InstrLowerer::emitRegistration() {
16370b57cec5SDimitry Andric if (!needsRuntimeRegistrationOfSectionRange(TT))
16380b57cec5SDimitry Andric return;
16390b57cec5SDimitry Andric
16400b57cec5SDimitry Andric // Construct the function.
16415f757f3fSDimitry Andric auto *VoidTy = Type::getVoidTy(M.getContext());
16425f757f3fSDimitry Andric auto *VoidPtrTy = PointerType::getUnqual(M.getContext());
16435f757f3fSDimitry Andric auto *Int64Ty = Type::getInt64Ty(M.getContext());
16440b57cec5SDimitry Andric auto *RegisterFTy = FunctionType::get(VoidTy, false);
16450b57cec5SDimitry Andric auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
16460b57cec5SDimitry Andric getInstrProfRegFuncsName(), M);
16470b57cec5SDimitry Andric RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
16480b57cec5SDimitry Andric if (Options.NoRedZone)
16490b57cec5SDimitry Andric RegisterF->addFnAttr(Attribute::NoRedZone);
16500b57cec5SDimitry Andric
16510b57cec5SDimitry Andric auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
16520b57cec5SDimitry Andric auto *RuntimeRegisterF =
16530b57cec5SDimitry Andric Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
16540b57cec5SDimitry Andric getInstrProfRegFuncName(), M);
16550b57cec5SDimitry Andric
16565f757f3fSDimitry Andric IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", RegisterF));
1657fe6060f1SDimitry Andric for (Value *Data : CompilerUsedVars)
1658fe6060f1SDimitry Andric if (!isa<Function>(Data))
16595f757f3fSDimitry Andric IRB.CreateCall(RuntimeRegisterF, Data);
16600b57cec5SDimitry Andric for (Value *Data : UsedVars)
16610b57cec5SDimitry Andric if (Data != NamesVar && !isa<Function>(Data))
16625f757f3fSDimitry Andric IRB.CreateCall(RuntimeRegisterF, Data);
16630b57cec5SDimitry Andric
16640b57cec5SDimitry Andric if (NamesVar) {
16650b57cec5SDimitry Andric Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
16660b57cec5SDimitry Andric auto *NamesRegisterTy =
1667bdd1243dSDimitry Andric FunctionType::get(VoidTy, ArrayRef(ParamTypes), false);
16680b57cec5SDimitry Andric auto *NamesRegisterF =
16690b57cec5SDimitry Andric Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
16700b57cec5SDimitry Andric getInstrProfNamesRegFuncName(), M);
16715f757f3fSDimitry Andric IRB.CreateCall(NamesRegisterF, {NamesVar, IRB.getInt64(NamesSize)});
16720b57cec5SDimitry Andric }
16730b57cec5SDimitry Andric
16740b57cec5SDimitry Andric IRB.CreateRetVoid();
16750b57cec5SDimitry Andric }
16760b57cec5SDimitry Andric
emitRuntimeHook()16775f757f3fSDimitry Andric bool InstrLowerer::emitRuntimeHook() {
1678349cc55cSDimitry Andric // We expect the linker to be invoked with -u<hook_var> flag for Linux
1679349cc55cSDimitry Andric // in which case there is no need to emit the external variable.
1680bdd1243dSDimitry Andric if (TT.isOSLinux() || TT.isOSAIX())
16810b57cec5SDimitry Andric return false;
16820b57cec5SDimitry Andric
16830b57cec5SDimitry Andric // If the module's provided its own runtime, we don't need to do anything.
16845f757f3fSDimitry Andric if (M.getGlobalVariable(getInstrProfRuntimeHookVarName()))
16850b57cec5SDimitry Andric return false;
16860b57cec5SDimitry Andric
16870b57cec5SDimitry Andric // Declare an external variable that will pull in the runtime initialization.
16885f757f3fSDimitry Andric auto *Int32Ty = Type::getInt32Ty(M.getContext());
16890b57cec5SDimitry Andric auto *Var =
16905f757f3fSDimitry Andric new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
16910b57cec5SDimitry Andric nullptr, getInstrProfRuntimeHookVarName());
1692753f127fSDimitry Andric Var->setVisibility(GlobalValue::HiddenVisibility);
16930b57cec5SDimitry Andric
169481ad6265SDimitry Andric if (TT.isOSBinFormatELF() && !TT.isPS()) {
1695349cc55cSDimitry Andric // Mark the user variable as used so that it isn't stripped out.
1696349cc55cSDimitry Andric CompilerUsedVars.push_back(Var);
1697349cc55cSDimitry Andric } else {
16980b57cec5SDimitry Andric // Make a function that uses it.
16990b57cec5SDimitry Andric auto *User = Function::Create(FunctionType::get(Int32Ty, false),
17000b57cec5SDimitry Andric GlobalValue::LinkOnceODRLinkage,
17010b57cec5SDimitry Andric getInstrProfRuntimeHookVarUseFuncName(), M);
17020b57cec5SDimitry Andric User->addFnAttr(Attribute::NoInline);
17030b57cec5SDimitry Andric if (Options.NoRedZone)
17040b57cec5SDimitry Andric User->addFnAttr(Attribute::NoRedZone);
17050b57cec5SDimitry Andric User->setVisibility(GlobalValue::HiddenVisibility);
17060b57cec5SDimitry Andric if (TT.supportsCOMDAT())
17075f757f3fSDimitry Andric User->setComdat(M.getOrInsertComdat(User->getName()));
17080b57cec5SDimitry Andric
17095f757f3fSDimitry Andric IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", User));
17100b57cec5SDimitry Andric auto *Load = IRB.CreateLoad(Int32Ty, Var);
17110b57cec5SDimitry Andric IRB.CreateRet(Load);
17120b57cec5SDimitry Andric
1713349cc55cSDimitry Andric // Mark the function as used so that it isn't stripped out.
1714fe6060f1SDimitry Andric CompilerUsedVars.push_back(User);
1715349cc55cSDimitry Andric }
17160b57cec5SDimitry Andric return true;
17170b57cec5SDimitry Andric }
17180b57cec5SDimitry Andric
emitUses()17195f757f3fSDimitry Andric void InstrLowerer::emitUses() {
1720fe6060f1SDimitry Andric // The metadata sections are parallel arrays. Optimizers (e.g.
1721fe6060f1SDimitry Andric // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1722fe6060f1SDimitry Andric // we conservatively retain all unconditionally in the compiler.
1723fe6060f1SDimitry Andric //
1724349cc55cSDimitry Andric // On ELF and Mach-O, the linker can guarantee the associated sections will be
1725349cc55cSDimitry Andric // retained or discarded as a unit, so llvm.compiler.used is sufficient.
1726349cc55cSDimitry Andric // Similarly on COFF, if prof data is not referenced by code we use one comdat
1727349cc55cSDimitry Andric // and ensure this GC property as well. Otherwise, we have to conservatively
1728349cc55cSDimitry Andric // make all of the sections retained by the linker.
1729349cc55cSDimitry Andric if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
17305f757f3fSDimitry Andric (TT.isOSBinFormatCOFF() && !profDataReferencedByCode(M)))
17315f757f3fSDimitry Andric appendToCompilerUsed(M, CompilerUsedVars);
1732fe6060f1SDimitry Andric else
17335f757f3fSDimitry Andric appendToUsed(M, CompilerUsedVars);
1734fe6060f1SDimitry Andric
1735fe6060f1SDimitry Andric // We do not add proper references from used metadata sections to NamesVar and
1736fe6060f1SDimitry Andric // VNodesVar, so we have to be conservative and place them in llvm.used
1737fe6060f1SDimitry Andric // regardless of the target,
17385f757f3fSDimitry Andric appendToUsed(M, UsedVars);
17390b57cec5SDimitry Andric }
17400b57cec5SDimitry Andric
emitInitialization()17415f757f3fSDimitry Andric void InstrLowerer::emitInitialization() {
17420b57cec5SDimitry Andric // Create ProfileFileName variable. Don't don't this for the
17430b57cec5SDimitry Andric // context-sensitive instrumentation lowering: This lowering is after
17440b57cec5SDimitry Andric // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
17450b57cec5SDimitry Andric // have already create the variable before LTO/ThinLTO linking.
17460b57cec5SDimitry Andric if (!IsCS)
17475f757f3fSDimitry Andric createProfileFileNameVar(M, Options.InstrProfileOutput);
17485f757f3fSDimitry Andric Function *RegisterF = M.getFunction(getInstrProfRegFuncsName());
17490b57cec5SDimitry Andric if (!RegisterF)
17500b57cec5SDimitry Andric return;
17510b57cec5SDimitry Andric
17520b57cec5SDimitry Andric // Create the initialization function.
17535f757f3fSDimitry Andric auto *VoidTy = Type::getVoidTy(M.getContext());
17540b57cec5SDimitry Andric auto *F = Function::Create(FunctionType::get(VoidTy, false),
17550b57cec5SDimitry Andric GlobalValue::InternalLinkage,
17560b57cec5SDimitry Andric getInstrProfInitFuncName(), M);
17570b57cec5SDimitry Andric F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
17580b57cec5SDimitry Andric F->addFnAttr(Attribute::NoInline);
17590b57cec5SDimitry Andric if (Options.NoRedZone)
17600b57cec5SDimitry Andric F->addFnAttr(Attribute::NoRedZone);
17610b57cec5SDimitry Andric
17620b57cec5SDimitry Andric // Add the basic block and the necessary calls.
17635f757f3fSDimitry Andric IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", F));
17640b57cec5SDimitry Andric IRB.CreateCall(RegisterF, {});
17650b57cec5SDimitry Andric IRB.CreateRetVoid();
17660b57cec5SDimitry Andric
17675f757f3fSDimitry Andric appendToGlobalCtors(M, F, 0);
17680b57cec5SDimitry Andric }
1769