1e8d8bef9SDimitry Andric //===- Transforms/IPO/SampleProfileProbe.h ----------*- C++ -*-===// 2e8d8bef9SDimitry Andric // 3e8d8bef9SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4e8d8bef9SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 5e8d8bef9SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6e8d8bef9SDimitry Andric // 7e8d8bef9SDimitry Andric //===----------------------------------------------------------------------===// 8e8d8bef9SDimitry Andric // 9e8d8bef9SDimitry Andric /// \file 10e8d8bef9SDimitry Andric /// This file provides the interface for the pseudo probe implementation for 11e8d8bef9SDimitry Andric /// AutoFDO. 12e8d8bef9SDimitry Andric // 13e8d8bef9SDimitry Andric //===----------------------------------------------------------------------===// 14e8d8bef9SDimitry Andric 15e8d8bef9SDimitry Andric #ifndef LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H 16e8d8bef9SDimitry Andric #define LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H 17e8d8bef9SDimitry Andric 18d409305fSDimitry Andric #include "llvm/Analysis/LazyCallGraph.h" 19e8d8bef9SDimitry Andric #include "llvm/IR/PassManager.h" 20e8d8bef9SDimitry Andric #include "llvm/ProfileData/SampleProf.h" 21e8d8bef9SDimitry Andric #include <unordered_map> 22e8d8bef9SDimitry Andric 23e8d8bef9SDimitry Andric namespace llvm { 24*81ad6265SDimitry Andric class BasicBlock; 25*81ad6265SDimitry Andric class Function; 26*81ad6265SDimitry Andric class Instruction; 27*81ad6265SDimitry Andric class Loop; 28*81ad6265SDimitry Andric class PassInstrumentationCallbacks; 29*81ad6265SDimitry Andric class TargetMachine; 30e8d8bef9SDimitry Andric 31e8d8bef9SDimitry Andric class Module; 32e8d8bef9SDimitry Andric 33e8d8bef9SDimitry Andric using namespace sampleprof; 34e8d8bef9SDimitry Andric using BlockIdMap = std::unordered_map<BasicBlock *, uint32_t>; 35e8d8bef9SDimitry Andric using InstructionIdMap = std::unordered_map<Instruction *, uint32_t>; 36fe6060f1SDimitry Andric // Map from tuples of Probe id and inline stack hash code to distribution 37fe6060f1SDimitry Andric // factors. 38fe6060f1SDimitry Andric using ProbeFactorMap = std::unordered_map<std::pair<uint64_t, uint64_t>, float, 39fe6060f1SDimitry Andric pair_hash<uint64_t, uint64_t>>; 40d409305fSDimitry Andric using FuncProbeFactorMap = StringMap<ProbeFactorMap>; 41e8d8bef9SDimitry Andric 42e8d8bef9SDimitry Andric 43d409305fSDimitry Andric // A pseudo probe verifier that can be run after each IR passes to detect the 44d409305fSDimitry Andric // violation of updating probe factors. In principle, the sum of distribution 45d409305fSDimitry Andric // factor for a probe should be identical before and after a pass. For a 46d409305fSDimitry Andric // function pass, the factor sum for a probe would be typically 100%. 47d409305fSDimitry Andric class PseudoProbeVerifier { 48d409305fSDimitry Andric public: 49d409305fSDimitry Andric void registerCallbacks(PassInstrumentationCallbacks &PIC); 50d409305fSDimitry Andric 51d409305fSDimitry Andric // Implementation of pass instrumentation callbacks for new pass manager. 52d409305fSDimitry Andric void runAfterPass(StringRef PassID, Any IR); 53d409305fSDimitry Andric 54d409305fSDimitry Andric private: 55d409305fSDimitry Andric // Allow a little bias due the rounding to integral factors. 56d409305fSDimitry Andric constexpr static float DistributionFactorVariance = 0.02f; 57d409305fSDimitry Andric // Distribution factors from last pass. 58d409305fSDimitry Andric FuncProbeFactorMap FunctionProbeFactors; 59d409305fSDimitry Andric 60d409305fSDimitry Andric void collectProbeFactors(const BasicBlock *BB, ProbeFactorMap &ProbeFactors); 61d409305fSDimitry Andric void runAfterPass(const Module *M); 62d409305fSDimitry Andric void runAfterPass(const LazyCallGraph::SCC *C); 63d409305fSDimitry Andric void runAfterPass(const Function *F); 64d409305fSDimitry Andric void runAfterPass(const Loop *L); 65d409305fSDimitry Andric bool shouldVerifyFunction(const Function *F); 66d409305fSDimitry Andric void verifyProbeFactors(const Function *F, 67d409305fSDimitry Andric const ProbeFactorMap &ProbeFactors); 68d409305fSDimitry Andric }; 69d409305fSDimitry Andric 70e8d8bef9SDimitry Andric /// Sample profile pseudo prober. 71e8d8bef9SDimitry Andric /// 72e8d8bef9SDimitry Andric /// Insert pseudo probes for block sampling and value sampling. 73e8d8bef9SDimitry Andric class SampleProfileProber { 74e8d8bef9SDimitry Andric public: 75e8d8bef9SDimitry Andric // Give an empty module id when the prober is not used for instrumentation. 76e8d8bef9SDimitry Andric SampleProfileProber(Function &F, const std::string &CurModuleUniqueId); 77e8d8bef9SDimitry Andric void instrumentOneFunc(Function &F, TargetMachine *TM); 78e8d8bef9SDimitry Andric 79e8d8bef9SDimitry Andric private: getFunction()80e8d8bef9SDimitry Andric Function *getFunction() const { return F; } getFunctionHash()81e8d8bef9SDimitry Andric uint64_t getFunctionHash() const { return FunctionHash; } 82e8d8bef9SDimitry Andric uint32_t getBlockId(const BasicBlock *BB) const; 83e8d8bef9SDimitry Andric uint32_t getCallsiteId(const Instruction *Call) const; 84e8d8bef9SDimitry Andric void computeCFGHash(); 85e8d8bef9SDimitry Andric void computeProbeIdForBlocks(); 86e8d8bef9SDimitry Andric void computeProbeIdForCallsites(); 87e8d8bef9SDimitry Andric 88e8d8bef9SDimitry Andric Function *F; 89e8d8bef9SDimitry Andric 90e8d8bef9SDimitry Andric /// The current module ID that is used to name a static object as a comdat 91e8d8bef9SDimitry Andric /// group. 92e8d8bef9SDimitry Andric std::string CurModuleUniqueId; 93e8d8bef9SDimitry Andric 94e8d8bef9SDimitry Andric /// A CFG hash code used to identify a function code changes. 95e8d8bef9SDimitry Andric uint64_t FunctionHash; 96e8d8bef9SDimitry Andric 97e8d8bef9SDimitry Andric /// Map basic blocks to the their pseudo probe ids. 98e8d8bef9SDimitry Andric BlockIdMap BlockProbeIds; 99e8d8bef9SDimitry Andric 100e8d8bef9SDimitry Andric /// Map indirect calls to the their pseudo probe ids. 101e8d8bef9SDimitry Andric InstructionIdMap CallProbeIds; 102e8d8bef9SDimitry Andric 103e8d8bef9SDimitry Andric /// The ID of the last probe, Can be used to number a new probe. 104e8d8bef9SDimitry Andric uint32_t LastProbeId; 105e8d8bef9SDimitry Andric }; 106e8d8bef9SDimitry Andric 107e8d8bef9SDimitry Andric class SampleProfileProbePass : public PassInfoMixin<SampleProfileProbePass> { 108e8d8bef9SDimitry Andric TargetMachine *TM; 109e8d8bef9SDimitry Andric 110e8d8bef9SDimitry Andric public: SampleProfileProbePass(TargetMachine * TM)111e8d8bef9SDimitry Andric SampleProfileProbePass(TargetMachine *TM) : TM(TM) {} 112e8d8bef9SDimitry Andric PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); 113e8d8bef9SDimitry Andric }; 114e8d8bef9SDimitry Andric 115fe6060f1SDimitry Andric // Pseudo probe distribution factor updater. 116fe6060f1SDimitry Andric // Sample profile annotation can happen in both LTO prelink and postlink. The 117fe6060f1SDimitry Andric // postlink-time re-annotation can degrade profile quality because of prelink 118fe6060f1SDimitry Andric // code duplication transformation, such as loop unrolling, jump threading, 119fe6060f1SDimitry Andric // indirect call promotion etc. As such, samples corresponding to a source 120fe6060f1SDimitry Andric // location may be aggregated multiple times in postlink. With a concept of 121fe6060f1SDimitry Andric // distribution factor for pseudo probes, samples can be distributed among 122fe6060f1SDimitry Andric // duplicated probes reasonable based on the assumption that optimizations 123fe6060f1SDimitry Andric // duplicating code well-maintain the branch frequency information (BFI). This 124fe6060f1SDimitry Andric // pass updates distribution factors for each pseudo probe at the end of the 125fe6060f1SDimitry Andric // prelink pipeline, to reflect an estimated portion of the real execution 126fe6060f1SDimitry Andric // count. 127d409305fSDimitry Andric class PseudoProbeUpdatePass : public PassInfoMixin<PseudoProbeUpdatePass> { 128d409305fSDimitry Andric void runOnFunction(Function &F, FunctionAnalysisManager &FAM); 129d409305fSDimitry Andric 130d409305fSDimitry Andric public: 1311fd87a68SDimitry Andric PseudoProbeUpdatePass() = default; 132d409305fSDimitry Andric PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); 133d409305fSDimitry Andric }; 134d409305fSDimitry Andric 135e8d8bef9SDimitry Andric } // end namespace llvm 136e8d8bef9SDimitry Andric #endif // LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H 137