1 //===- ProfileSummaryInfo.cpp - Global profile summary information --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass that provides access to the global profile summary
10 // information.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Analysis/ProfileSummaryInfo.h"
15 #include "llvm/Analysis/BlockFrequencyInfo.h"
16 #include "llvm/IR/BasicBlock.h"
17 #include "llvm/IR/Instructions.h"
18 #include "llvm/IR/Module.h"
19 #include "llvm/IR/ProfileSummary.h"
20 #include "llvm/InitializePasses.h"
21 #include "llvm/ProfileData/ProfileCommon.h"
22 #include "llvm/Support/CommandLine.h"
23 #include <optional>
24 using namespace llvm;
25 
26 // Knobs for profile summary based thresholds.
27 namespace llvm {
28 extern cl::opt<int> ProfileSummaryCutoffHot;
29 extern cl::opt<int> ProfileSummaryCutoffCold;
30 extern cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold;
31 extern cl::opt<unsigned> ProfileSummaryLargeWorkingSetSizeThreshold;
32 extern cl::opt<int> ProfileSummaryHotCount;
33 extern cl::opt<int> ProfileSummaryColdCount;
34 } // namespace llvm
35 
36 static cl::opt<bool> PartialProfile(
37     "partial-profile", cl::Hidden, cl::init(false),
38     cl::desc("Specify the current profile is used as a partial profile."));
39 
40 cl::opt<bool> ScalePartialSampleProfileWorkingSetSize(
41     "scale-partial-sample-profile-working-set-size", cl::Hidden, cl::init(true),
42     cl::desc(
43         "If true, scale the working set size of the partial sample profile "
44         "by the partial profile ratio to reflect the size of the program "
45         "being compiled."));
46 
47 static cl::opt<double> PartialSampleProfileWorkingSetSizeScaleFactor(
48     "partial-sample-profile-working-set-size-scale-factor", cl::Hidden,
49     cl::init(0.008),
50     cl::desc("The scale factor used to scale the working set size of the "
51              "partial sample profile along with the partial profile ratio. "
52              "This includes the factor of the profile counter per block "
53              "and the factor to scale the working set size to use the same "
54              "shared thresholds as PGO."));
55 
56 // The profile summary metadata may be attached either by the frontend or by
57 // any backend passes (IR level instrumentation, for example). This method
58 // checks if the Summary is null and if so checks if the summary metadata is now
59 // available in the module and parses it to get the Summary object.
60 void ProfileSummaryInfo::refresh() {
61   if (hasProfileSummary())
62     return;
63   // First try to get context sensitive ProfileSummary.
64   auto *SummaryMD = M->getProfileSummary(/* IsCS */ true);
65   if (SummaryMD)
66     Summary.reset(ProfileSummary::getFromMD(SummaryMD));
67 
68   if (!hasProfileSummary()) {
69     // This will actually return PSK_Instr or PSK_Sample summary.
70     SummaryMD = M->getProfileSummary(/* IsCS */ false);
71     if (SummaryMD)
72       Summary.reset(ProfileSummary::getFromMD(SummaryMD));
73   }
74   if (!hasProfileSummary())
75     return;
76   computeThresholds();
77 }
78 
79 std::optional<uint64_t> ProfileSummaryInfo::getProfileCount(
80     const CallBase &Call, BlockFrequencyInfo *BFI, bool AllowSynthetic) const {
81   assert((isa<CallInst>(Call) || isa<InvokeInst>(Call)) &&
82          "We can only get profile count for call/invoke instruction.");
83   if (hasSampleProfile()) {
84     // In sample PGO mode, check if there is a profile metadata on the
85     // instruction. If it is present, determine hotness solely based on that,
86     // since the sampled entry count may not be accurate. If there is no
87     // annotated on the instruction, return std::nullopt.
88     uint64_t TotalCount;
89     if (Call.extractProfTotalWeight(TotalCount))
90       return TotalCount;
91     return std::nullopt;
92   }
93   if (BFI)
94     return BFI->getBlockProfileCount(Call.getParent(), AllowSynthetic);
95   return std::nullopt;
96 }
97 
98 bool ProfileSummaryInfo::isFunctionHotnessUnknown(const Function &F) const {
99   assert(hasPartialSampleProfile() && "Expect partial sample profile");
100   return !F.getEntryCount();
101 }
102 
103 /// Returns true if the function's entry is a cold. If it returns false, it
104 /// either means it is not cold or it is unknown whether it is cold or not (for
105 /// example, no profile data is available).
106 bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) const {
107   if (!F)
108     return false;
109   if (F->hasFnAttribute(Attribute::Cold))
110     return true;
111   if (!hasProfileSummary())
112     return false;
113   auto FunctionCount = F->getEntryCount();
114   // FIXME: The heuristic used below for determining coldness is based on
115   // preliminary SPEC tuning for inliner. This will eventually be a
116   // convenience method that calls isHotCount.
117   return FunctionCount && isColdCount(FunctionCount->getCount());
118 }
119 
120 /// Compute the hot and cold thresholds.
121 void ProfileSummaryInfo::computeThresholds() {
122   auto &DetailedSummary = Summary->getDetailedSummary();
123   auto &HotEntry = ProfileSummaryBuilder::getEntryForPercentile(
124       DetailedSummary, ProfileSummaryCutoffHot);
125   HotCountThreshold =
126       ProfileSummaryBuilder::getHotCountThreshold(DetailedSummary);
127   ColdCountThreshold =
128       ProfileSummaryBuilder::getColdCountThreshold(DetailedSummary);
129   assert(ColdCountThreshold <= HotCountThreshold &&
130          "Cold count threshold cannot exceed hot count threshold!");
131   if (!hasPartialSampleProfile() || !ScalePartialSampleProfileWorkingSetSize) {
132     HasHugeWorkingSetSize =
133         HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
134     HasLargeWorkingSetSize =
135         HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
136   } else {
137     // Scale the working set size of the partial sample profile to reflect the
138     // size of the program being compiled.
139     double PartialProfileRatio = Summary->getPartialProfileRatio();
140     uint64_t ScaledHotEntryNumCounts =
141         static_cast<uint64_t>(HotEntry.NumCounts * PartialProfileRatio *
142                               PartialSampleProfileWorkingSetSizeScaleFactor);
143     HasHugeWorkingSetSize =
144         ScaledHotEntryNumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
145     HasLargeWorkingSetSize =
146         ScaledHotEntryNumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
147   }
148 }
149 
150 std::optional<uint64_t>
151 ProfileSummaryInfo::computeThreshold(int PercentileCutoff) const {
152   if (!hasProfileSummary())
153     return std::nullopt;
154   auto iter = ThresholdCache.find(PercentileCutoff);
155   if (iter != ThresholdCache.end()) {
156     return iter->second;
157   }
158   auto &DetailedSummary = Summary->getDetailedSummary();
159   auto &Entry = ProfileSummaryBuilder::getEntryForPercentile(DetailedSummary,
160                                                              PercentileCutoff);
161   uint64_t CountThreshold = Entry.MinCount;
162   ThresholdCache[PercentileCutoff] = CountThreshold;
163   return CountThreshold;
164 }
165 
166 bool ProfileSummaryInfo::hasHugeWorkingSetSize() const {
167   return HasHugeWorkingSetSize && *HasHugeWorkingSetSize;
168 }
169 
170 bool ProfileSummaryInfo::hasLargeWorkingSetSize() const {
171   return HasLargeWorkingSetSize && *HasLargeWorkingSetSize;
172 }
173 
174 bool ProfileSummaryInfo::isHotCount(uint64_t C) const {
175   return HotCountThreshold && C >= *HotCountThreshold;
176 }
177 
178 bool ProfileSummaryInfo::isColdCount(uint64_t C) const {
179   return ColdCountThreshold && C <= *ColdCountThreshold;
180 }
181 
182 template <bool isHot>
183 bool ProfileSummaryInfo::isHotOrColdCountNthPercentile(int PercentileCutoff,
184                                                        uint64_t C) const {
185   auto CountThreshold = computeThreshold(PercentileCutoff);
186   if (isHot)
187     return CountThreshold && C >= *CountThreshold;
188   else
189     return CountThreshold && C <= *CountThreshold;
190 }
191 
192 bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff,
193                                                  uint64_t C) const {
194   return isHotOrColdCountNthPercentile<true>(PercentileCutoff, C);
195 }
196 
197 bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff,
198                                                   uint64_t C) const {
199   return isHotOrColdCountNthPercentile<false>(PercentileCutoff, C);
200 }
201 
202 uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() const {
203   return HotCountThreshold.value_or(UINT64_MAX);
204 }
205 
206 uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() const {
207   return ColdCountThreshold.value_or(0);
208 }
209 
210 bool ProfileSummaryInfo::isHotCallSite(const CallBase &CB,
211                                        BlockFrequencyInfo *BFI) const {
212   auto C = getProfileCount(CB, BFI);
213   return C && isHotCount(*C);
214 }
215 
216 bool ProfileSummaryInfo::isColdCallSite(const CallBase &CB,
217                                         BlockFrequencyInfo *BFI) const {
218   auto C = getProfileCount(CB, BFI);
219   if (C)
220     return isColdCount(*C);
221 
222   // In SamplePGO, if the caller has been sampled, and there is no profile
223   // annotated on the callsite, we consider the callsite as cold.
224   return hasSampleProfile() && CB.getCaller()->hasProfileData();
225 }
226 
227 bool ProfileSummaryInfo::hasPartialSampleProfile() const {
228   return hasProfileSummary() &&
229          Summary->getKind() == ProfileSummary::PSK_Sample &&
230          (PartialProfile || Summary->isPartialProfile());
231 }
232 
233 INITIALIZE_PASS(ProfileSummaryInfoWrapperPass, "profile-summary-info",
234                 "Profile summary info", false, true)
235 
236 ProfileSummaryInfoWrapperPass::ProfileSummaryInfoWrapperPass()
237     : ImmutablePass(ID) {
238   initializeProfileSummaryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
239 }
240 
241 bool ProfileSummaryInfoWrapperPass::doInitialization(Module &M) {
242   PSI.reset(new ProfileSummaryInfo(M));
243   return false;
244 }
245 
246 bool ProfileSummaryInfoWrapperPass::doFinalization(Module &M) {
247   PSI.reset();
248   return false;
249 }
250 
251 AnalysisKey ProfileSummaryAnalysis::Key;
252 ProfileSummaryInfo ProfileSummaryAnalysis::run(Module &M,
253                                                ModuleAnalysisManager &) {
254   return ProfileSummaryInfo(M);
255 }
256 
257 PreservedAnalyses ProfileSummaryPrinterPass::run(Module &M,
258                                                  ModuleAnalysisManager &AM) {
259   ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
260 
261   OS << "Functions in " << M.getName() << " with hot/cold annotations: \n";
262   for (auto &F : M) {
263     OS << F.getName();
264     if (PSI.isFunctionEntryHot(&F))
265       OS << " :hot entry ";
266     else if (PSI.isFunctionEntryCold(&F))
267       OS << " :cold entry ";
268     OS << "\n";
269   }
270   return PreservedAnalyses::all();
271 }
272 
273 char ProfileSummaryInfoWrapperPass::ID = 0;
274