1 //=-- SampleProf.cpp - Sample profiling format support --------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains common definitions used in the reading and writing of
10 // sample profile data.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/ProfileData/SampleProf.h"
15 #include "llvm/Config/llvm-config.h"
16 #include "llvm/IR/DebugInfoMetadata.h"
17 #include "llvm/IR/PseudoProbe.h"
18 #include "llvm/ProfileData/SampleProfReader.h"
19 #include "llvm/Support/Compiler.h"
20 #include "llvm/Support/Debug.h"
21 #include "llvm/Support/Error.h"
22 #include "llvm/Support/ErrorHandling.h"
23 #include "llvm/Support/LEB128.h"
24 #include "llvm/Support/ManagedStatic.h"
25 #include "llvm/Support/raw_ostream.h"
26 #include <string>
27 #include <system_error>
28 
29 using namespace llvm;
30 using namespace sampleprof;
31 
32 namespace llvm {
33 namespace sampleprof {
34 SampleProfileFormat FunctionSamples::Format;
35 bool FunctionSamples::ProfileIsProbeBased = false;
36 bool FunctionSamples::ProfileIsCS = false;
37 bool FunctionSamples::UseMD5;
38 } // namespace sampleprof
39 } // namespace llvm
40 
41 namespace {
42 
43 // FIXME: This class is only here to support the transition to llvm::Error. It
44 // will be removed once this transition is complete. Clients should prefer to
45 // deal with the Error value directly, rather than converting to error_code.
46 class SampleProfErrorCategoryType : public std::error_category {
47   const char *name() const noexcept override { return "llvm.sampleprof"; }
48 
49   std::string message(int IE) const override {
50     sampleprof_error E = static_cast<sampleprof_error>(IE);
51     switch (E) {
52     case sampleprof_error::success:
53       return "Success";
54     case sampleprof_error::bad_magic:
55       return "Invalid sample profile data (bad magic)";
56     case sampleprof_error::unsupported_version:
57       return "Unsupported sample profile format version";
58     case sampleprof_error::too_large:
59       return "Too much profile data";
60     case sampleprof_error::truncated:
61       return "Truncated profile data";
62     case sampleprof_error::malformed:
63       return "Malformed sample profile data";
64     case sampleprof_error::unrecognized_format:
65       return "Unrecognized sample profile encoding format";
66     case sampleprof_error::unsupported_writing_format:
67       return "Profile encoding format unsupported for writing operations";
68     case sampleprof_error::truncated_name_table:
69       return "Truncated function name table";
70     case sampleprof_error::not_implemented:
71       return "Unimplemented feature";
72     case sampleprof_error::counter_overflow:
73       return "Counter overflow";
74     case sampleprof_error::ostream_seek_unsupported:
75       return "Ostream does not support seek";
76     case sampleprof_error::compress_failed:
77       return "Compress failure";
78     case sampleprof_error::uncompress_failed:
79       return "Uncompress failure";
80     case sampleprof_error::zlib_unavailable:
81       return "Zlib is unavailable";
82     case sampleprof_error::hash_mismatch:
83       return "Function hash mismatch";
84     }
85     llvm_unreachable("A value of sampleprof_error has no message.");
86   }
87 };
88 
89 } // end anonymous namespace
90 
91 static ManagedStatic<SampleProfErrorCategoryType> ErrorCategory;
92 
93 const std::error_category &llvm::sampleprof_category() {
94   return *ErrorCategory;
95 }
96 
97 void LineLocation::print(raw_ostream &OS) const {
98   OS << LineOffset;
99   if (Discriminator > 0)
100     OS << "." << Discriminator;
101 }
102 
103 raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
104                                           const LineLocation &Loc) {
105   Loc.print(OS);
106   return OS;
107 }
108 
109 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
110 LLVM_DUMP_METHOD void LineLocation::dump() const { print(dbgs()); }
111 #endif
112 
113 /// Print the sample record to the stream \p OS indented by \p Indent.
114 void SampleRecord::print(raw_ostream &OS, unsigned Indent) const {
115   OS << NumSamples;
116   if (hasCalls()) {
117     OS << ", calls:";
118     for (const auto &I : getSortedCallTargets())
119       OS << " " << I.first << ":" << I.second;
120   }
121   OS << "\n";
122 }
123 
124 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
125 LLVM_DUMP_METHOD void SampleRecord::dump() const { print(dbgs(), 0); }
126 #endif
127 
128 raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
129                                           const SampleRecord &Sample) {
130   Sample.print(OS, 0);
131   return OS;
132 }
133 
134 /// Print the samples collected for a function on stream \p OS.
135 void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
136   if (getFunctionHash())
137     OS << "CFG checksum " << getFunctionHash() << "\n";
138 
139   OS << TotalSamples << ", " << TotalHeadSamples << ", " << BodySamples.size()
140      << " sampled lines\n";
141 
142   OS.indent(Indent);
143   if (!BodySamples.empty()) {
144     OS << "Samples collected in the function's body {\n";
145     SampleSorter<LineLocation, SampleRecord> SortedBodySamples(BodySamples);
146     for (const auto &SI : SortedBodySamples.get()) {
147       OS.indent(Indent + 2);
148       OS << SI->first << ": " << SI->second;
149     }
150     OS.indent(Indent);
151     OS << "}\n";
152   } else {
153     OS << "No samples collected in the function's body\n";
154   }
155 
156   OS.indent(Indent);
157   if (!CallsiteSamples.empty()) {
158     OS << "Samples collected in inlined callsites {\n";
159     SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
160         CallsiteSamples);
161     for (const auto &CS : SortedCallsiteSamples.get()) {
162       for (const auto &FS : CS->second) {
163         OS.indent(Indent + 2);
164         OS << CS->first << ": inlined callee: " << FS.second.getName() << ": ";
165         FS.second.print(OS, Indent + 4);
166       }
167     }
168     OS.indent(Indent);
169     OS << "}\n";
170   } else {
171     OS << "No inlined callsites in this function\n";
172   }
173 }
174 
175 raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
176                                           const FunctionSamples &FS) {
177   FS.print(OS);
178   return OS;
179 }
180 
181 unsigned FunctionSamples::getOffset(const DILocation *DIL) {
182   return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &
183       0xffff;
184 }
185 
186 LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL) {
187   if (FunctionSamples::ProfileIsProbeBased)
188     // In a pseudo-probe based profile, a callsite is simply represented by the
189     // ID of the probe associated with the call instruction. The probe ID is
190     // encoded in the Discriminator field of the call instruction's debug
191     // metadata.
192     return LineLocation(PseudoProbeDwarfDiscriminator::extractProbeIndex(
193                             DIL->getDiscriminator()),
194                         0);
195   else
196     return LineLocation(FunctionSamples::getOffset(DIL),
197                         DIL->getBaseDiscriminator());
198 }
199 
200 const FunctionSamples *FunctionSamples::findFunctionSamples(
201     const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper) const {
202   assert(DIL);
203   SmallVector<std::pair<LineLocation, StringRef>, 10> S;
204 
205   const DILocation *PrevDIL = DIL;
206   for (DIL = DIL->getInlinedAt(); DIL; DIL = DIL->getInlinedAt()) {
207     S.push_back(std::make_pair(
208         LineLocation(getOffset(DIL), DIL->getBaseDiscriminator()),
209         PrevDIL->getScope()->getSubprogram()->getLinkageName()));
210     PrevDIL = DIL;
211   }
212   if (S.size() == 0)
213     return this;
214   const FunctionSamples *FS = this;
215   for (int i = S.size() - 1; i >= 0 && FS != nullptr; i--) {
216     FS = FS->findFunctionSamplesAt(S[i].first, S[i].second, Remapper);
217   }
218   return FS;
219 }
220 
221 void FunctionSamples::findAllNames(DenseSet<StringRef> &NameSet) const {
222   NameSet.insert(Name);
223   for (const auto &BS : BodySamples)
224     for (const auto &TS : BS.second.getCallTargets())
225       NameSet.insert(TS.getKey());
226 
227   for (const auto &CS : CallsiteSamples) {
228     for (const auto &NameFS : CS.second) {
229       NameSet.insert(NameFS.first);
230       NameFS.second.findAllNames(NameSet);
231     }
232   }
233 }
234 
235 const FunctionSamples *FunctionSamples::findFunctionSamplesAt(
236     const LineLocation &Loc, StringRef CalleeName,
237     SampleProfileReaderItaniumRemapper *Remapper) const {
238   std::string CalleeGUID;
239   CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID);
240 
241   auto iter = CallsiteSamples.find(Loc);
242   if (iter == CallsiteSamples.end())
243     return nullptr;
244   auto FS = iter->second.find(CalleeName);
245   if (FS != iter->second.end())
246     return &FS->second;
247   if (Remapper) {
248     if (auto NameInProfile = Remapper->lookUpNameInProfile(CalleeName)) {
249       auto FS = iter->second.find(*NameInProfile);
250       if (FS != iter->second.end())
251         return &FS->second;
252     }
253   }
254   // If we cannot find exact match of the callee name, return the FS with
255   // the max total count. Only do this when CalleeName is not provided,
256   // i.e., only for indirect calls.
257   if (!CalleeName.empty())
258     return nullptr;
259   uint64_t MaxTotalSamples = 0;
260   const FunctionSamples *R = nullptr;
261   for (const auto &NameFS : iter->second)
262     if (NameFS.second.getTotalSamples() >= MaxTotalSamples) {
263       MaxTotalSamples = NameFS.second.getTotalSamples();
264       R = &NameFS.second;
265     }
266   return R;
267 }
268 
269 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
270 LLVM_DUMP_METHOD void FunctionSamples::dump() const { print(dbgs(), 0); }
271 #endif
272 
273 std::error_code ProfileSymbolList::read(const uint8_t *Data,
274                                         uint64_t ListSize) {
275   const char *ListStart = reinterpret_cast<const char *>(Data);
276   uint64_t Size = 0;
277   while (Size < ListSize) {
278     StringRef Str(ListStart + Size);
279     add(Str);
280     Size += Str.size() + 1;
281   }
282   if (Size != ListSize)
283     return sampleprof_error::malformed;
284   return sampleprof_error::success;
285 }
286 
287 std::error_code ProfileSymbolList::write(raw_ostream &OS) {
288   // Sort the symbols before output. If doing compression.
289   // It will make the compression much more effective.
290   std::vector<StringRef> SortedList(Syms.begin(), Syms.end());
291   llvm::sort(SortedList);
292 
293   std::string OutputString;
294   for (auto &Sym : SortedList) {
295     OutputString.append(Sym.str());
296     OutputString.append(1, '\0');
297   }
298 
299   OS << OutputString;
300   return sampleprof_error::success;
301 }
302 
303 void ProfileSymbolList::dump(raw_ostream &OS) const {
304   OS << "======== Dump profile symbol list ========\n";
305   std::vector<StringRef> SortedList(Syms.begin(), Syms.end());
306   llvm::sort(SortedList);
307 
308   for (auto &Sym : SortedList)
309     OS << Sym << "\n";
310 }
311