1 //===- SampleProf.h - Sampling profiling format support ---------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains common definitions used in the reading and writing of
10 // sample profile data.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #ifndef LLVM_PROFILEDATA_SAMPLEPROF_H
15 #define LLVM_PROFILEDATA_SAMPLEPROF_H
16
17 #include "llvm/ADT/DenseSet.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringMap.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/ADT/StringSet.h"
22 #include "llvm/IR/Function.h"
23 #include "llvm/IR/GlobalValue.h"
24 #include "llvm/IR/Module.h"
25 #include "llvm/Support/Allocator.h"
26 #include "llvm/Support/Debug.h"
27 #include "llvm/Support/ErrorOr.h"
28 #include "llvm/Support/MathExtras.h"
29 #include "llvm/Support/raw_ostream.h"
30 #include <algorithm>
31 #include <cstdint>
32 #include <map>
33 #include <set>
34 #include <string>
35 #include <system_error>
36 #include <utility>
37
38 namespace llvm {
39
40 const std::error_category &sampleprof_category();
41
42 enum class sampleprof_error {
43 success = 0,
44 bad_magic,
45 unsupported_version,
46 too_large,
47 truncated,
48 malformed,
49 unrecognized_format,
50 unsupported_writing_format,
51 truncated_name_table,
52 not_implemented,
53 counter_overflow,
54 ostream_seek_unsupported,
55 compress_failed,
56 uncompress_failed,
57 zlib_unavailable
58 };
59
make_error_code(sampleprof_error E)60 inline std::error_code make_error_code(sampleprof_error E) {
61 return std::error_code(static_cast<int>(E), sampleprof_category());
62 }
63
MergeResult(sampleprof_error & Accumulator,sampleprof_error Result)64 inline sampleprof_error MergeResult(sampleprof_error &Accumulator,
65 sampleprof_error Result) {
66 // Prefer first error encountered as later errors may be secondary effects of
67 // the initial problem.
68 if (Accumulator == sampleprof_error::success &&
69 Result != sampleprof_error::success)
70 Accumulator = Result;
71 return Accumulator;
72 }
73
74 } // end namespace llvm
75
76 namespace std {
77
78 template <>
79 struct is_error_code_enum<llvm::sampleprof_error> : std::true_type {};
80
81 } // end namespace std
82
83 namespace llvm {
84 namespace sampleprof {
85
86 enum SampleProfileFormat {
87 SPF_None = 0,
88 SPF_Text = 0x1,
89 SPF_Compact_Binary = 0x2,
90 SPF_GCC = 0x3,
91 SPF_Ext_Binary = 0x4,
92 SPF_Binary = 0xff
93 };
94
95 static inline uint64_t SPMagic(SampleProfileFormat Format = SPF_Binary) {
96 return uint64_t('S') << (64 - 8) | uint64_t('P') << (64 - 16) |
97 uint64_t('R') << (64 - 24) | uint64_t('O') << (64 - 32) |
98 uint64_t('F') << (64 - 40) | uint64_t('4') << (64 - 48) |
99 uint64_t('2') << (64 - 56) | uint64_t(Format);
100 }
101
102 /// Get the proper representation of a string according to whether the
103 /// current Format uses MD5 to represent the string.
104 static inline StringRef getRepInFormat(StringRef Name, bool UseMD5,
105 std::string &GUIDBuf) {
106 if (Name.empty())
107 return Name;
108 GUIDBuf = std::to_string(Function::getGUID(Name));
109 return UseMD5 ? StringRef(GUIDBuf) : Name;
110 }
111
112 static inline uint64_t SPVersion() { return 103; }
113
114 // Section Type used by SampleProfileExtBinaryBaseReader and
115 // SampleProfileExtBinaryBaseWriter. Never change the existing
116 // value of enum. Only append new ones.
117 enum SecType {
118 SecInValid = 0,
119 SecProfSummary = 1,
120 SecNameTable = 2,
121 SecProfileSymbolList = 3,
122 SecFuncOffsetTable = 4,
123 // marker for the first type of profile.
124 SecFuncProfileFirst = 32,
125 SecLBRProfile = SecFuncProfileFirst
126 };
127
128 static inline std::string getSecName(SecType Type) {
129 switch (Type) {
130 case SecInValid:
131 return "InvalidSection";
132 case SecProfSummary:
133 return "ProfileSummarySection";
134 case SecNameTable:
135 return "NameTableSection";
136 case SecProfileSymbolList:
137 return "ProfileSymbolListSection";
138 case SecFuncOffsetTable:
139 return "FuncOffsetTableSection";
140 case SecLBRProfile:
141 return "LBRProfileSection";
142 }
143 llvm_unreachable("A SecType has no name for output");
144 }
145
146 // Entry type of section header table used by SampleProfileExtBinaryBaseReader
147 // and SampleProfileExtBinaryBaseWriter.
148 struct SecHdrTableEntry {
149 SecType Type;
150 uint64_t Flags;
151 uint64_t Offset;
152 uint64_t Size;
153 };
154
155 // Flags common for all sections are defined here. In SecHdrTableEntry::Flags,
156 // common flags will be saved in the lower 32bits and section specific flags
157 // will be saved in the higher 32 bits.
158 enum class SecCommonFlags : uint32_t {
159 SecFlagInValid = 0,
160 SecFlagCompress = (1 << 0)
161 };
162
163 // Section specific flags are defined here.
164 // !!!Note: Everytime a new enum class is created here, please add
165 // a new check in verifySecFlag.
166 enum class SecNameTableFlags : uint32_t {
167 SecFlagInValid = 0,
168 SecFlagMD5Name = (1 << 0)
169 };
170 enum class SecProfSummaryFlags : uint32_t {
171 SecFlagInValid = 0,
172 /// SecFlagPartial means the profile is for common/shared code.
173 /// The common profile is usually merged from profiles collected
174 /// from running other targets.
175 SecFlagPartial = (1 << 0)
176 };
177
178 // Verify section specific flag is used for the correct section.
179 template <class SecFlagType>
180 static inline void verifySecFlag(SecType Type, SecFlagType Flag) {
181 // No verification is needed for common flags.
182 if (std::is_same<SecCommonFlags, SecFlagType>())
183 return;
184
185 // Verification starts here for section specific flag.
186 bool IsFlagLegal = false;
187 switch (Type) {
188 case SecNameTable:
189 IsFlagLegal = std::is_same<SecNameTableFlags, SecFlagType>();
190 break;
191 case SecProfSummary:
192 IsFlagLegal = std::is_same<SecProfSummaryFlags, SecFlagType>();
193 break;
194 default:
195 break;
196 }
197 if (!IsFlagLegal)
198 llvm_unreachable("Misuse of a flag in an incompatible section");
199 }
200
201 template <class SecFlagType>
202 static inline void addSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {
203 verifySecFlag(Entry.Type, Flag);
204 auto FVal = static_cast<uint64_t>(Flag);
205 bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
206 Entry.Flags |= IsCommon ? FVal : (FVal << 32);
207 }
208
209 template <class SecFlagType>
210 static inline void removeSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {
211 verifySecFlag(Entry.Type, Flag);
212 auto FVal = static_cast<uint64_t>(Flag);
213 bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
214 Entry.Flags &= ~(IsCommon ? FVal : (FVal << 32));
215 }
216
217 template <class SecFlagType>
218 static inline bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag) {
219 verifySecFlag(Entry.Type, Flag);
220 auto FVal = static_cast<uint64_t>(Flag);
221 bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
222 return Entry.Flags & (IsCommon ? FVal : (FVal << 32));
223 }
224
225 /// Represents the relative location of an instruction.
226 ///
227 /// Instruction locations are specified by the line offset from the
228 /// beginning of the function (marked by the line where the function
229 /// header is) and the discriminator value within that line.
230 ///
231 /// The discriminator value is useful to distinguish instructions
232 /// that are on the same line but belong to different basic blocks
233 /// (e.g., the two post-increment instructions in "if (p) x++; else y++;").
234 struct LineLocation {
235 LineLocation(uint32_t L, uint32_t D) : LineOffset(L), Discriminator(D) {}
236
237 void print(raw_ostream &OS) const;
238 void dump() const;
239
240 bool operator<(const LineLocation &O) const {
241 return LineOffset < O.LineOffset ||
242 (LineOffset == O.LineOffset && Discriminator < O.Discriminator);
243 }
244
245 uint32_t LineOffset;
246 uint32_t Discriminator;
247 };
248
249 raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc);
250
251 /// Representation of a single sample record.
252 ///
253 /// A sample record is represented by a positive integer value, which
254 /// indicates how frequently was the associated line location executed.
255 ///
256 /// Additionally, if the associated location contains a function call,
257 /// the record will hold a list of all the possible called targets. For
258 /// direct calls, this will be the exact function being invoked. For
259 /// indirect calls (function pointers, virtual table dispatch), this
260 /// will be a list of one or more functions.
261 class SampleRecord {
262 public:
263 using CallTarget = std::pair<StringRef, uint64_t>;
264 struct CallTargetComparator {
265 bool operator()(const CallTarget &LHS, const CallTarget &RHS) const {
266 if (LHS.second != RHS.second)
267 return LHS.second > RHS.second;
268
269 return LHS.first < RHS.first;
270 }
271 };
272
273 using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>;
274 using CallTargetMap = StringMap<uint64_t>;
275 SampleRecord() = default;
276
277 /// Increment the number of samples for this record by \p S.
278 /// Optionally scale sample count \p S by \p Weight.
279 ///
280 /// Sample counts accumulate using saturating arithmetic, to avoid wrapping
281 /// around unsigned integers.
282 sampleprof_error addSamples(uint64_t S, uint64_t Weight = 1) {
283 bool Overflowed;
284 NumSamples = SaturatingMultiplyAdd(S, Weight, NumSamples, &Overflowed);
285 return Overflowed ? sampleprof_error::counter_overflow
286 : sampleprof_error::success;
287 }
288
289 /// Add called function \p F with samples \p S.
290 /// Optionally scale sample count \p S by \p Weight.
291 ///
292 /// Sample counts accumulate using saturating arithmetic, to avoid wrapping
293 /// around unsigned integers.
294 sampleprof_error addCalledTarget(StringRef F, uint64_t S,
295 uint64_t Weight = 1) {
296 uint64_t &TargetSamples = CallTargets[F];
297 bool Overflowed;
298 TargetSamples =
299 SaturatingMultiplyAdd(S, Weight, TargetSamples, &Overflowed);
300 return Overflowed ? sampleprof_error::counter_overflow
301 : sampleprof_error::success;
302 }
303
304 /// Return true if this sample record contains function calls.
305 bool hasCalls() const { return !CallTargets.empty(); }
306
307 uint64_t getSamples() const { return NumSamples; }
308 const CallTargetMap &getCallTargets() const { return CallTargets; }
309 const SortedCallTargetSet getSortedCallTargets() const {
310 return SortCallTargets(CallTargets);
311 }
312
313 /// Sort call targets in descending order of call frequency.
314 static const SortedCallTargetSet SortCallTargets(const CallTargetMap &Targets) {
315 SortedCallTargetSet SortedTargets;
316 for (const auto &I : Targets) {
317 SortedTargets.emplace(I.first(), I.second);
318 }
319 return SortedTargets;
320 }
321
322 /// Merge the samples in \p Other into this record.
323 /// Optionally scale sample counts by \p Weight.
324 sampleprof_error merge(const SampleRecord &Other, uint64_t Weight = 1) {
325 sampleprof_error Result = addSamples(Other.getSamples(), Weight);
326 for (const auto &I : Other.getCallTargets()) {
327 MergeResult(Result, addCalledTarget(I.first(), I.second, Weight));
328 }
329 return Result;
330 }
331
332 void print(raw_ostream &OS, unsigned Indent) const;
333 void dump() const;
334
335 private:
336 uint64_t NumSamples = 0;
337 CallTargetMap CallTargets;
338 };
339
340 raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample);
341
342 class FunctionSamples;
343 class SampleProfileReaderItaniumRemapper;
344
345 using BodySampleMap = std::map<LineLocation, SampleRecord>;
346 // NOTE: Using a StringMap here makes parsed profiles consume around 17% more
347 // memory, which is *very* significant for large profiles.
348 using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>;
349 using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;
350
351 /// Representation of the samples collected for a function.
352 ///
353 /// This data structure contains all the collected samples for the body
354 /// of a function. Each sample corresponds to a LineLocation instance
355 /// within the body of the function.
356 class FunctionSamples {
357 public:
358 FunctionSamples() = default;
359
360 void print(raw_ostream &OS = dbgs(), unsigned Indent = 0) const;
361 void dump() const;
362
363 sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight = 1) {
364 bool Overflowed;
365 TotalSamples =
366 SaturatingMultiplyAdd(Num, Weight, TotalSamples, &Overflowed);
367 return Overflowed ? sampleprof_error::counter_overflow
368 : sampleprof_error::success;
369 }
370
371 sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) {
372 bool Overflowed;
373 TotalHeadSamples =
374 SaturatingMultiplyAdd(Num, Weight, TotalHeadSamples, &Overflowed);
375 return Overflowed ? sampleprof_error::counter_overflow
376 : sampleprof_error::success;
377 }
378
379 sampleprof_error addBodySamples(uint32_t LineOffset, uint32_t Discriminator,
380 uint64_t Num, uint64_t Weight = 1) {
381 return BodySamples[LineLocation(LineOffset, Discriminator)].addSamples(
382 Num, Weight);
383 }
384
385 sampleprof_error addCalledTargetSamples(uint32_t LineOffset,
386 uint32_t Discriminator,
387 StringRef FName, uint64_t Num,
388 uint64_t Weight = 1) {
389 return BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(
390 FName, Num, Weight);
391 }
392
393 /// Return the number of samples collected at the given location.
394 /// Each location is specified by \p LineOffset and \p Discriminator.
395 /// If the location is not found in profile, return error.
396 ErrorOr<uint64_t> findSamplesAt(uint32_t LineOffset,
397 uint32_t Discriminator) const {
398 const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator));
399 if (ret == BodySamples.end())
400 return std::error_code();
401 else
402 return ret->second.getSamples();
403 }
404
405 /// Returns the call target map collected at a given location.
406 /// Each location is specified by \p LineOffset and \p Discriminator.
407 /// If the location is not found in profile, return error.
408 ErrorOr<SampleRecord::CallTargetMap>
409 findCallTargetMapAt(uint32_t LineOffset, uint32_t Discriminator) const {
410 const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator));
411 if (ret == BodySamples.end())
412 return std::error_code();
413 return ret->second.getCallTargets();
414 }
415
416 /// Return the function samples at the given callsite location.
417 FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) {
418 return CallsiteSamples[Loc];
419 }
420
421 /// Returns the FunctionSamplesMap at the given \p Loc.
422 const FunctionSamplesMap *
423 findFunctionSamplesMapAt(const LineLocation &Loc) const {
424 auto iter = CallsiteSamples.find(Loc);
425 if (iter == CallsiteSamples.end())
426 return nullptr;
427 return &iter->second;
428 }
429
430 /// Returns a pointer to FunctionSamples at the given callsite location
431 /// \p Loc with callee \p CalleeName. If no callsite can be found, relax
432 /// the restriction to return the FunctionSamples at callsite location
433 /// \p Loc with the maximum total sample count. If \p Remapper is not
434 /// nullptr, use \p Remapper to find FunctionSamples with equivalent name
435 /// as \p CalleeName.
436 const FunctionSamples *
437 findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName,
438 SampleProfileReaderItaniumRemapper *Remapper) const;
439
440 bool empty() const { return TotalSamples == 0; }
441
442 /// Return the total number of samples collected inside the function.
443 uint64_t getTotalSamples() const { return TotalSamples; }
444
445 /// Return the total number of branch samples that have the function as the
446 /// branch target. This should be equivalent to the sample of the first
447 /// instruction of the symbol. But as we directly get this info for raw
448 /// profile without referring to potentially inaccurate debug info, this
449 /// gives more accurate profile data and is preferred for standalone symbols.
450 uint64_t getHeadSamples() const { return TotalHeadSamples; }
451
452 /// Return the sample count of the first instruction of the function.
453 /// The function can be either a standalone symbol or an inlined function.
454 uint64_t getEntrySamples() const {
455 uint64_t Count = 0;
456 // Use either BodySamples or CallsiteSamples which ever has the smaller
457 // lineno.
458 if (!BodySamples.empty() &&
459 (CallsiteSamples.empty() ||
460 BodySamples.begin()->first < CallsiteSamples.begin()->first))
461 Count = BodySamples.begin()->second.getSamples();
462 else if (!CallsiteSamples.empty()) {
463 // An indirect callsite may be promoted to several inlined direct calls.
464 // We need to get the sum of them.
465 for (const auto &N_FS : CallsiteSamples.begin()->second)
466 Count += N_FS.second.getEntrySamples();
467 }
468 // Return at least 1 if total sample is not 0.
469 return Count ? Count : TotalSamples > 0;
470 }
471
472 /// Return all the samples collected in the body of the function.
473 const BodySampleMap &getBodySamples() const { return BodySamples; }
474
475 /// Return all the callsite samples collected in the body of the function.
476 const CallsiteSampleMap &getCallsiteSamples() const {
477 return CallsiteSamples;
478 }
479
480 /// Return the maximum of sample counts in a function body including functions
481 /// inlined in it.
482 uint64_t getMaxCountInside() const {
483 uint64_t MaxCount = 0;
484 for (const auto &L : getBodySamples())
485 MaxCount = std::max(MaxCount, L.second.getSamples());
486 for (const auto &C : getCallsiteSamples())
487 for (const FunctionSamplesMap::value_type &F : C.second)
488 MaxCount = std::max(MaxCount, F.second.getMaxCountInside());
489 return MaxCount;
490 }
491
492 /// Merge the samples in \p Other into this one.
493 /// Optionally scale samples by \p Weight.
494 sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight = 1) {
495 sampleprof_error Result = sampleprof_error::success;
496 Name = Other.getName();
497 if (!GUIDToFuncNameMap)
498 GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
499 MergeResult(Result, addTotalSamples(Other.getTotalSamples(), Weight));
500 MergeResult(Result, addHeadSamples(Other.getHeadSamples(), Weight));
501 for (const auto &I : Other.getBodySamples()) {
502 const LineLocation &Loc = I.first;
503 const SampleRecord &Rec = I.second;
504 MergeResult(Result, BodySamples[Loc].merge(Rec, Weight));
505 }
506 for (const auto &I : Other.getCallsiteSamples()) {
507 const LineLocation &Loc = I.first;
508 FunctionSamplesMap &FSMap = functionSamplesAt(Loc);
509 for (const auto &Rec : I.second)
510 MergeResult(Result, FSMap[Rec.first].merge(Rec.second, Weight));
511 }
512 return Result;
513 }
514
515 /// Recursively traverses all children, if the total sample count of the
516 /// corresponding function is no less than \p Threshold, add its corresponding
517 /// GUID to \p S. Also traverse the BodySamples to add hot CallTarget's GUID
518 /// to \p S.
519 void findInlinedFunctions(DenseSet<GlobalValue::GUID> &S, const Module *M,
520 uint64_t Threshold) const {
521 if (TotalSamples <= Threshold)
522 return;
523 auto isDeclaration = [](const Function *F) {
524 return !F || F->isDeclaration();
525 };
526 if (isDeclaration(M->getFunction(getFuncName()))) {
527 // Add to the import list only when it's defined out of module.
528 S.insert(getGUID(Name));
529 }
530 // Import hot CallTargets, which may not be available in IR because full
531 // profile annotation cannot be done until backend compilation in ThinLTO.
532 for (const auto &BS : BodySamples)
533 for (const auto &TS : BS.second.getCallTargets())
534 if (TS.getValue() > Threshold) {
535 const Function *Callee = M->getFunction(getFuncName(TS.getKey()));
536 if (isDeclaration(Callee))
537 S.insert(getGUID(TS.getKey()));
538 }
539 for (const auto &CS : CallsiteSamples)
540 for (const auto &NameFS : CS.second)
541 NameFS.second.findInlinedFunctions(S, M, Threshold);
542 }
543
544 /// Set the name of the function.
545 void setName(StringRef FunctionName) { Name = FunctionName; }
546
547 /// Return the function name.
548 StringRef getName() const { return Name; }
549
550 /// Return the original function name.
551 StringRef getFuncName() const { return getFuncName(Name); }
552
553 /// Return the canonical name for a function, taking into account
554 /// suffix elision policy attributes.
555 static StringRef getCanonicalFnName(const Function &F) {
556 static const char *knownSuffixes[] = { ".llvm.", ".part." };
557 auto AttrName = "sample-profile-suffix-elision-policy";
558 auto Attr = F.getFnAttribute(AttrName).getValueAsString();
559 if (Attr == "" || Attr == "all") {
560 return F.getName().split('.').first;
561 } else if (Attr == "selected") {
562 StringRef Cand(F.getName());
563 for (const auto &Suf : knownSuffixes) {
564 StringRef Suffix(Suf);
565 auto It = Cand.rfind(Suffix);
566 if (It == StringRef::npos)
567 return Cand;
568 auto Dit = Cand.rfind('.');
569 if (Dit == It + Suffix.size() - 1)
570 Cand = Cand.substr(0, It);
571 }
572 return Cand;
573 } else if (Attr == "none") {
574 return F.getName();
575 } else {
576 assert(false && "internal error: unknown suffix elision policy");
577 }
578 return F.getName();
579 }
580
581 /// Translate \p Name into its original name.
582 /// When profile doesn't use MD5, \p Name needs no translation.
583 /// When profile uses MD5, \p Name in current FunctionSamples
584 /// is actually GUID of the original function name. getFuncName will
585 /// translate \p Name in current FunctionSamples into its original name
586 /// by looking up in the function map GUIDToFuncNameMap.
587 /// If the original name doesn't exist in the map, return empty StringRef.
588 StringRef getFuncName(StringRef Name) const {
589 if (!UseMD5)
590 return Name;
591
592 assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be popluated first");
593 auto iter = GUIDToFuncNameMap->find(std::stoull(Name.data()));
594 if (iter == GUIDToFuncNameMap->end())
595 return StringRef();
596 return iter->second;
597 }
598
599 /// Returns the line offset to the start line of the subprogram.
600 /// We assume that a single function will not exceed 65535 LOC.
601 static unsigned getOffset(const DILocation *DIL);
602
603 /// Get the FunctionSamples of the inline instance where DIL originates
604 /// from.
605 ///
606 /// The FunctionSamples of the instruction (Machine or IR) associated to
607 /// \p DIL is the inlined instance in which that instruction is coming from.
608 /// We traverse the inline stack of that instruction, and match it with the
609 /// tree nodes in the profile.
610 ///
611 /// \returns the FunctionSamples pointer to the inlined instance.
612 /// If \p Remapper is not nullptr, it will be used to find matching
613 /// FunctionSamples with not exactly the same but equivalent name.
614 const FunctionSamples *findFunctionSamples(
615 const DILocation *DIL,
616 SampleProfileReaderItaniumRemapper *Remapper = nullptr) const;
617
618 static SampleProfileFormat Format;
619
620 /// Whether the profile uses MD5 to represent string.
621 static bool UseMD5;
622
623 /// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
624 /// all the function symbols defined or declared in current module.
625 DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap = nullptr;
626
627 // Assume the input \p Name is a name coming from FunctionSamples itself.
628 // If UseMD5 is true, the name is already a GUID and we
629 // don't want to return the GUID of GUID.
630 static uint64_t getGUID(StringRef Name) {
631 return UseMD5 ? std::stoull(Name.data()) : Function::getGUID(Name);
632 }
633
634 // Find all the names in the current FunctionSamples including names in
635 // all the inline instances and names of call targets.
636 void findAllNames(DenseSet<StringRef> &NameSet) const;
637
638 private:
639 /// Mangled name of the function.
640 StringRef Name;
641
642 /// Total number of samples collected inside this function.
643 ///
644 /// Samples are cumulative, they include all the samples collected
645 /// inside this function and all its inlined callees.
646 uint64_t TotalSamples = 0;
647
648 /// Total number of samples collected at the head of the function.
649 /// This is an approximation of the number of calls made to this function
650 /// at runtime.
651 uint64_t TotalHeadSamples = 0;
652
653 /// Map instruction locations to collected samples.
654 ///
655 /// Each entry in this map contains the number of samples
656 /// collected at the corresponding line offset. All line locations
657 /// are an offset from the start of the function.
658 BodySampleMap BodySamples;
659
660 /// Map call sites to collected samples for the called function.
661 ///
662 /// Each entry in this map corresponds to all the samples
663 /// collected for the inlined function call at the given
664 /// location. For example, given:
665 ///
666 /// void foo() {
667 /// 1 bar();
668 /// ...
669 /// 8 baz();
670 /// }
671 ///
672 /// If the bar() and baz() calls were inlined inside foo(), this
673 /// map will contain two entries. One for all the samples collected
674 /// in the call to bar() at line offset 1, the other for all the samples
675 /// collected in the call to baz() at line offset 8.
676 CallsiteSampleMap CallsiteSamples;
677 };
678
679 raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS);
680
681 /// Sort a LocationT->SampleT map by LocationT.
682 ///
683 /// It produces a sorted list of <LocationT, SampleT> records by ascending
684 /// order of LocationT.
685 template <class LocationT, class SampleT> class SampleSorter {
686 public:
687 using SamplesWithLoc = std::pair<const LocationT, SampleT>;
688 using SamplesWithLocList = SmallVector<const SamplesWithLoc *, 20>;
689
690 SampleSorter(const std::map<LocationT, SampleT> &Samples) {
691 for (const auto &I : Samples)
692 V.push_back(&I);
693 llvm::stable_sort(V, [](const SamplesWithLoc *A, const SamplesWithLoc *B) {
694 return A->first < B->first;
695 });
696 }
697
698 const SamplesWithLocList &get() const { return V; }
699
700 private:
701 SamplesWithLocList V;
702 };
703
704 /// ProfileSymbolList records the list of function symbols shown up
705 /// in the binary used to generate the profile. It is useful to
706 /// to discriminate a function being so cold as not to shown up
707 /// in the profile and a function newly added.
708 class ProfileSymbolList {
709 public:
710 /// copy indicates whether we need to copy the underlying memory
711 /// for the input Name.
712 void add(StringRef Name, bool copy = false) {
713 if (!copy) {
714 Syms.insert(Name);
715 return;
716 }
717 Syms.insert(Name.copy(Allocator));
718 }
719
720 bool contains(StringRef Name) { return Syms.count(Name); }
721
722 void merge(const ProfileSymbolList &List) {
723 for (auto Sym : List.Syms)
724 add(Sym, true);
725 }
726
727 unsigned size() { return Syms.size(); }
728
729 void setToCompress(bool TC) { ToCompress = TC; }
730 bool toCompress() { return ToCompress; }
731
732 std::error_code read(const uint8_t *Data, uint64_t ListSize);
733 std::error_code write(raw_ostream &OS);
734 void dump(raw_ostream &OS = dbgs()) const;
735
736 private:
737 // Determine whether or not to compress the symbol list when
738 // writing it into profile. The variable is unused when the symbol
739 // list is read from an existing profile.
740 bool ToCompress = false;
741 DenseSet<StringRef> Syms;
742 BumpPtrAllocator Allocator;
743 };
744
745 } // end namespace sampleprof
746 } // end namespace llvm
747
748 #endif // LLVM_PROFILEDATA_SAMPLEPROF_H
749