1 //===- llvm-profdata.cpp - LLVM profile data tool -------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // llvm-profdata merges .profdata files.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/ADT/SmallSet.h"
14 #include "llvm/ADT/SmallVector.h"
15 #include "llvm/ADT/StringRef.h"
16 #include "llvm/IR/LLVMContext.h"
17 #include "llvm/ProfileData/InstrProfReader.h"
18 #include "llvm/ProfileData/InstrProfWriter.h"
19 #include "llvm/ProfileData/ProfileCommon.h"
20 #include "llvm/ProfileData/SampleProfReader.h"
21 #include "llvm/ProfileData/SampleProfWriter.h"
22 #include "llvm/Support/CommandLine.h"
23 #include "llvm/Support/Errc.h"
24 #include "llvm/Support/FileSystem.h"
25 #include "llvm/Support/Format.h"
26 #include "llvm/Support/FormattedStream.h"
27 #include "llvm/Support/InitLLVM.h"
28 #include "llvm/Support/MemoryBuffer.h"
29 #include "llvm/Support/Path.h"
30 #include "llvm/Support/ThreadPool.h"
31 #include "llvm/Support/Threading.h"
32 #include "llvm/Support/WithColor.h"
33 #include "llvm/Support/raw_ostream.h"
34 #include <algorithm>
35
36 using namespace llvm;
37
38 enum ProfileFormat {
39 PF_None = 0,
40 PF_Text,
41 PF_Compact_Binary,
42 PF_Ext_Binary,
43 PF_GCC,
44 PF_Binary
45 };
46
warn(Twine Message,std::string Whence="",std::string Hint="")47 static void warn(Twine Message, std::string Whence = "",
48 std::string Hint = "") {
49 WithColor::warning();
50 if (!Whence.empty())
51 errs() << Whence << ": ";
52 errs() << Message << "\n";
53 if (!Hint.empty())
54 WithColor::note() << Hint << "\n";
55 }
56
exitWithError(Twine Message,std::string Whence="",std::string Hint="")57 static void exitWithError(Twine Message, std::string Whence = "",
58 std::string Hint = "") {
59 WithColor::error();
60 if (!Whence.empty())
61 errs() << Whence << ": ";
62 errs() << Message << "\n";
63 if (!Hint.empty())
64 WithColor::note() << Hint << "\n";
65 ::exit(1);
66 }
67
exitWithError(Error E,StringRef Whence="")68 static void exitWithError(Error E, StringRef Whence = "") {
69 if (E.isA<InstrProfError>()) {
70 handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
71 instrprof_error instrError = IPE.get();
72 StringRef Hint = "";
73 if (instrError == instrprof_error::unrecognized_format) {
74 // Hint for common error of forgetting --sample for sample profiles.
75 Hint = "Perhaps you forgot to use the --sample option?";
76 }
77 exitWithError(IPE.message(), std::string(Whence), std::string(Hint));
78 });
79 }
80
81 exitWithError(toString(std::move(E)), std::string(Whence));
82 }
83
exitWithErrorCode(std::error_code EC,StringRef Whence="")84 static void exitWithErrorCode(std::error_code EC, StringRef Whence = "") {
85 exitWithError(EC.message(), std::string(Whence));
86 }
87
88 namespace {
89 enum ProfileKinds { instr, sample };
90 enum FailureMode { failIfAnyAreInvalid, failIfAllAreInvalid };
91 }
92
warnOrExitGivenError(FailureMode FailMode,std::error_code EC,StringRef Whence="")93 static void warnOrExitGivenError(FailureMode FailMode, std::error_code EC,
94 StringRef Whence = "") {
95 if (FailMode == failIfAnyAreInvalid)
96 exitWithErrorCode(EC, Whence);
97 else
98 warn(EC.message(), std::string(Whence));
99 }
100
handleMergeWriterError(Error E,StringRef WhenceFile="",StringRef WhenceFunction="",bool ShowHint=true)101 static void handleMergeWriterError(Error E, StringRef WhenceFile = "",
102 StringRef WhenceFunction = "",
103 bool ShowHint = true) {
104 if (!WhenceFile.empty())
105 errs() << WhenceFile << ": ";
106 if (!WhenceFunction.empty())
107 errs() << WhenceFunction << ": ";
108
109 auto IPE = instrprof_error::success;
110 E = handleErrors(std::move(E),
111 [&IPE](std::unique_ptr<InstrProfError> E) -> Error {
112 IPE = E->get();
113 return Error(std::move(E));
114 });
115 errs() << toString(std::move(E)) << "\n";
116
117 if (ShowHint) {
118 StringRef Hint = "";
119 if (IPE != instrprof_error::success) {
120 switch (IPE) {
121 case instrprof_error::hash_mismatch:
122 case instrprof_error::count_mismatch:
123 case instrprof_error::value_site_count_mismatch:
124 Hint = "Make sure that all profile data to be merged is generated "
125 "from the same binary.";
126 break;
127 default:
128 break;
129 }
130 }
131
132 if (!Hint.empty())
133 errs() << Hint << "\n";
134 }
135 }
136
137 namespace {
138 /// A remapper from original symbol names to new symbol names based on a file
139 /// containing a list of mappings from old name to new name.
140 class SymbolRemapper {
141 std::unique_ptr<MemoryBuffer> File;
142 DenseMap<StringRef, StringRef> RemappingTable;
143
144 public:
145 /// Build a SymbolRemapper from a file containing a list of old/new symbols.
create(StringRef InputFile)146 static std::unique_ptr<SymbolRemapper> create(StringRef InputFile) {
147 auto BufOrError = MemoryBuffer::getFileOrSTDIN(InputFile);
148 if (!BufOrError)
149 exitWithErrorCode(BufOrError.getError(), InputFile);
150
151 auto Remapper = std::make_unique<SymbolRemapper>();
152 Remapper->File = std::move(BufOrError.get());
153
154 for (line_iterator LineIt(*Remapper->File, /*SkipBlanks=*/true, '#');
155 !LineIt.is_at_eof(); ++LineIt) {
156 std::pair<StringRef, StringRef> Parts = LineIt->split(' ');
157 if (Parts.first.empty() || Parts.second.empty() ||
158 Parts.second.count(' ')) {
159 exitWithError("unexpected line in remapping file",
160 (InputFile + ":" + Twine(LineIt.line_number())).str(),
161 "expected 'old_symbol new_symbol'");
162 }
163 Remapper->RemappingTable.insert(Parts);
164 }
165 return Remapper;
166 }
167
168 /// Attempt to map the given old symbol into a new symbol.
169 ///
170 /// \return The new symbol, or \p Name if no such symbol was found.
operator ()(StringRef Name)171 StringRef operator()(StringRef Name) {
172 StringRef New = RemappingTable.lookup(Name);
173 return New.empty() ? Name : New;
174 }
175 };
176 }
177
178 struct WeightedFile {
179 std::string Filename;
180 uint64_t Weight;
181 };
182 typedef SmallVector<WeightedFile, 5> WeightedFileVector;
183
184 /// Keep track of merged data and reported errors.
185 struct WriterContext {
186 std::mutex Lock;
187 InstrProfWriter Writer;
188 std::vector<std::pair<Error, std::string>> Errors;
189 std::mutex &ErrLock;
190 SmallSet<instrprof_error, 4> &WriterErrorCodes;
191
WriterContextWriterContext192 WriterContext(bool IsSparse, std::mutex &ErrLock,
193 SmallSet<instrprof_error, 4> &WriterErrorCodes)
194 : Lock(), Writer(IsSparse), Errors(), ErrLock(ErrLock),
195 WriterErrorCodes(WriterErrorCodes) {}
196 };
197
198 /// Computer the overlap b/w profile BaseFilename and TestFileName,
199 /// and store the program level result to Overlap.
overlapInput(const std::string & BaseFilename,const std::string & TestFilename,WriterContext * WC,OverlapStats & Overlap,const OverlapFuncFilters & FuncFilter,raw_fd_ostream & OS,bool IsCS)200 static void overlapInput(const std::string &BaseFilename,
201 const std::string &TestFilename, WriterContext *WC,
202 OverlapStats &Overlap,
203 const OverlapFuncFilters &FuncFilter,
204 raw_fd_ostream &OS, bool IsCS) {
205 auto ReaderOrErr = InstrProfReader::create(TestFilename);
206 if (Error E = ReaderOrErr.takeError()) {
207 // Skip the empty profiles by returning sliently.
208 instrprof_error IPE = InstrProfError::take(std::move(E));
209 if (IPE != instrprof_error::empty_raw_profile)
210 WC->Errors.emplace_back(make_error<InstrProfError>(IPE), TestFilename);
211 return;
212 }
213
214 auto Reader = std::move(ReaderOrErr.get());
215 for (auto &I : *Reader) {
216 OverlapStats FuncOverlap(OverlapStats::FunctionLevel);
217 FuncOverlap.setFuncInfo(I.Name, I.Hash);
218
219 WC->Writer.overlapRecord(std::move(I), Overlap, FuncOverlap, FuncFilter);
220 FuncOverlap.dump(OS);
221 }
222 }
223
224 /// Load an input into a writer context.
loadInput(const WeightedFile & Input,SymbolRemapper * Remapper,WriterContext * WC)225 static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
226 WriterContext *WC) {
227 std::unique_lock<std::mutex> CtxGuard{WC->Lock};
228
229 // Copy the filename, because llvm::ThreadPool copied the input "const
230 // WeightedFile &" by value, making a reference to the filename within it
231 // invalid outside of this packaged task.
232 std::string Filename = Input.Filename;
233
234 auto ReaderOrErr = InstrProfReader::create(Input.Filename);
235 if (Error E = ReaderOrErr.takeError()) {
236 // Skip the empty profiles by returning sliently.
237 instrprof_error IPE = InstrProfError::take(std::move(E));
238 if (IPE != instrprof_error::empty_raw_profile)
239 WC->Errors.emplace_back(make_error<InstrProfError>(IPE), Filename);
240 return;
241 }
242
243 auto Reader = std::move(ReaderOrErr.get());
244 bool IsIRProfile = Reader->isIRLevelProfile();
245 bool HasCSIRProfile = Reader->hasCSIRLevelProfile();
246 if (WC->Writer.setIsIRLevelProfile(IsIRProfile, HasCSIRProfile)) {
247 WC->Errors.emplace_back(
248 make_error<StringError>(
249 "Merge IR generated profile with Clang generated profile.",
250 std::error_code()),
251 Filename);
252 return;
253 }
254 WC->Writer.setInstrEntryBBEnabled(Reader->instrEntryBBEnabled());
255
256 for (auto &I : *Reader) {
257 if (Remapper)
258 I.Name = (*Remapper)(I.Name);
259 const StringRef FuncName = I.Name;
260 bool Reported = false;
261 WC->Writer.addRecord(std::move(I), Input.Weight, [&](Error E) {
262 if (Reported) {
263 consumeError(std::move(E));
264 return;
265 }
266 Reported = true;
267 // Only show hint the first time an error occurs.
268 instrprof_error IPE = InstrProfError::take(std::move(E));
269 std::unique_lock<std::mutex> ErrGuard{WC->ErrLock};
270 bool firstTime = WC->WriterErrorCodes.insert(IPE).second;
271 handleMergeWriterError(make_error<InstrProfError>(IPE), Input.Filename,
272 FuncName, firstTime);
273 });
274 }
275 if (Reader->hasError())
276 if (Error E = Reader->getError())
277 WC->Errors.emplace_back(std::move(E), Filename);
278 }
279
280 /// Merge the \p Src writer context into \p Dst.
mergeWriterContexts(WriterContext * Dst,WriterContext * Src)281 static void mergeWriterContexts(WriterContext *Dst, WriterContext *Src) {
282 for (auto &ErrorPair : Src->Errors)
283 Dst->Errors.push_back(std::move(ErrorPair));
284 Src->Errors.clear();
285
286 Dst->Writer.mergeRecordsFromWriter(std::move(Src->Writer), [&](Error E) {
287 instrprof_error IPE = InstrProfError::take(std::move(E));
288 std::unique_lock<std::mutex> ErrGuard{Dst->ErrLock};
289 bool firstTime = Dst->WriterErrorCodes.insert(IPE).second;
290 if (firstTime)
291 warn(toString(make_error<InstrProfError>(IPE)));
292 });
293 }
294
writeInstrProfile(StringRef OutputFilename,ProfileFormat OutputFormat,InstrProfWriter & Writer)295 static void writeInstrProfile(StringRef OutputFilename,
296 ProfileFormat OutputFormat,
297 InstrProfWriter &Writer) {
298 std::error_code EC;
299 raw_fd_ostream Output(OutputFilename.data(), EC, sys::fs::OF_None);
300 if (EC)
301 exitWithErrorCode(EC, OutputFilename);
302
303 if (OutputFormat == PF_Text) {
304 if (Error E = Writer.writeText(Output))
305 exitWithError(std::move(E));
306 } else {
307 Writer.write(Output);
308 }
309 }
310
mergeInstrProfile(const WeightedFileVector & Inputs,SymbolRemapper * Remapper,StringRef OutputFilename,ProfileFormat OutputFormat,bool OutputSparse,unsigned NumThreads,FailureMode FailMode)311 static void mergeInstrProfile(const WeightedFileVector &Inputs,
312 SymbolRemapper *Remapper,
313 StringRef OutputFilename,
314 ProfileFormat OutputFormat, bool OutputSparse,
315 unsigned NumThreads, FailureMode FailMode) {
316 if (OutputFilename.compare("-") == 0)
317 exitWithError("Cannot write indexed profdata format to stdout.");
318
319 if (OutputFormat != PF_Binary && OutputFormat != PF_Compact_Binary &&
320 OutputFormat != PF_Ext_Binary && OutputFormat != PF_Text)
321 exitWithError("Unknown format is specified.");
322
323 std::mutex ErrorLock;
324 SmallSet<instrprof_error, 4> WriterErrorCodes;
325
326 // If NumThreads is not specified, auto-detect a good default.
327 if (NumThreads == 0)
328 NumThreads = std::min(hardware_concurrency().compute_thread_count(),
329 unsigned((Inputs.size() + 1) / 2));
330 // FIXME: There's a bug here, where setting NumThreads = Inputs.size() fails
331 // the merge_empty_profile.test because the InstrProfWriter.ProfileKind isn't
332 // merged, thus the emitted file ends up with a PF_Unknown kind.
333
334 // Initialize the writer contexts.
335 SmallVector<std::unique_ptr<WriterContext>, 4> Contexts;
336 for (unsigned I = 0; I < NumThreads; ++I)
337 Contexts.emplace_back(std::make_unique<WriterContext>(
338 OutputSparse, ErrorLock, WriterErrorCodes));
339
340 if (NumThreads == 1) {
341 for (const auto &Input : Inputs)
342 loadInput(Input, Remapper, Contexts[0].get());
343 } else {
344 ThreadPool Pool(hardware_concurrency(NumThreads));
345
346 // Load the inputs in parallel (N/NumThreads serial steps).
347 unsigned Ctx = 0;
348 for (const auto &Input : Inputs) {
349 Pool.async(loadInput, Input, Remapper, Contexts[Ctx].get());
350 Ctx = (Ctx + 1) % NumThreads;
351 }
352 Pool.wait();
353
354 // Merge the writer contexts together (~ lg(NumThreads) serial steps).
355 unsigned Mid = Contexts.size() / 2;
356 unsigned End = Contexts.size();
357 assert(Mid > 0 && "Expected more than one context");
358 do {
359 for (unsigned I = 0; I < Mid; ++I)
360 Pool.async(mergeWriterContexts, Contexts[I].get(),
361 Contexts[I + Mid].get());
362 Pool.wait();
363 if (End & 1) {
364 Pool.async(mergeWriterContexts, Contexts[0].get(),
365 Contexts[End - 1].get());
366 Pool.wait();
367 }
368 End = Mid;
369 Mid /= 2;
370 } while (Mid > 0);
371 }
372
373 // Handle deferred errors encountered during merging. If the number of errors
374 // is equal to the number of inputs the merge failed.
375 unsigned NumErrors = 0;
376 for (std::unique_ptr<WriterContext> &WC : Contexts) {
377 for (auto &ErrorPair : WC->Errors) {
378 ++NumErrors;
379 warn(toString(std::move(ErrorPair.first)), ErrorPair.second);
380 }
381 }
382 if (NumErrors == Inputs.size() ||
383 (NumErrors > 0 && FailMode == failIfAnyAreInvalid))
384 exitWithError("No profiles could be merged.");
385
386 writeInstrProfile(OutputFilename, OutputFormat, Contexts[0]->Writer);
387 }
388
389 /// The profile entry for a function in instrumentation profile.
390 struct InstrProfileEntry {
391 uint64_t MaxCount = 0;
392 float ZeroCounterRatio = 0.0;
393 InstrProfRecord *ProfRecord;
394 InstrProfileEntry(InstrProfRecord *Record);
395 InstrProfileEntry() = default;
396 };
397
InstrProfileEntry(InstrProfRecord * Record)398 InstrProfileEntry::InstrProfileEntry(InstrProfRecord *Record) {
399 ProfRecord = Record;
400 uint64_t CntNum = Record->Counts.size();
401 uint64_t ZeroCntNum = 0;
402 for (size_t I = 0; I < CntNum; ++I) {
403 MaxCount = std::max(MaxCount, Record->Counts[I]);
404 ZeroCntNum += !Record->Counts[I];
405 }
406 ZeroCounterRatio = (float)ZeroCntNum / CntNum;
407 }
408
409 /// Either set all the counters in the instr profile entry \p IFE to -1
410 /// in order to drop the profile or scale up the counters in \p IFP to
411 /// be above hot threshold. We use the ratio of zero counters in the
412 /// profile of a function to decide the profile is helpful or harmful
413 /// for performance, and to choose whether to scale up or drop it.
updateInstrProfileEntry(InstrProfileEntry & IFE,uint64_t HotInstrThreshold,float ZeroCounterThreshold)414 static void updateInstrProfileEntry(InstrProfileEntry &IFE,
415 uint64_t HotInstrThreshold,
416 float ZeroCounterThreshold) {
417 InstrProfRecord *ProfRecord = IFE.ProfRecord;
418 if (!IFE.MaxCount || IFE.ZeroCounterRatio > ZeroCounterThreshold) {
419 // If all or most of the counters of the function are zero, the
420 // profile is unaccountable and shuld be dropped. Reset all the
421 // counters to be -1 and PGO profile-use will drop the profile.
422 // All counters being -1 also implies that the function is hot so
423 // PGO profile-use will also set the entry count metadata to be
424 // above hot threshold.
425 for (size_t I = 0; I < ProfRecord->Counts.size(); ++I)
426 ProfRecord->Counts[I] = -1;
427 return;
428 }
429
430 // Scale up the MaxCount to be multiple times above hot threshold.
431 const unsigned MultiplyFactor = 3;
432 uint64_t Numerator = HotInstrThreshold * MultiplyFactor;
433 uint64_t Denominator = IFE.MaxCount;
434 ProfRecord->scale(Numerator, Denominator, [&](instrprof_error E) {
435 warn(toString(make_error<InstrProfError>(E)));
436 });
437 }
438
439 const uint64_t ColdPercentileIdx = 15;
440 const uint64_t HotPercentileIdx = 11;
441
442 /// Adjust the instr profile in \p WC based on the sample profile in
443 /// \p Reader.
444 static void
adjustInstrProfile(std::unique_ptr<WriterContext> & WC,std::unique_ptr<sampleprof::SampleProfileReader> & Reader,unsigned SupplMinSizeThreshold,float ZeroCounterThreshold,unsigned InstrProfColdThreshold)445 adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
446 std::unique_ptr<sampleprof::SampleProfileReader> &Reader,
447 unsigned SupplMinSizeThreshold, float ZeroCounterThreshold,
448 unsigned InstrProfColdThreshold) {
449 // Function to its entry in instr profile.
450 StringMap<InstrProfileEntry> InstrProfileMap;
451 InstrProfSummaryBuilder IPBuilder(ProfileSummaryBuilder::DefaultCutoffs);
452 for (auto &PD : WC->Writer.getProfileData()) {
453 // Populate IPBuilder.
454 for (const auto &PDV : PD.getValue()) {
455 InstrProfRecord Record = PDV.second;
456 IPBuilder.addRecord(Record);
457 }
458
459 // If a function has multiple entries in instr profile, skip it.
460 if (PD.getValue().size() != 1)
461 continue;
462
463 // Initialize InstrProfileMap.
464 InstrProfRecord *R = &PD.getValue().begin()->second;
465 InstrProfileMap[PD.getKey()] = InstrProfileEntry(R);
466 }
467
468 ProfileSummary InstrPS = *IPBuilder.getSummary();
469 ProfileSummary SamplePS = Reader->getSummary();
470
471 // Compute cold thresholds for instr profile and sample profile.
472 uint64_t ColdSampleThreshold =
473 ProfileSummaryBuilder::getEntryForPercentile(
474 SamplePS.getDetailedSummary(),
475 ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx])
476 .MinCount;
477 uint64_t HotInstrThreshold =
478 ProfileSummaryBuilder::getEntryForPercentile(
479 InstrPS.getDetailedSummary(),
480 ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx])
481 .MinCount;
482 uint64_t ColdInstrThreshold =
483 InstrProfColdThreshold
484 ? InstrProfColdThreshold
485 : ProfileSummaryBuilder::getEntryForPercentile(
486 InstrPS.getDetailedSummary(),
487 ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx])
488 .MinCount;
489
490 // Find hot/warm functions in sample profile which is cold in instr profile
491 // and adjust the profiles of those functions in the instr profile.
492 for (const auto &PD : Reader->getProfiles()) {
493 StringRef FName = PD.getKey();
494 const sampleprof::FunctionSamples &FS = PD.getValue();
495 auto It = InstrProfileMap.find(FName);
496 if (FS.getHeadSamples() > ColdSampleThreshold &&
497 It != InstrProfileMap.end() &&
498 It->second.MaxCount <= ColdInstrThreshold &&
499 FS.getBodySamples().size() >= SupplMinSizeThreshold) {
500 updateInstrProfileEntry(It->second, HotInstrThreshold,
501 ZeroCounterThreshold);
502 }
503 }
504 }
505
506 /// The main function to supplement instr profile with sample profile.
507 /// \Inputs contains the instr profile. \p SampleFilename specifies the
508 /// sample profile. \p OutputFilename specifies the output profile name.
509 /// \p OutputFormat specifies the output profile format. \p OutputSparse
510 /// specifies whether to generate sparse profile. \p SupplMinSizeThreshold
511 /// specifies the minimal size for the functions whose profile will be
512 /// adjusted. \p ZeroCounterThreshold is the threshold to check whether
513 /// a function contains too many zero counters and whether its profile
514 /// should be dropped. \p InstrProfColdThreshold is the user specified
515 /// cold threshold which will override the cold threshold got from the
516 /// instr profile summary.
supplementInstrProfile(const WeightedFileVector & Inputs,StringRef SampleFilename,StringRef OutputFilename,ProfileFormat OutputFormat,bool OutputSparse,unsigned SupplMinSizeThreshold,float ZeroCounterThreshold,unsigned InstrProfColdThreshold)517 static void supplementInstrProfile(
518 const WeightedFileVector &Inputs, StringRef SampleFilename,
519 StringRef OutputFilename, ProfileFormat OutputFormat, bool OutputSparse,
520 unsigned SupplMinSizeThreshold, float ZeroCounterThreshold,
521 unsigned InstrProfColdThreshold) {
522 if (OutputFilename.compare("-") == 0)
523 exitWithError("Cannot write indexed profdata format to stdout.");
524 if (Inputs.size() != 1)
525 exitWithError("Expect one input to be an instr profile.");
526 if (Inputs[0].Weight != 1)
527 exitWithError("Expect instr profile doesn't have weight.");
528
529 StringRef InstrFilename = Inputs[0].Filename;
530
531 // Read sample profile.
532 LLVMContext Context;
533 auto ReaderOrErr =
534 sampleprof::SampleProfileReader::create(SampleFilename.str(), Context);
535 if (std::error_code EC = ReaderOrErr.getError())
536 exitWithErrorCode(EC, SampleFilename);
537 auto Reader = std::move(ReaderOrErr.get());
538 if (std::error_code EC = Reader->read())
539 exitWithErrorCode(EC, SampleFilename);
540
541 // Read instr profile.
542 std::mutex ErrorLock;
543 SmallSet<instrprof_error, 4> WriterErrorCodes;
544 auto WC = std::make_unique<WriterContext>(OutputSparse, ErrorLock,
545 WriterErrorCodes);
546 loadInput(Inputs[0], nullptr, WC.get());
547 if (WC->Errors.size() > 0)
548 exitWithError(std::move(WC->Errors[0].first), InstrFilename);
549
550 adjustInstrProfile(WC, Reader, SupplMinSizeThreshold, ZeroCounterThreshold,
551 InstrProfColdThreshold);
552 writeInstrProfile(OutputFilename, OutputFormat, WC->Writer);
553 }
554
555 /// Make a copy of the given function samples with all symbol names remapped
556 /// by the provided symbol remapper.
557 static sampleprof::FunctionSamples
remapSamples(const sampleprof::FunctionSamples & Samples,SymbolRemapper & Remapper,sampleprof_error & Error)558 remapSamples(const sampleprof::FunctionSamples &Samples,
559 SymbolRemapper &Remapper, sampleprof_error &Error) {
560 sampleprof::FunctionSamples Result;
561 Result.setName(Remapper(Samples.getName()));
562 Result.addTotalSamples(Samples.getTotalSamples());
563 Result.addHeadSamples(Samples.getHeadSamples());
564 for (const auto &BodySample : Samples.getBodySamples()) {
565 Result.addBodySamples(BodySample.first.LineOffset,
566 BodySample.first.Discriminator,
567 BodySample.second.getSamples());
568 for (const auto &Target : BodySample.second.getCallTargets()) {
569 Result.addCalledTargetSamples(BodySample.first.LineOffset,
570 BodySample.first.Discriminator,
571 Remapper(Target.first()), Target.second);
572 }
573 }
574 for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
575 sampleprof::FunctionSamplesMap &Target =
576 Result.functionSamplesAt(CallsiteSamples.first);
577 for (const auto &Callsite : CallsiteSamples.second) {
578 sampleprof::FunctionSamples Remapped =
579 remapSamples(Callsite.second, Remapper, Error);
580 MergeResult(Error,
581 Target[std::string(Remapped.getName())].merge(Remapped));
582 }
583 }
584 return Result;
585 }
586
587 static sampleprof::SampleProfileFormat FormatMap[] = {
588 sampleprof::SPF_None,
589 sampleprof::SPF_Text,
590 sampleprof::SPF_Compact_Binary,
591 sampleprof::SPF_Ext_Binary,
592 sampleprof::SPF_GCC,
593 sampleprof::SPF_Binary};
594
595 static std::unique_ptr<MemoryBuffer>
getInputFileBuf(const StringRef & InputFile)596 getInputFileBuf(const StringRef &InputFile) {
597 if (InputFile == "")
598 return {};
599
600 auto BufOrError = MemoryBuffer::getFileOrSTDIN(InputFile);
601 if (!BufOrError)
602 exitWithErrorCode(BufOrError.getError(), InputFile);
603
604 return std::move(*BufOrError);
605 }
606
populateProfileSymbolList(MemoryBuffer * Buffer,sampleprof::ProfileSymbolList & PSL)607 static void populateProfileSymbolList(MemoryBuffer *Buffer,
608 sampleprof::ProfileSymbolList &PSL) {
609 if (!Buffer)
610 return;
611
612 SmallVector<StringRef, 32> SymbolVec;
613 StringRef Data = Buffer->getBuffer();
614 Data.split(SymbolVec, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
615
616 for (StringRef symbol : SymbolVec)
617 PSL.add(symbol);
618 }
619
handleExtBinaryWriter(sampleprof::SampleProfileWriter & Writer,ProfileFormat OutputFormat,MemoryBuffer * Buffer,sampleprof::ProfileSymbolList & WriterList,bool CompressAllSections,bool UseMD5,bool GenPartialProfile)620 static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer,
621 ProfileFormat OutputFormat,
622 MemoryBuffer *Buffer,
623 sampleprof::ProfileSymbolList &WriterList,
624 bool CompressAllSections, bool UseMD5,
625 bool GenPartialProfile) {
626 populateProfileSymbolList(Buffer, WriterList);
627 if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary)
628 warn("Profile Symbol list is not empty but the output format is not "
629 "ExtBinary format. The list will be lost in the output. ");
630
631 Writer.setProfileSymbolList(&WriterList);
632
633 if (CompressAllSections) {
634 if (OutputFormat != PF_Ext_Binary)
635 warn("-compress-all-section is ignored. Specify -extbinary to enable it");
636 else
637 Writer.setToCompressAllSections();
638 }
639 if (UseMD5) {
640 if (OutputFormat != PF_Ext_Binary)
641 warn("-use-md5 is ignored. Specify -extbinary to enable it");
642 else
643 Writer.setUseMD5();
644 }
645 if (GenPartialProfile) {
646 if (OutputFormat != PF_Ext_Binary)
647 warn("-gen-partial-profile is ignored. Specify -extbinary to enable it");
648 else
649 Writer.setPartialProfile();
650 }
651 }
652
653 static void
mergeSampleProfile(const WeightedFileVector & Inputs,SymbolRemapper * Remapper,StringRef OutputFilename,ProfileFormat OutputFormat,StringRef ProfileSymbolListFile,bool CompressAllSections,bool UseMD5,bool GenPartialProfile,FailureMode FailMode)654 mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
655 StringRef OutputFilename, ProfileFormat OutputFormat,
656 StringRef ProfileSymbolListFile, bool CompressAllSections,
657 bool UseMD5, bool GenPartialProfile, FailureMode FailMode) {
658 using namespace sampleprof;
659 StringMap<FunctionSamples> ProfileMap;
660 SmallVector<std::unique_ptr<sampleprof::SampleProfileReader>, 5> Readers;
661 LLVMContext Context;
662 sampleprof::ProfileSymbolList WriterList;
663 for (const auto &Input : Inputs) {
664 auto ReaderOrErr = SampleProfileReader::create(Input.Filename, Context);
665 if (std::error_code EC = ReaderOrErr.getError()) {
666 warnOrExitGivenError(FailMode, EC, Input.Filename);
667 continue;
668 }
669
670 // We need to keep the readers around until after all the files are
671 // read so that we do not lose the function names stored in each
672 // reader's memory. The function names are needed to write out the
673 // merged profile map.
674 Readers.push_back(std::move(ReaderOrErr.get()));
675 const auto Reader = Readers.back().get();
676 if (std::error_code EC = Reader->read()) {
677 warnOrExitGivenError(FailMode, EC, Input.Filename);
678 Readers.pop_back();
679 continue;
680 }
681
682 StringMap<FunctionSamples> &Profiles = Reader->getProfiles();
683 for (StringMap<FunctionSamples>::iterator I = Profiles.begin(),
684 E = Profiles.end();
685 I != E; ++I) {
686 sampleprof_error Result = sampleprof_error::success;
687 FunctionSamples Remapped =
688 Remapper ? remapSamples(I->second, *Remapper, Result)
689 : FunctionSamples();
690 FunctionSamples &Samples = Remapper ? Remapped : I->second;
691 StringRef FName = Samples.getName();
692 MergeResult(Result, ProfileMap[FName].merge(Samples, Input.Weight));
693 if (Result != sampleprof_error::success) {
694 std::error_code EC = make_error_code(Result);
695 handleMergeWriterError(errorCodeToError(EC), Input.Filename, FName);
696 }
697 }
698
699 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList =
700 Reader->getProfileSymbolList();
701 if (ReaderList)
702 WriterList.merge(*ReaderList);
703 }
704 auto WriterOrErr =
705 SampleProfileWriter::create(OutputFilename, FormatMap[OutputFormat]);
706 if (std::error_code EC = WriterOrErr.getError())
707 exitWithErrorCode(EC, OutputFilename);
708
709 auto Writer = std::move(WriterOrErr.get());
710 // WriterList will have StringRef refering to string in Buffer.
711 // Make sure Buffer lives as long as WriterList.
712 auto Buffer = getInputFileBuf(ProfileSymbolListFile);
713 handleExtBinaryWriter(*Writer, OutputFormat, Buffer.get(), WriterList,
714 CompressAllSections, UseMD5, GenPartialProfile);
715 Writer->write(ProfileMap);
716 }
717
parseWeightedFile(const StringRef & WeightedFilename)718 static WeightedFile parseWeightedFile(const StringRef &WeightedFilename) {
719 StringRef WeightStr, FileName;
720 std::tie(WeightStr, FileName) = WeightedFilename.split(',');
721
722 uint64_t Weight;
723 if (WeightStr.getAsInteger(10, Weight) || Weight < 1)
724 exitWithError("Input weight must be a positive integer.");
725
726 return {std::string(FileName), Weight};
727 }
728
addWeightedInput(WeightedFileVector & WNI,const WeightedFile & WF)729 static void addWeightedInput(WeightedFileVector &WNI, const WeightedFile &WF) {
730 StringRef Filename = WF.Filename;
731 uint64_t Weight = WF.Weight;
732
733 // If it's STDIN just pass it on.
734 if (Filename == "-") {
735 WNI.push_back({std::string(Filename), Weight});
736 return;
737 }
738
739 llvm::sys::fs::file_status Status;
740 llvm::sys::fs::status(Filename, Status);
741 if (!llvm::sys::fs::exists(Status))
742 exitWithErrorCode(make_error_code(errc::no_such_file_or_directory),
743 Filename);
744 // If it's a source file, collect it.
745 if (llvm::sys::fs::is_regular_file(Status)) {
746 WNI.push_back({std::string(Filename), Weight});
747 return;
748 }
749
750 if (llvm::sys::fs::is_directory(Status)) {
751 std::error_code EC;
752 for (llvm::sys::fs::recursive_directory_iterator F(Filename, EC), E;
753 F != E && !EC; F.increment(EC)) {
754 if (llvm::sys::fs::is_regular_file(F->path())) {
755 addWeightedInput(WNI, {F->path(), Weight});
756 }
757 }
758 if (EC)
759 exitWithErrorCode(EC, Filename);
760 }
761 }
762
parseInputFilenamesFile(MemoryBuffer * Buffer,WeightedFileVector & WFV)763 static void parseInputFilenamesFile(MemoryBuffer *Buffer,
764 WeightedFileVector &WFV) {
765 if (!Buffer)
766 return;
767
768 SmallVector<StringRef, 8> Entries;
769 StringRef Data = Buffer->getBuffer();
770 Data.split(Entries, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
771 for (const StringRef &FileWeightEntry : Entries) {
772 StringRef SanitizedEntry = FileWeightEntry.trim(" \t\v\f\r");
773 // Skip comments.
774 if (SanitizedEntry.startswith("#"))
775 continue;
776 // If there's no comma, it's an unweighted profile.
777 else if (SanitizedEntry.find(',') == StringRef::npos)
778 addWeightedInput(WFV, {std::string(SanitizedEntry), 1});
779 else
780 addWeightedInput(WFV, parseWeightedFile(SanitizedEntry));
781 }
782 }
783
merge_main(int argc,const char * argv[])784 static int merge_main(int argc, const char *argv[]) {
785 cl::list<std::string> InputFilenames(cl::Positional,
786 cl::desc("<filename...>"));
787 cl::list<std::string> WeightedInputFilenames("weighted-input",
788 cl::desc("<weight>,<filename>"));
789 cl::opt<std::string> InputFilenamesFile(
790 "input-files", cl::init(""),
791 cl::desc("Path to file containing newline-separated "
792 "[<weight>,]<filename> entries"));
793 cl::alias InputFilenamesFileA("f", cl::desc("Alias for --input-files"),
794 cl::aliasopt(InputFilenamesFile));
795 cl::opt<bool> DumpInputFileList(
796 "dump-input-file-list", cl::init(false), cl::Hidden,
797 cl::desc("Dump the list of input files and their weights, then exit"));
798 cl::opt<std::string> RemappingFile("remapping-file", cl::value_desc("file"),
799 cl::desc("Symbol remapping file"));
800 cl::alias RemappingFileA("r", cl::desc("Alias for --remapping-file"),
801 cl::aliasopt(RemappingFile));
802 cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
803 cl::init("-"), cl::Required,
804 cl::desc("Output file"));
805 cl::alias OutputFilenameA("o", cl::desc("Alias for --output"),
806 cl::aliasopt(OutputFilename));
807 cl::opt<ProfileKinds> ProfileKind(
808 cl::desc("Profile kind:"), cl::init(instr),
809 cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
810 clEnumVal(sample, "Sample profile")));
811 cl::opt<ProfileFormat> OutputFormat(
812 cl::desc("Format of output profile"), cl::init(PF_Binary),
813 cl::values(
814 clEnumValN(PF_Binary, "binary", "Binary encoding (default)"),
815 clEnumValN(PF_Compact_Binary, "compbinary",
816 "Compact binary encoding"),
817 clEnumValN(PF_Ext_Binary, "extbinary", "Extensible binary encoding"),
818 clEnumValN(PF_Text, "text", "Text encoding"),
819 clEnumValN(PF_GCC, "gcc",
820 "GCC encoding (only meaningful for -sample)")));
821 cl::opt<FailureMode> FailureMode(
822 "failure-mode", cl::init(failIfAnyAreInvalid), cl::desc("Failure mode:"),
823 cl::values(clEnumValN(failIfAnyAreInvalid, "any",
824 "Fail if any profile is invalid."),
825 clEnumValN(failIfAllAreInvalid, "all",
826 "Fail only if all profiles are invalid.")));
827 cl::opt<bool> OutputSparse("sparse", cl::init(false),
828 cl::desc("Generate a sparse profile (only meaningful for -instr)"));
829 cl::opt<unsigned> NumThreads(
830 "num-threads", cl::init(0),
831 cl::desc("Number of merge threads to use (default: autodetect)"));
832 cl::alias NumThreadsA("j", cl::desc("Alias for --num-threads"),
833 cl::aliasopt(NumThreads));
834 cl::opt<std::string> ProfileSymbolListFile(
835 "prof-sym-list", cl::init(""),
836 cl::desc("Path to file containing the list of function symbols "
837 "used to populate profile symbol list"));
838 cl::opt<bool> CompressAllSections(
839 "compress-all-sections", cl::init(false), cl::Hidden,
840 cl::desc("Compress all sections when writing the profile (only "
841 "meaningful for -extbinary)"));
842 cl::opt<bool> UseMD5(
843 "use-md5", cl::init(false), cl::Hidden,
844 cl::desc("Choose to use MD5 to represent string in name table (only "
845 "meaningful for -extbinary)"));
846 cl::opt<bool> GenPartialProfile(
847 "gen-partial-profile", cl::init(false), cl::Hidden,
848 cl::desc("Generate a partial profile (only meaningful for -extbinary)"));
849 cl::opt<std::string> SupplInstrWithSample(
850 "supplement-instr-with-sample", cl::init(""), cl::Hidden,
851 cl::desc("Supplement an instr profile with sample profile, to correct "
852 "the profile unrepresentativeness issue. The sample "
853 "profile is the input of the flag. Output will be in instr "
854 "format (The flag only works with -instr)"));
855 cl::opt<float> ZeroCounterThreshold(
856 "zero-counter-threshold", cl::init(0.7), cl::Hidden,
857 cl::desc("For the function which is cold in instr profile but hot in "
858 "sample profile, if the ratio of the number of zero counters "
859 "divided by the the total number of counters is above the "
860 "threshold, the profile of the function will be regarded as "
861 "being harmful for performance and will be dropped. "));
862 cl::opt<unsigned> SupplMinSizeThreshold(
863 "suppl-min-size-threshold", cl::init(10), cl::Hidden,
864 cl::desc("If the size of a function is smaller than the threshold, "
865 "assume it can be inlined by PGO early inliner and it won't "
866 "be adjusted based on sample profile. "));
867 cl::opt<unsigned> InstrProfColdThreshold(
868 "instr-prof-cold-threshold", cl::init(0), cl::Hidden,
869 cl::desc("User specified cold threshold for instr profile which will "
870 "override the cold threshold got from profile summary. "));
871
872 cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n");
873
874 WeightedFileVector WeightedInputs;
875 for (StringRef Filename : InputFilenames)
876 addWeightedInput(WeightedInputs, {std::string(Filename), 1});
877 for (StringRef WeightedFilename : WeightedInputFilenames)
878 addWeightedInput(WeightedInputs, parseWeightedFile(WeightedFilename));
879
880 // Make sure that the file buffer stays alive for the duration of the
881 // weighted input vector's lifetime.
882 auto Buffer = getInputFileBuf(InputFilenamesFile);
883 parseInputFilenamesFile(Buffer.get(), WeightedInputs);
884
885 if (WeightedInputs.empty())
886 exitWithError("No input files specified. See " +
887 sys::path::filename(argv[0]) + " -help");
888
889 if (DumpInputFileList) {
890 for (auto &WF : WeightedInputs)
891 outs() << WF.Weight << "," << WF.Filename << "\n";
892 return 0;
893 }
894
895 std::unique_ptr<SymbolRemapper> Remapper;
896 if (!RemappingFile.empty())
897 Remapper = SymbolRemapper::create(RemappingFile);
898
899 if (!SupplInstrWithSample.empty()) {
900 if (ProfileKind != instr)
901 exitWithError(
902 "-supplement-instr-with-sample can only work with -instr. ");
903
904 supplementInstrProfile(WeightedInputs, SupplInstrWithSample, OutputFilename,
905 OutputFormat, OutputSparse, SupplMinSizeThreshold,
906 ZeroCounterThreshold, InstrProfColdThreshold);
907 return 0;
908 }
909
910 if (ProfileKind == instr)
911 mergeInstrProfile(WeightedInputs, Remapper.get(), OutputFilename,
912 OutputFormat, OutputSparse, NumThreads, FailureMode);
913 else
914 mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename,
915 OutputFormat, ProfileSymbolListFile, CompressAllSections,
916 UseMD5, GenPartialProfile, FailureMode);
917
918 return 0;
919 }
920
921 /// Computer the overlap b/w profile BaseFilename and profile TestFilename.
overlapInstrProfile(const std::string & BaseFilename,const std::string & TestFilename,const OverlapFuncFilters & FuncFilter,raw_fd_ostream & OS,bool IsCS)922 static void overlapInstrProfile(const std::string &BaseFilename,
923 const std::string &TestFilename,
924 const OverlapFuncFilters &FuncFilter,
925 raw_fd_ostream &OS, bool IsCS) {
926 std::mutex ErrorLock;
927 SmallSet<instrprof_error, 4> WriterErrorCodes;
928 WriterContext Context(false, ErrorLock, WriterErrorCodes);
929 WeightedFile WeightedInput{BaseFilename, 1};
930 OverlapStats Overlap;
931 Error E = Overlap.accumulateCounts(BaseFilename, TestFilename, IsCS);
932 if (E)
933 exitWithError(std::move(E), "Error in getting profile count sums");
934 if (Overlap.Base.CountSum < 1.0f) {
935 OS << "Sum of edge counts for profile " << BaseFilename << " is 0.\n";
936 exit(0);
937 }
938 if (Overlap.Test.CountSum < 1.0f) {
939 OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n";
940 exit(0);
941 }
942 loadInput(WeightedInput, nullptr, &Context);
943 overlapInput(BaseFilename, TestFilename, &Context, Overlap, FuncFilter, OS,
944 IsCS);
945 Overlap.dump(OS);
946 }
947
948 namespace {
949 struct SampleOverlapStats {
950 StringRef BaseName;
951 StringRef TestName;
952 // Number of overlap units
953 uint64_t OverlapCount;
954 // Total samples of overlap units
955 uint64_t OverlapSample;
956 // Number of and total samples of units that only present in base or test
957 // profile
958 uint64_t BaseUniqueCount;
959 uint64_t BaseUniqueSample;
960 uint64_t TestUniqueCount;
961 uint64_t TestUniqueSample;
962 // Number of units and total samples in base or test profile
963 uint64_t BaseCount;
964 uint64_t BaseSample;
965 uint64_t TestCount;
966 uint64_t TestSample;
967 // Number of and total samples of units that present in at least one profile
968 uint64_t UnionCount;
969 uint64_t UnionSample;
970 // Weighted similarity
971 double Similarity;
972 // For SampleOverlapStats instances representing functions, weights of the
973 // function in base and test profiles
974 double BaseWeight;
975 double TestWeight;
976
SampleOverlapStats__anonf47fdd840811::SampleOverlapStats977 SampleOverlapStats()
978 : OverlapCount(0), OverlapSample(0), BaseUniqueCount(0),
979 BaseUniqueSample(0), TestUniqueCount(0), TestUniqueSample(0),
980 BaseCount(0), BaseSample(0), TestCount(0), TestSample(0), UnionCount(0),
981 UnionSample(0), Similarity(0.0), BaseWeight(0.0), TestWeight(0.0) {}
982 };
983 } // end anonymous namespace
984
985 namespace {
986 struct FuncSampleStats {
987 uint64_t SampleSum;
988 uint64_t MaxSample;
989 uint64_t HotBlockCount;
FuncSampleStats__anonf47fdd840911::FuncSampleStats990 FuncSampleStats() : SampleSum(0), MaxSample(0), HotBlockCount(0) {}
FuncSampleStats__anonf47fdd840911::FuncSampleStats991 FuncSampleStats(uint64_t SampleSum, uint64_t MaxSample,
992 uint64_t HotBlockCount)
993 : SampleSum(SampleSum), MaxSample(MaxSample),
994 HotBlockCount(HotBlockCount) {}
995 };
996 } // end anonymous namespace
997
998 namespace {
999 enum MatchStatus { MS_Match, MS_FirstUnique, MS_SecondUnique, MS_None };
1000
1001 // Class for updating merging steps for two sorted maps. The class should be
1002 // instantiated with a map iterator type.
1003 template <class T> class MatchStep {
1004 public:
1005 MatchStep() = delete;
1006
MatchStep(T FirstIter,T FirstEnd,T SecondIter,T SecondEnd)1007 MatchStep(T FirstIter, T FirstEnd, T SecondIter, T SecondEnd)
1008 : FirstIter(FirstIter), FirstEnd(FirstEnd), SecondIter(SecondIter),
1009 SecondEnd(SecondEnd), Status(MS_None) {}
1010
areBothFinished() const1011 bool areBothFinished() const {
1012 return (FirstIter == FirstEnd && SecondIter == SecondEnd);
1013 }
1014
isFirstFinished() const1015 bool isFirstFinished() const { return FirstIter == FirstEnd; }
1016
isSecondFinished() const1017 bool isSecondFinished() const { return SecondIter == SecondEnd; }
1018
1019 /// Advance one step based on the previous match status unless the previous
1020 /// status is MS_None. Then update Status based on the comparison between two
1021 /// container iterators at the current step. If the previous status is
1022 /// MS_None, it means two iterators are at the beginning and no comparison has
1023 /// been made, so we simply update Status without advancing the iterators.
1024 void updateOneStep();
1025
getFirstIter() const1026 T getFirstIter() const { return FirstIter; }
1027
getSecondIter() const1028 T getSecondIter() const { return SecondIter; }
1029
getMatchStatus() const1030 MatchStatus getMatchStatus() const { return Status; }
1031
1032 private:
1033 // Current iterator and end iterator of the first container.
1034 T FirstIter;
1035 T FirstEnd;
1036 // Current iterator and end iterator of the second container.
1037 T SecondIter;
1038 T SecondEnd;
1039 // Match status of the current step.
1040 MatchStatus Status;
1041 };
1042 } // end anonymous namespace
1043
updateOneStep()1044 template <class T> void MatchStep<T>::updateOneStep() {
1045 switch (Status) {
1046 case MS_Match:
1047 ++FirstIter;
1048 ++SecondIter;
1049 break;
1050 case MS_FirstUnique:
1051 ++FirstIter;
1052 break;
1053 case MS_SecondUnique:
1054 ++SecondIter;
1055 break;
1056 case MS_None:
1057 break;
1058 }
1059
1060 // Update Status according to iterators at the current step.
1061 if (areBothFinished())
1062 return;
1063 if (FirstIter != FirstEnd &&
1064 (SecondIter == SecondEnd || FirstIter->first < SecondIter->first))
1065 Status = MS_FirstUnique;
1066 else if (SecondIter != SecondEnd &&
1067 (FirstIter == FirstEnd || SecondIter->first < FirstIter->first))
1068 Status = MS_SecondUnique;
1069 else
1070 Status = MS_Match;
1071 }
1072
1073 // Return the sum of line/block samples, the max line/block sample, and the
1074 // number of line/block samples above the given threshold in a function
1075 // including its inlinees.
getFuncSampleStats(const sampleprof::FunctionSamples & Func,FuncSampleStats & FuncStats,uint64_t HotThreshold)1076 static void getFuncSampleStats(const sampleprof::FunctionSamples &Func,
1077 FuncSampleStats &FuncStats,
1078 uint64_t HotThreshold) {
1079 for (const auto &L : Func.getBodySamples()) {
1080 uint64_t Sample = L.second.getSamples();
1081 FuncStats.SampleSum += Sample;
1082 FuncStats.MaxSample = std::max(FuncStats.MaxSample, Sample);
1083 if (Sample >= HotThreshold)
1084 ++FuncStats.HotBlockCount;
1085 }
1086
1087 for (const auto &C : Func.getCallsiteSamples()) {
1088 for (const auto &F : C.second)
1089 getFuncSampleStats(F.second, FuncStats, HotThreshold);
1090 }
1091 }
1092
1093 /// Predicate that determines if a function is hot with a given threshold. We
1094 /// keep it separate from its callsites for possible extension in the future.
isFunctionHot(const FuncSampleStats & FuncStats,uint64_t HotThreshold)1095 static bool isFunctionHot(const FuncSampleStats &FuncStats,
1096 uint64_t HotThreshold) {
1097 // We intentionally compare the maximum sample count in a function with the
1098 // HotThreshold to get an approximate determination on hot functions.
1099 return (FuncStats.MaxSample >= HotThreshold);
1100 }
1101
1102 namespace {
1103 class SampleOverlapAggregator {
1104 public:
SampleOverlapAggregator(const std::string & BaseFilename,const std::string & TestFilename,double LowSimilarityThreshold,double Epsilon,const OverlapFuncFilters & FuncFilter)1105 SampleOverlapAggregator(const std::string &BaseFilename,
1106 const std::string &TestFilename,
1107 double LowSimilarityThreshold, double Epsilon,
1108 const OverlapFuncFilters &FuncFilter)
1109 : BaseFilename(BaseFilename), TestFilename(TestFilename),
1110 LowSimilarityThreshold(LowSimilarityThreshold), Epsilon(Epsilon),
1111 FuncFilter(FuncFilter) {}
1112
1113 /// Detect 0-sample input profile and report to output stream. This interface
1114 /// should be called after loadProfiles().
1115 bool detectZeroSampleProfile(raw_fd_ostream &OS) const;
1116
1117 /// Write out function-level similarity statistics for functions specified by
1118 /// options --function, --value-cutoff, and --similarity-cutoff.
1119 void dumpFuncSimilarity(raw_fd_ostream &OS) const;
1120
1121 /// Write out program-level similarity and overlap statistics.
1122 void dumpProgramSummary(raw_fd_ostream &OS) const;
1123
1124 /// Write out hot-function and hot-block statistics for base_profile,
1125 /// test_profile, and their overlap. For both cases, the overlap HO is
1126 /// calculated as follows:
1127 /// Given the number of functions (or blocks) that are hot in both profiles
1128 /// HCommon and the number of functions (or blocks) that are hot in at
1129 /// least one profile HUnion, HO = HCommon / HUnion.
1130 void dumpHotFuncAndBlockOverlap(raw_fd_ostream &OS) const;
1131
1132 /// This function tries matching functions in base and test profiles. For each
1133 /// pair of matched functions, it aggregates the function-level
1134 /// similarity into a profile-level similarity. It also dump function-level
1135 /// similarity information of functions specified by --function,
1136 /// --value-cutoff, and --similarity-cutoff options. The program-level
1137 /// similarity PS is computed as follows:
1138 /// Given function-level similarity FS(A) for all function A, the
1139 /// weight of function A in base profile WB(A), and the weight of function
1140 /// A in test profile WT(A), compute PS(base_profile, test_profile) =
1141 /// sum_A(FS(A) * avg(WB(A), WT(A))) ranging in [0.0f to 1.0f] with 0.0
1142 /// meaning no-overlap.
1143 void computeSampleProfileOverlap(raw_fd_ostream &OS);
1144
1145 /// Initialize ProfOverlap with the sum of samples in base and test
1146 /// profiles. This function also computes and keeps the sum of samples and
1147 /// max sample counts of each function in BaseStats and TestStats for later
1148 /// use to avoid re-computations.
1149 void initializeSampleProfileOverlap();
1150
1151 /// Load profiles specified by BaseFilename and TestFilename.
1152 std::error_code loadProfiles();
1153
1154 private:
1155 SampleOverlapStats ProfOverlap;
1156 SampleOverlapStats HotFuncOverlap;
1157 SampleOverlapStats HotBlockOverlap;
1158 std::string BaseFilename;
1159 std::string TestFilename;
1160 std::unique_ptr<sampleprof::SampleProfileReader> BaseReader;
1161 std::unique_ptr<sampleprof::SampleProfileReader> TestReader;
1162 // BaseStats and TestStats hold FuncSampleStats for each function, with
1163 // function name as the key.
1164 StringMap<FuncSampleStats> BaseStats;
1165 StringMap<FuncSampleStats> TestStats;
1166 // Low similarity threshold in floating point number
1167 double LowSimilarityThreshold;
1168 // Block samples above BaseHotThreshold or TestHotThreshold are considered hot
1169 // for tracking hot blocks.
1170 uint64_t BaseHotThreshold;
1171 uint64_t TestHotThreshold;
1172 // A small threshold used to round the results of floating point accumulations
1173 // to resolve imprecision.
1174 const double Epsilon;
1175 std::multimap<double, SampleOverlapStats, std::greater<double>>
1176 FuncSimilarityDump;
1177 // FuncFilter carries specifications in options --value-cutoff and
1178 // --function.
1179 OverlapFuncFilters FuncFilter;
1180 // Column offsets for printing the function-level details table.
1181 static const unsigned int TestWeightCol = 15;
1182 static const unsigned int SimilarityCol = 30;
1183 static const unsigned int OverlapCol = 43;
1184 static const unsigned int BaseUniqueCol = 53;
1185 static const unsigned int TestUniqueCol = 67;
1186 static const unsigned int BaseSampleCol = 81;
1187 static const unsigned int TestSampleCol = 96;
1188 static const unsigned int FuncNameCol = 111;
1189
1190 /// Return a similarity of two line/block sample counters in the same
1191 /// function in base and test profiles. The line/block-similarity BS(i) is
1192 /// computed as follows:
1193 /// For an offsets i, given the sample count at i in base profile BB(i),
1194 /// the sample count at i in test profile BT(i), the sum of sample counts
1195 /// in this function in base profile SB, and the sum of sample counts in
1196 /// this function in test profile ST, compute BS(i) = 1.0 - fabs(BB(i)/SB -
1197 /// BT(i)/ST), ranging in [0.0f to 1.0f] with 0.0 meaning no-overlap.
1198 double computeBlockSimilarity(uint64_t BaseSample, uint64_t TestSample,
1199 const SampleOverlapStats &FuncOverlap) const;
1200
1201 void updateHotBlockOverlap(uint64_t BaseSample, uint64_t TestSample,
1202 uint64_t HotBlockCount);
1203
1204 void getHotFunctions(const StringMap<FuncSampleStats> &ProfStats,
1205 StringMap<FuncSampleStats> &HotFunc,
1206 uint64_t HotThreshold) const;
1207
1208 void computeHotFuncOverlap();
1209
1210 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and
1211 /// Difference for two sample units in a matched function according to the
1212 /// given match status.
1213 void updateOverlapStatsForFunction(uint64_t BaseSample, uint64_t TestSample,
1214 uint64_t HotBlockCount,
1215 SampleOverlapStats &FuncOverlap,
1216 double &Difference, MatchStatus Status);
1217
1218 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and
1219 /// Difference for unmatched callees that only present in one profile in a
1220 /// matched caller function.
1221 void updateForUnmatchedCallee(const sampleprof::FunctionSamples &Func,
1222 SampleOverlapStats &FuncOverlap,
1223 double &Difference, MatchStatus Status);
1224
1225 /// This function updates sample overlap statistics of an overlap function in
1226 /// base and test profile. It also calculates a function-internal similarity
1227 /// FIS as follows:
1228 /// For offsets i that have samples in at least one profile in this
1229 /// function A, given BS(i) returned by computeBlockSimilarity(), compute
1230 /// FIS(A) = (2.0 - sum_i(1.0 - BS(i))) / 2, ranging in [0.0f to 1.0f] with
1231 /// 0.0 meaning no overlap.
1232 double computeSampleFunctionInternalOverlap(
1233 const sampleprof::FunctionSamples &BaseFunc,
1234 const sampleprof::FunctionSamples &TestFunc,
1235 SampleOverlapStats &FuncOverlap);
1236
1237 /// Function-level similarity (FS) is a weighted value over function internal
1238 /// similarity (FIS). This function computes a function's FS from its FIS by
1239 /// applying the weight.
1240 double weightForFuncSimilarity(double FuncSimilarity, uint64_t BaseFuncSample,
1241 uint64_t TestFuncSample) const;
1242
1243 /// The function-level similarity FS(A) for a function A is computed as
1244 /// follows:
1245 /// Compute a function-internal similarity FIS(A) by
1246 /// computeSampleFunctionInternalOverlap(). Then, with the weight of
1247 /// function A in base profile WB(A), and the weight of function A in test
1248 /// profile WT(A), compute FS(A) = FIS(A) * (1.0 - fabs(WB(A) - WT(A)))
1249 /// ranging in [0.0f to 1.0f] with 0.0 meaning no overlap.
1250 double
1251 computeSampleFunctionOverlap(const sampleprof::FunctionSamples *BaseFunc,
1252 const sampleprof::FunctionSamples *TestFunc,
1253 SampleOverlapStats *FuncOverlap,
1254 uint64_t BaseFuncSample,
1255 uint64_t TestFuncSample);
1256
1257 /// Profile-level similarity (PS) is a weighted aggregate over function-level
1258 /// similarities (FS). This method weights the FS value by the function
1259 /// weights in the base and test profiles for the aggregation.
1260 double weightByImportance(double FuncSimilarity, uint64_t BaseFuncSample,
1261 uint64_t TestFuncSample) const;
1262 };
1263 } // end anonymous namespace
1264
detectZeroSampleProfile(raw_fd_ostream & OS) const1265 bool SampleOverlapAggregator::detectZeroSampleProfile(
1266 raw_fd_ostream &OS) const {
1267 bool HaveZeroSample = false;
1268 if (ProfOverlap.BaseSample == 0) {
1269 OS << "Sum of sample counts for profile " << BaseFilename << " is 0.\n";
1270 HaveZeroSample = true;
1271 }
1272 if (ProfOverlap.TestSample == 0) {
1273 OS << "Sum of sample counts for profile " << TestFilename << " is 0.\n";
1274 HaveZeroSample = true;
1275 }
1276 return HaveZeroSample;
1277 }
1278
computeBlockSimilarity(uint64_t BaseSample,uint64_t TestSample,const SampleOverlapStats & FuncOverlap) const1279 double SampleOverlapAggregator::computeBlockSimilarity(
1280 uint64_t BaseSample, uint64_t TestSample,
1281 const SampleOverlapStats &FuncOverlap) const {
1282 double BaseFrac = 0.0;
1283 double TestFrac = 0.0;
1284 if (FuncOverlap.BaseSample > 0)
1285 BaseFrac = static_cast<double>(BaseSample) / FuncOverlap.BaseSample;
1286 if (FuncOverlap.TestSample > 0)
1287 TestFrac = static_cast<double>(TestSample) / FuncOverlap.TestSample;
1288 return 1.0 - std::fabs(BaseFrac - TestFrac);
1289 }
1290
updateHotBlockOverlap(uint64_t BaseSample,uint64_t TestSample,uint64_t HotBlockCount)1291 void SampleOverlapAggregator::updateHotBlockOverlap(uint64_t BaseSample,
1292 uint64_t TestSample,
1293 uint64_t HotBlockCount) {
1294 bool IsBaseHot = (BaseSample >= BaseHotThreshold);
1295 bool IsTestHot = (TestSample >= TestHotThreshold);
1296 if (!IsBaseHot && !IsTestHot)
1297 return;
1298
1299 HotBlockOverlap.UnionCount += HotBlockCount;
1300 if (IsBaseHot)
1301 HotBlockOverlap.BaseCount += HotBlockCount;
1302 if (IsTestHot)
1303 HotBlockOverlap.TestCount += HotBlockCount;
1304 if (IsBaseHot && IsTestHot)
1305 HotBlockOverlap.OverlapCount += HotBlockCount;
1306 }
1307
getHotFunctions(const StringMap<FuncSampleStats> & ProfStats,StringMap<FuncSampleStats> & HotFunc,uint64_t HotThreshold) const1308 void SampleOverlapAggregator::getHotFunctions(
1309 const StringMap<FuncSampleStats> &ProfStats,
1310 StringMap<FuncSampleStats> &HotFunc, uint64_t HotThreshold) const {
1311 for (const auto &F : ProfStats) {
1312 if (isFunctionHot(F.second, HotThreshold))
1313 HotFunc.try_emplace(F.first(), F.second);
1314 }
1315 }
1316
computeHotFuncOverlap()1317 void SampleOverlapAggregator::computeHotFuncOverlap() {
1318 StringMap<FuncSampleStats> BaseHotFunc;
1319 getHotFunctions(BaseStats, BaseHotFunc, BaseHotThreshold);
1320 HotFuncOverlap.BaseCount = BaseHotFunc.size();
1321
1322 StringMap<FuncSampleStats> TestHotFunc;
1323 getHotFunctions(TestStats, TestHotFunc, TestHotThreshold);
1324 HotFuncOverlap.TestCount = TestHotFunc.size();
1325 HotFuncOverlap.UnionCount = HotFuncOverlap.TestCount;
1326
1327 for (const auto &F : BaseHotFunc) {
1328 if (TestHotFunc.count(F.first()))
1329 ++HotFuncOverlap.OverlapCount;
1330 else
1331 ++HotFuncOverlap.UnionCount;
1332 }
1333 }
1334
updateOverlapStatsForFunction(uint64_t BaseSample,uint64_t TestSample,uint64_t HotBlockCount,SampleOverlapStats & FuncOverlap,double & Difference,MatchStatus Status)1335 void SampleOverlapAggregator::updateOverlapStatsForFunction(
1336 uint64_t BaseSample, uint64_t TestSample, uint64_t HotBlockCount,
1337 SampleOverlapStats &FuncOverlap, double &Difference, MatchStatus Status) {
1338 assert(Status != MS_None &&
1339 "Match status should be updated before updating overlap statistics");
1340 if (Status == MS_FirstUnique) {
1341 TestSample = 0;
1342 FuncOverlap.BaseUniqueSample += BaseSample;
1343 } else if (Status == MS_SecondUnique) {
1344 BaseSample = 0;
1345 FuncOverlap.TestUniqueSample += TestSample;
1346 } else {
1347 ++FuncOverlap.OverlapCount;
1348 }
1349
1350 FuncOverlap.UnionSample += std::max(BaseSample, TestSample);
1351 FuncOverlap.OverlapSample += std::min(BaseSample, TestSample);
1352 Difference +=
1353 1.0 - computeBlockSimilarity(BaseSample, TestSample, FuncOverlap);
1354 updateHotBlockOverlap(BaseSample, TestSample, HotBlockCount);
1355 }
1356
updateForUnmatchedCallee(const sampleprof::FunctionSamples & Func,SampleOverlapStats & FuncOverlap,double & Difference,MatchStatus Status)1357 void SampleOverlapAggregator::updateForUnmatchedCallee(
1358 const sampleprof::FunctionSamples &Func, SampleOverlapStats &FuncOverlap,
1359 double &Difference, MatchStatus Status) {
1360 assert((Status == MS_FirstUnique || Status == MS_SecondUnique) &&
1361 "Status must be either of the two unmatched cases");
1362 FuncSampleStats FuncStats;
1363 if (Status == MS_FirstUnique) {
1364 getFuncSampleStats(Func, FuncStats, BaseHotThreshold);
1365 updateOverlapStatsForFunction(FuncStats.SampleSum, 0,
1366 FuncStats.HotBlockCount, FuncOverlap,
1367 Difference, Status);
1368 } else {
1369 getFuncSampleStats(Func, FuncStats, TestHotThreshold);
1370 updateOverlapStatsForFunction(0, FuncStats.SampleSum,
1371 FuncStats.HotBlockCount, FuncOverlap,
1372 Difference, Status);
1373 }
1374 }
1375
computeSampleFunctionInternalOverlap(const sampleprof::FunctionSamples & BaseFunc,const sampleprof::FunctionSamples & TestFunc,SampleOverlapStats & FuncOverlap)1376 double SampleOverlapAggregator::computeSampleFunctionInternalOverlap(
1377 const sampleprof::FunctionSamples &BaseFunc,
1378 const sampleprof::FunctionSamples &TestFunc,
1379 SampleOverlapStats &FuncOverlap) {
1380
1381 using namespace sampleprof;
1382
1383 double Difference = 0;
1384
1385 // Accumulate Difference for regular line/block samples in the function.
1386 // We match them through sort-merge join algorithm because
1387 // FunctionSamples::getBodySamples() returns a map of sample counters ordered
1388 // by their offsets.
1389 MatchStep<BodySampleMap::const_iterator> BlockIterStep(
1390 BaseFunc.getBodySamples().cbegin(), BaseFunc.getBodySamples().cend(),
1391 TestFunc.getBodySamples().cbegin(), TestFunc.getBodySamples().cend());
1392 BlockIterStep.updateOneStep();
1393 while (!BlockIterStep.areBothFinished()) {
1394 uint64_t BaseSample =
1395 BlockIterStep.isFirstFinished()
1396 ? 0
1397 : BlockIterStep.getFirstIter()->second.getSamples();
1398 uint64_t TestSample =
1399 BlockIterStep.isSecondFinished()
1400 ? 0
1401 : BlockIterStep.getSecondIter()->second.getSamples();
1402 updateOverlapStatsForFunction(BaseSample, TestSample, 1, FuncOverlap,
1403 Difference, BlockIterStep.getMatchStatus());
1404
1405 BlockIterStep.updateOneStep();
1406 }
1407
1408 // Accumulate Difference for callsite lines in the function. We match
1409 // them through sort-merge algorithm because
1410 // FunctionSamples::getCallsiteSamples() returns a map of callsite records
1411 // ordered by their offsets.
1412 MatchStep<CallsiteSampleMap::const_iterator> CallsiteIterStep(
1413 BaseFunc.getCallsiteSamples().cbegin(),
1414 BaseFunc.getCallsiteSamples().cend(),
1415 TestFunc.getCallsiteSamples().cbegin(),
1416 TestFunc.getCallsiteSamples().cend());
1417 CallsiteIterStep.updateOneStep();
1418 while (!CallsiteIterStep.areBothFinished()) {
1419 MatchStatus CallsiteStepStatus = CallsiteIterStep.getMatchStatus();
1420 assert(CallsiteStepStatus != MS_None &&
1421 "Match status should be updated before entering loop body");
1422
1423 if (CallsiteStepStatus != MS_Match) {
1424 auto Callsite = (CallsiteStepStatus == MS_FirstUnique)
1425 ? CallsiteIterStep.getFirstIter()
1426 : CallsiteIterStep.getSecondIter();
1427 for (const auto &F : Callsite->second)
1428 updateForUnmatchedCallee(F.second, FuncOverlap, Difference,
1429 CallsiteStepStatus);
1430 } else {
1431 // There may be multiple inlinees at the same offset, so we need to try
1432 // matching all of them. This match is implemented through sort-merge
1433 // algorithm because callsite records at the same offset are ordered by
1434 // function names.
1435 MatchStep<FunctionSamplesMap::const_iterator> CalleeIterStep(
1436 CallsiteIterStep.getFirstIter()->second.cbegin(),
1437 CallsiteIterStep.getFirstIter()->second.cend(),
1438 CallsiteIterStep.getSecondIter()->second.cbegin(),
1439 CallsiteIterStep.getSecondIter()->second.cend());
1440 CalleeIterStep.updateOneStep();
1441 while (!CalleeIterStep.areBothFinished()) {
1442 MatchStatus CalleeStepStatus = CalleeIterStep.getMatchStatus();
1443 if (CalleeStepStatus != MS_Match) {
1444 auto Callee = (CalleeStepStatus == MS_FirstUnique)
1445 ? CalleeIterStep.getFirstIter()
1446 : CalleeIterStep.getSecondIter();
1447 updateForUnmatchedCallee(Callee->second, FuncOverlap, Difference,
1448 CalleeStepStatus);
1449 } else {
1450 // An inlined function can contain other inlinees inside, so compute
1451 // the Difference recursively.
1452 Difference += 2.0 - 2 * computeSampleFunctionInternalOverlap(
1453 CalleeIterStep.getFirstIter()->second,
1454 CalleeIterStep.getSecondIter()->second,
1455 FuncOverlap);
1456 }
1457 CalleeIterStep.updateOneStep();
1458 }
1459 }
1460 CallsiteIterStep.updateOneStep();
1461 }
1462
1463 // Difference reflects the total differences of line/block samples in this
1464 // function and ranges in [0.0f to 2.0f]. Take (2.0 - Difference) / 2 to
1465 // reflect the similarity between function profiles in [0.0f to 1.0f].
1466 return (2.0 - Difference) / 2;
1467 }
1468
weightForFuncSimilarity(double FuncInternalSimilarity,uint64_t BaseFuncSample,uint64_t TestFuncSample) const1469 double SampleOverlapAggregator::weightForFuncSimilarity(
1470 double FuncInternalSimilarity, uint64_t BaseFuncSample,
1471 uint64_t TestFuncSample) const {
1472 // Compute the weight as the distance between the function weights in two
1473 // profiles.
1474 double BaseFrac = 0.0;
1475 double TestFrac = 0.0;
1476 assert(ProfOverlap.BaseSample > 0 &&
1477 "Total samples in base profile should be greater than 0");
1478 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample;
1479 assert(ProfOverlap.TestSample > 0 &&
1480 "Total samples in test profile should be greater than 0");
1481 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample;
1482 double WeightDistance = std::fabs(BaseFrac - TestFrac);
1483
1484 // Take WeightDistance into the similarity.
1485 return FuncInternalSimilarity * (1 - WeightDistance);
1486 }
1487
1488 double
weightByImportance(double FuncSimilarity,uint64_t BaseFuncSample,uint64_t TestFuncSample) const1489 SampleOverlapAggregator::weightByImportance(double FuncSimilarity,
1490 uint64_t BaseFuncSample,
1491 uint64_t TestFuncSample) const {
1492
1493 double BaseFrac = 0.0;
1494 double TestFrac = 0.0;
1495 assert(ProfOverlap.BaseSample > 0 &&
1496 "Total samples in base profile should be greater than 0");
1497 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample / 2.0;
1498 assert(ProfOverlap.TestSample > 0 &&
1499 "Total samples in test profile should be greater than 0");
1500 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample / 2.0;
1501 return FuncSimilarity * (BaseFrac + TestFrac);
1502 }
1503
computeSampleFunctionOverlap(const sampleprof::FunctionSamples * BaseFunc,const sampleprof::FunctionSamples * TestFunc,SampleOverlapStats * FuncOverlap,uint64_t BaseFuncSample,uint64_t TestFuncSample)1504 double SampleOverlapAggregator::computeSampleFunctionOverlap(
1505 const sampleprof::FunctionSamples *BaseFunc,
1506 const sampleprof::FunctionSamples *TestFunc,
1507 SampleOverlapStats *FuncOverlap, uint64_t BaseFuncSample,
1508 uint64_t TestFuncSample) {
1509 // Default function internal similarity before weighted, meaning two functions
1510 // has no overlap.
1511 const double DefaultFuncInternalSimilarity = 0;
1512 double FuncSimilarity;
1513 double FuncInternalSimilarity;
1514
1515 // If BaseFunc or TestFunc is nullptr, it means the functions do not overlap.
1516 // In this case, we use DefaultFuncInternalSimilarity as the function internal
1517 // similarity.
1518 if (!BaseFunc || !TestFunc) {
1519 FuncInternalSimilarity = DefaultFuncInternalSimilarity;
1520 } else {
1521 assert(FuncOverlap != nullptr &&
1522 "FuncOverlap should be provided in this case");
1523 FuncInternalSimilarity = computeSampleFunctionInternalOverlap(
1524 *BaseFunc, *TestFunc, *FuncOverlap);
1525 // Now, FuncInternalSimilarity may be a little less than 0 due to
1526 // imprecision of floating point accumulations. Make it zero if the
1527 // difference is below Epsilon.
1528 FuncInternalSimilarity = (std::fabs(FuncInternalSimilarity - 0) < Epsilon)
1529 ? 0
1530 : FuncInternalSimilarity;
1531 }
1532 FuncSimilarity = weightForFuncSimilarity(FuncInternalSimilarity,
1533 BaseFuncSample, TestFuncSample);
1534 return FuncSimilarity;
1535 }
1536
computeSampleProfileOverlap(raw_fd_ostream & OS)1537 void SampleOverlapAggregator::computeSampleProfileOverlap(raw_fd_ostream &OS) {
1538 using namespace sampleprof;
1539
1540 StringMap<const FunctionSamples *> BaseFuncProf;
1541 const auto &BaseProfiles = BaseReader->getProfiles();
1542 for (const auto &BaseFunc : BaseProfiles) {
1543 BaseFuncProf.try_emplace(BaseFunc.second.getName(), &(BaseFunc.second));
1544 }
1545 ProfOverlap.UnionCount = BaseFuncProf.size();
1546
1547 const auto &TestProfiles = TestReader->getProfiles();
1548 for (const auto &TestFunc : TestProfiles) {
1549 SampleOverlapStats FuncOverlap;
1550 FuncOverlap.TestName = TestFunc.second.getName();
1551 assert(TestStats.count(FuncOverlap.TestName) &&
1552 "TestStats should have records for all functions in test profile "
1553 "except inlinees");
1554 FuncOverlap.TestSample = TestStats[FuncOverlap.TestName].SampleSum;
1555
1556 const auto Match = BaseFuncProf.find(FuncOverlap.TestName);
1557 if (Match == BaseFuncProf.end()) {
1558 const FuncSampleStats &FuncStats = TestStats[FuncOverlap.TestName];
1559 ++ProfOverlap.TestUniqueCount;
1560 ProfOverlap.TestUniqueSample += FuncStats.SampleSum;
1561 FuncOverlap.TestUniqueSample = FuncStats.SampleSum;
1562
1563 updateHotBlockOverlap(0, FuncStats.SampleSum, FuncStats.HotBlockCount);
1564
1565 double FuncSimilarity = computeSampleFunctionOverlap(
1566 nullptr, nullptr, nullptr, 0, FuncStats.SampleSum);
1567 ProfOverlap.Similarity +=
1568 weightByImportance(FuncSimilarity, 0, FuncStats.SampleSum);
1569
1570 ++ProfOverlap.UnionCount;
1571 ProfOverlap.UnionSample += FuncStats.SampleSum;
1572 } else {
1573 ++ProfOverlap.OverlapCount;
1574
1575 // Two functions match with each other. Compute function-level overlap and
1576 // aggregate them into profile-level overlap.
1577 FuncOverlap.BaseName = Match->second->getName();
1578 assert(BaseStats.count(FuncOverlap.BaseName) &&
1579 "BaseStats should have records for all functions in base profile "
1580 "except inlinees");
1581 FuncOverlap.BaseSample = BaseStats[FuncOverlap.BaseName].SampleSum;
1582
1583 FuncOverlap.Similarity = computeSampleFunctionOverlap(
1584 Match->second, &TestFunc.second, &FuncOverlap, FuncOverlap.BaseSample,
1585 FuncOverlap.TestSample);
1586 ProfOverlap.Similarity +=
1587 weightByImportance(FuncOverlap.Similarity, FuncOverlap.BaseSample,
1588 FuncOverlap.TestSample);
1589 ProfOverlap.OverlapSample += FuncOverlap.OverlapSample;
1590 ProfOverlap.UnionSample += FuncOverlap.UnionSample;
1591
1592 // Accumulate the percentage of base unique and test unique samples into
1593 // ProfOverlap.
1594 ProfOverlap.BaseUniqueSample += FuncOverlap.BaseUniqueSample;
1595 ProfOverlap.TestUniqueSample += FuncOverlap.TestUniqueSample;
1596
1597 // Remove matched base functions for later reporting functions not found
1598 // in test profile.
1599 BaseFuncProf.erase(Match);
1600 }
1601
1602 // Print function-level similarity information if specified by options.
1603 assert(TestStats.count(FuncOverlap.TestName) &&
1604 "TestStats should have records for all functions in test profile "
1605 "except inlinees");
1606 if (TestStats[FuncOverlap.TestName].MaxSample >= FuncFilter.ValueCutoff ||
1607 (Match != BaseFuncProf.end() &&
1608 FuncOverlap.Similarity < LowSimilarityThreshold) ||
1609 (Match != BaseFuncProf.end() && !FuncFilter.NameFilter.empty() &&
1610 FuncOverlap.BaseName.find(FuncFilter.NameFilter) !=
1611 FuncOverlap.BaseName.npos)) {
1612 assert(ProfOverlap.BaseSample > 0 &&
1613 "Total samples in base profile should be greater than 0");
1614 FuncOverlap.BaseWeight =
1615 static_cast<double>(FuncOverlap.BaseSample) / ProfOverlap.BaseSample;
1616 assert(ProfOverlap.TestSample > 0 &&
1617 "Total samples in test profile should be greater than 0");
1618 FuncOverlap.TestWeight =
1619 static_cast<double>(FuncOverlap.TestSample) / ProfOverlap.TestSample;
1620 FuncSimilarityDump.emplace(FuncOverlap.BaseWeight, FuncOverlap);
1621 }
1622 }
1623
1624 // Traverse through functions in base profile but not in test profile.
1625 for (const auto &F : BaseFuncProf) {
1626 assert(BaseStats.count(F.second->getName()) &&
1627 "BaseStats should have records for all functions in base profile "
1628 "except inlinees");
1629 const FuncSampleStats &FuncStats = BaseStats[F.second->getName()];
1630 ++ProfOverlap.BaseUniqueCount;
1631 ProfOverlap.BaseUniqueSample += FuncStats.SampleSum;
1632
1633 updateHotBlockOverlap(FuncStats.SampleSum, 0, FuncStats.HotBlockCount);
1634
1635 double FuncSimilarity = computeSampleFunctionOverlap(
1636 nullptr, nullptr, nullptr, FuncStats.SampleSum, 0);
1637 ProfOverlap.Similarity +=
1638 weightByImportance(FuncSimilarity, FuncStats.SampleSum, 0);
1639
1640 ProfOverlap.UnionSample += FuncStats.SampleSum;
1641 }
1642
1643 // Now, ProfSimilarity may be a little greater than 1 due to imprecision
1644 // of floating point accumulations. Make it 1.0 if the difference is below
1645 // Epsilon.
1646 ProfOverlap.Similarity = (std::fabs(ProfOverlap.Similarity - 1) < Epsilon)
1647 ? 1
1648 : ProfOverlap.Similarity;
1649
1650 computeHotFuncOverlap();
1651 }
1652
initializeSampleProfileOverlap()1653 void SampleOverlapAggregator::initializeSampleProfileOverlap() {
1654 const auto &BaseProf = BaseReader->getProfiles();
1655 for (const auto &I : BaseProf) {
1656 ++ProfOverlap.BaseCount;
1657 FuncSampleStats FuncStats;
1658 getFuncSampleStats(I.second, FuncStats, BaseHotThreshold);
1659 ProfOverlap.BaseSample += FuncStats.SampleSum;
1660 BaseStats.try_emplace(I.second.getName(), FuncStats);
1661 }
1662
1663 const auto &TestProf = TestReader->getProfiles();
1664 for (const auto &I : TestProf) {
1665 ++ProfOverlap.TestCount;
1666 FuncSampleStats FuncStats;
1667 getFuncSampleStats(I.second, FuncStats, TestHotThreshold);
1668 ProfOverlap.TestSample += FuncStats.SampleSum;
1669 TestStats.try_emplace(I.second.getName(), FuncStats);
1670 }
1671
1672 ProfOverlap.BaseName = StringRef(BaseFilename);
1673 ProfOverlap.TestName = StringRef(TestFilename);
1674 }
1675
dumpFuncSimilarity(raw_fd_ostream & OS) const1676 void SampleOverlapAggregator::dumpFuncSimilarity(raw_fd_ostream &OS) const {
1677 using namespace sampleprof;
1678
1679 if (FuncSimilarityDump.empty())
1680 return;
1681
1682 formatted_raw_ostream FOS(OS);
1683 FOS << "Function-level details:\n";
1684 FOS << "Base weight";
1685 FOS.PadToColumn(TestWeightCol);
1686 FOS << "Test weight";
1687 FOS.PadToColumn(SimilarityCol);
1688 FOS << "Similarity";
1689 FOS.PadToColumn(OverlapCol);
1690 FOS << "Overlap";
1691 FOS.PadToColumn(BaseUniqueCol);
1692 FOS << "Base unique";
1693 FOS.PadToColumn(TestUniqueCol);
1694 FOS << "Test unique";
1695 FOS.PadToColumn(BaseSampleCol);
1696 FOS << "Base samples";
1697 FOS.PadToColumn(TestSampleCol);
1698 FOS << "Test samples";
1699 FOS.PadToColumn(FuncNameCol);
1700 FOS << "Function name\n";
1701 for (const auto &F : FuncSimilarityDump) {
1702 double OverlapPercent =
1703 F.second.UnionSample > 0
1704 ? static_cast<double>(F.second.OverlapSample) / F.second.UnionSample
1705 : 0;
1706 double BaseUniquePercent =
1707 F.second.BaseSample > 0
1708 ? static_cast<double>(F.second.BaseUniqueSample) /
1709 F.second.BaseSample
1710 : 0;
1711 double TestUniquePercent =
1712 F.second.TestSample > 0
1713 ? static_cast<double>(F.second.TestUniqueSample) /
1714 F.second.TestSample
1715 : 0;
1716
1717 FOS << format("%.2f%%", F.second.BaseWeight * 100);
1718 FOS.PadToColumn(TestWeightCol);
1719 FOS << format("%.2f%%", F.second.TestWeight * 100);
1720 FOS.PadToColumn(SimilarityCol);
1721 FOS << format("%.2f%%", F.second.Similarity * 100);
1722 FOS.PadToColumn(OverlapCol);
1723 FOS << format("%.2f%%", OverlapPercent * 100);
1724 FOS.PadToColumn(BaseUniqueCol);
1725 FOS << format("%.2f%%", BaseUniquePercent * 100);
1726 FOS.PadToColumn(TestUniqueCol);
1727 FOS << format("%.2f%%", TestUniquePercent * 100);
1728 FOS.PadToColumn(BaseSampleCol);
1729 FOS << F.second.BaseSample;
1730 FOS.PadToColumn(TestSampleCol);
1731 FOS << F.second.TestSample;
1732 FOS.PadToColumn(FuncNameCol);
1733 FOS << F.second.TestName << "\n";
1734 }
1735 }
1736
dumpProgramSummary(raw_fd_ostream & OS) const1737 void SampleOverlapAggregator::dumpProgramSummary(raw_fd_ostream &OS) const {
1738 OS << "Profile overlap infomation for base_profile: " << ProfOverlap.BaseName
1739 << " and test_profile: " << ProfOverlap.TestName << "\nProgram level:\n";
1740
1741 OS << " Whole program profile similarity: "
1742 << format("%.3f%%", ProfOverlap.Similarity * 100) << "\n";
1743
1744 assert(ProfOverlap.UnionSample > 0 &&
1745 "Total samples in two profile should be greater than 0");
1746 double OverlapPercent =
1747 static_cast<double>(ProfOverlap.OverlapSample) / ProfOverlap.UnionSample;
1748 assert(ProfOverlap.BaseSample > 0 &&
1749 "Total samples in base profile should be greater than 0");
1750 double BaseUniquePercent = static_cast<double>(ProfOverlap.BaseUniqueSample) /
1751 ProfOverlap.BaseSample;
1752 assert(ProfOverlap.TestSample > 0 &&
1753 "Total samples in test profile should be greater than 0");
1754 double TestUniquePercent = static_cast<double>(ProfOverlap.TestUniqueSample) /
1755 ProfOverlap.TestSample;
1756
1757 OS << " Whole program sample overlap: "
1758 << format("%.3f%%", OverlapPercent * 100) << "\n";
1759 OS << " percentage of samples unique in base profile: "
1760 << format("%.3f%%", BaseUniquePercent * 100) << "\n";
1761 OS << " percentage of samples unique in test profile: "
1762 << format("%.3f%%", TestUniquePercent * 100) << "\n";
1763 OS << " total samples in base profile: " << ProfOverlap.BaseSample << "\n"
1764 << " total samples in test profile: " << ProfOverlap.TestSample << "\n";
1765
1766 assert(ProfOverlap.UnionCount > 0 &&
1767 "There should be at least one function in two input profiles");
1768 double FuncOverlapPercent =
1769 static_cast<double>(ProfOverlap.OverlapCount) / ProfOverlap.UnionCount;
1770 OS << " Function overlap: " << format("%.3f%%", FuncOverlapPercent * 100)
1771 << "\n";
1772 OS << " overlap functions: " << ProfOverlap.OverlapCount << "\n";
1773 OS << " functions unique in base profile: " << ProfOverlap.BaseUniqueCount
1774 << "\n";
1775 OS << " functions unique in test profile: " << ProfOverlap.TestUniqueCount
1776 << "\n";
1777 }
1778
dumpHotFuncAndBlockOverlap(raw_fd_ostream & OS) const1779 void SampleOverlapAggregator::dumpHotFuncAndBlockOverlap(
1780 raw_fd_ostream &OS) const {
1781 assert(HotFuncOverlap.UnionCount > 0 &&
1782 "There should be at least one hot function in two input profiles");
1783 OS << " Hot-function overlap: "
1784 << format("%.3f%%", static_cast<double>(HotFuncOverlap.OverlapCount) /
1785 HotFuncOverlap.UnionCount * 100)
1786 << "\n";
1787 OS << " overlap hot functions: " << HotFuncOverlap.OverlapCount << "\n";
1788 OS << " hot functions unique in base profile: "
1789 << HotFuncOverlap.BaseCount - HotFuncOverlap.OverlapCount << "\n";
1790 OS << " hot functions unique in test profile: "
1791 << HotFuncOverlap.TestCount - HotFuncOverlap.OverlapCount << "\n";
1792
1793 assert(HotBlockOverlap.UnionCount > 0 &&
1794 "There should be at least one hot block in two input profiles");
1795 OS << " Hot-block overlap: "
1796 << format("%.3f%%", static_cast<double>(HotBlockOverlap.OverlapCount) /
1797 HotBlockOverlap.UnionCount * 100)
1798 << "\n";
1799 OS << " overlap hot blocks: " << HotBlockOverlap.OverlapCount << "\n";
1800 OS << " hot blocks unique in base profile: "
1801 << HotBlockOverlap.BaseCount - HotBlockOverlap.OverlapCount << "\n";
1802 OS << " hot blocks unique in test profile: "
1803 << HotBlockOverlap.TestCount - HotBlockOverlap.OverlapCount << "\n";
1804 }
1805
loadProfiles()1806 std::error_code SampleOverlapAggregator::loadProfiles() {
1807 using namespace sampleprof;
1808
1809 LLVMContext Context;
1810 auto BaseReaderOrErr = SampleProfileReader::create(BaseFilename, Context);
1811 if (std::error_code EC = BaseReaderOrErr.getError())
1812 exitWithErrorCode(EC, BaseFilename);
1813
1814 auto TestReaderOrErr = SampleProfileReader::create(TestFilename, Context);
1815 if (std::error_code EC = TestReaderOrErr.getError())
1816 exitWithErrorCode(EC, TestFilename);
1817
1818 BaseReader = std::move(BaseReaderOrErr.get());
1819 TestReader = std::move(TestReaderOrErr.get());
1820
1821 if (std::error_code EC = BaseReader->read())
1822 exitWithErrorCode(EC, BaseFilename);
1823 if (std::error_code EC = TestReader->read())
1824 exitWithErrorCode(EC, TestFilename);
1825
1826 // Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in
1827 // profile summary.
1828 const uint64_t HotCutoff = 990000;
1829 ProfileSummary &BasePS = BaseReader->getSummary();
1830 for (const auto &SummaryEntry : BasePS.getDetailedSummary()) {
1831 if (SummaryEntry.Cutoff == HotCutoff) {
1832 BaseHotThreshold = SummaryEntry.MinCount;
1833 break;
1834 }
1835 }
1836
1837 ProfileSummary &TestPS = TestReader->getSummary();
1838 for (const auto &SummaryEntry : TestPS.getDetailedSummary()) {
1839 if (SummaryEntry.Cutoff == HotCutoff) {
1840 TestHotThreshold = SummaryEntry.MinCount;
1841 break;
1842 }
1843 }
1844 return std::error_code();
1845 }
1846
overlapSampleProfile(const std::string & BaseFilename,const std::string & TestFilename,const OverlapFuncFilters & FuncFilter,uint64_t SimilarityCutoff,raw_fd_ostream & OS)1847 void overlapSampleProfile(const std::string &BaseFilename,
1848 const std::string &TestFilename,
1849 const OverlapFuncFilters &FuncFilter,
1850 uint64_t SimilarityCutoff, raw_fd_ostream &OS) {
1851 using namespace sampleprof;
1852
1853 // We use 0.000005 to initialize OverlapAggr.Epsilon because the final metrics
1854 // report 2--3 places after decimal point in percentage numbers.
1855 SampleOverlapAggregator OverlapAggr(
1856 BaseFilename, TestFilename,
1857 static_cast<double>(SimilarityCutoff) / 1000000, 0.000005, FuncFilter);
1858 if (std::error_code EC = OverlapAggr.loadProfiles())
1859 exitWithErrorCode(EC);
1860
1861 OverlapAggr.initializeSampleProfileOverlap();
1862 if (OverlapAggr.detectZeroSampleProfile(OS))
1863 return;
1864
1865 OverlapAggr.computeSampleProfileOverlap(OS);
1866
1867 OverlapAggr.dumpProgramSummary(OS);
1868 OverlapAggr.dumpHotFuncAndBlockOverlap(OS);
1869 OverlapAggr.dumpFuncSimilarity(OS);
1870 }
1871
overlap_main(int argc,const char * argv[])1872 static int overlap_main(int argc, const char *argv[]) {
1873 cl::opt<std::string> BaseFilename(cl::Positional, cl::Required,
1874 cl::desc("<base profile file>"));
1875 cl::opt<std::string> TestFilename(cl::Positional, cl::Required,
1876 cl::desc("<test profile file>"));
1877 cl::opt<std::string> Output("output", cl::value_desc("output"), cl::init("-"),
1878 cl::desc("Output file"));
1879 cl::alias OutputA("o", cl::desc("Alias for --output"), cl::aliasopt(Output));
1880 cl::opt<bool> IsCS("cs", cl::init(false),
1881 cl::desc("For context sensitive counts"));
1882 cl::opt<unsigned long long> ValueCutoff(
1883 "value-cutoff", cl::init(-1),
1884 cl::desc(
1885 "Function level overlap information for every function in test "
1886 "profile with max count value greater then the parameter value"));
1887 cl::opt<std::string> FuncNameFilter(
1888 "function",
1889 cl::desc("Function level overlap information for matching functions"));
1890 cl::opt<unsigned long long> SimilarityCutoff(
1891 "similarity-cutoff", cl::init(0),
1892 cl::desc(
1893 "For sample profiles, list function names for overlapped functions "
1894 "with similarities below the cutoff (percentage times 10000)."));
1895 cl::opt<ProfileKinds> ProfileKind(
1896 cl::desc("Profile kind:"), cl::init(instr),
1897 cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
1898 clEnumVal(sample, "Sample profile")));
1899 cl::ParseCommandLineOptions(argc, argv, "LLVM profile data overlap tool\n");
1900
1901 std::error_code EC;
1902 raw_fd_ostream OS(Output.data(), EC, sys::fs::OF_Text);
1903 if (EC)
1904 exitWithErrorCode(EC, Output);
1905
1906 if (ProfileKind == instr)
1907 overlapInstrProfile(BaseFilename, TestFilename,
1908 OverlapFuncFilters{ValueCutoff, FuncNameFilter}, OS,
1909 IsCS);
1910 else
1911 overlapSampleProfile(BaseFilename, TestFilename,
1912 OverlapFuncFilters{ValueCutoff, FuncNameFilter},
1913 SimilarityCutoff, OS);
1914
1915 return 0;
1916 }
1917
1918 typedef struct ValueSitesStats {
ValueSitesStatsValueSitesStats1919 ValueSitesStats()
1920 : TotalNumValueSites(0), TotalNumValueSitesWithValueProfile(0),
1921 TotalNumValues(0) {}
1922 uint64_t TotalNumValueSites;
1923 uint64_t TotalNumValueSitesWithValueProfile;
1924 uint64_t TotalNumValues;
1925 std::vector<unsigned> ValueSitesHistogram;
1926 } ValueSitesStats;
1927
traverseAllValueSites(const InstrProfRecord & Func,uint32_t VK,ValueSitesStats & Stats,raw_fd_ostream & OS,InstrProfSymtab * Symtab)1928 static void traverseAllValueSites(const InstrProfRecord &Func, uint32_t VK,
1929 ValueSitesStats &Stats, raw_fd_ostream &OS,
1930 InstrProfSymtab *Symtab) {
1931 uint32_t NS = Func.getNumValueSites(VK);
1932 Stats.TotalNumValueSites += NS;
1933 for (size_t I = 0; I < NS; ++I) {
1934 uint32_t NV = Func.getNumValueDataForSite(VK, I);
1935 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, I);
1936 Stats.TotalNumValues += NV;
1937 if (NV) {
1938 Stats.TotalNumValueSitesWithValueProfile++;
1939 if (NV > Stats.ValueSitesHistogram.size())
1940 Stats.ValueSitesHistogram.resize(NV, 0);
1941 Stats.ValueSitesHistogram[NV - 1]++;
1942 }
1943
1944 uint64_t SiteSum = 0;
1945 for (uint32_t V = 0; V < NV; V++)
1946 SiteSum += VD[V].Count;
1947 if (SiteSum == 0)
1948 SiteSum = 1;
1949
1950 for (uint32_t V = 0; V < NV; V++) {
1951 OS << "\t[ " << format("%2u", I) << ", ";
1952 if (Symtab == nullptr)
1953 OS << format("%4" PRIu64, VD[V].Value);
1954 else
1955 OS << Symtab->getFuncName(VD[V].Value);
1956 OS << ", " << format("%10" PRId64, VD[V].Count) << " ] ("
1957 << format("%.2f%%", (VD[V].Count * 100.0 / SiteSum)) << ")\n";
1958 }
1959 }
1960 }
1961
showValueSitesStats(raw_fd_ostream & OS,uint32_t VK,ValueSitesStats & Stats)1962 static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK,
1963 ValueSitesStats &Stats) {
1964 OS << " Total number of sites: " << Stats.TotalNumValueSites << "\n";
1965 OS << " Total number of sites with values: "
1966 << Stats.TotalNumValueSitesWithValueProfile << "\n";
1967 OS << " Total number of profiled values: " << Stats.TotalNumValues << "\n";
1968
1969 OS << " Value sites histogram:\n\tNumTargets, SiteCount\n";
1970 for (unsigned I = 0; I < Stats.ValueSitesHistogram.size(); I++) {
1971 if (Stats.ValueSitesHistogram[I] > 0)
1972 OS << "\t" << I + 1 << ", " << Stats.ValueSitesHistogram[I] << "\n";
1973 }
1974 }
1975
showInstrProfile(const std::string & Filename,bool ShowCounts,uint32_t TopN,bool ShowIndirectCallTargets,bool ShowMemOPSizes,bool ShowDetailedSummary,std::vector<uint32_t> DetailedSummaryCutoffs,bool ShowAllFunctions,bool ShowCS,uint64_t ValueCutoff,bool OnlyListBelow,const std::string & ShowFunction,bool TextFormat,raw_fd_ostream & OS)1976 static int showInstrProfile(const std::string &Filename, bool ShowCounts,
1977 uint32_t TopN, bool ShowIndirectCallTargets,
1978 bool ShowMemOPSizes, bool ShowDetailedSummary,
1979 std::vector<uint32_t> DetailedSummaryCutoffs,
1980 bool ShowAllFunctions, bool ShowCS,
1981 uint64_t ValueCutoff, bool OnlyListBelow,
1982 const std::string &ShowFunction, bool TextFormat,
1983 raw_fd_ostream &OS) {
1984 auto ReaderOrErr = InstrProfReader::create(Filename);
1985 std::vector<uint32_t> Cutoffs = std::move(DetailedSummaryCutoffs);
1986 if (ShowDetailedSummary && Cutoffs.empty()) {
1987 Cutoffs = {800000, 900000, 950000, 990000, 999000, 999900, 999990};
1988 }
1989 InstrProfSummaryBuilder Builder(std::move(Cutoffs));
1990 if (Error E = ReaderOrErr.takeError())
1991 exitWithError(std::move(E), Filename);
1992
1993 auto Reader = std::move(ReaderOrErr.get());
1994 bool IsIRInstr = Reader->isIRLevelProfile();
1995 size_t ShownFunctions = 0;
1996 size_t BelowCutoffFunctions = 0;
1997 int NumVPKind = IPVK_Last - IPVK_First + 1;
1998 std::vector<ValueSitesStats> VPStats(NumVPKind);
1999
2000 auto MinCmp = [](const std::pair<std::string, uint64_t> &v1,
2001 const std::pair<std::string, uint64_t> &v2) {
2002 return v1.second > v2.second;
2003 };
2004
2005 std::priority_queue<std::pair<std::string, uint64_t>,
2006 std::vector<std::pair<std::string, uint64_t>>,
2007 decltype(MinCmp)>
2008 HottestFuncs(MinCmp);
2009
2010 if (!TextFormat && OnlyListBelow) {
2011 OS << "The list of functions with the maximum counter less than "
2012 << ValueCutoff << ":\n";
2013 }
2014
2015 // Add marker so that IR-level instrumentation round-trips properly.
2016 if (TextFormat && IsIRInstr)
2017 OS << ":ir\n";
2018
2019 for (const auto &Func : *Reader) {
2020 if (Reader->isIRLevelProfile()) {
2021 bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(Func.Hash);
2022 if (FuncIsCS != ShowCS)
2023 continue;
2024 }
2025 bool Show =
2026 ShowAllFunctions || (!ShowFunction.empty() &&
2027 Func.Name.find(ShowFunction) != Func.Name.npos);
2028
2029 bool doTextFormatDump = (Show && TextFormat);
2030
2031 if (doTextFormatDump) {
2032 InstrProfSymtab &Symtab = Reader->getSymtab();
2033 InstrProfWriter::writeRecordInText(Func.Name, Func.Hash, Func, Symtab,
2034 OS);
2035 continue;
2036 }
2037
2038 assert(Func.Counts.size() > 0 && "function missing entry counter");
2039 Builder.addRecord(Func);
2040
2041 uint64_t FuncMax = 0;
2042 uint64_t FuncSum = 0;
2043 for (size_t I = 0, E = Func.Counts.size(); I < E; ++I) {
2044 if (Func.Counts[I] == (uint64_t)-1)
2045 continue;
2046 FuncMax = std::max(FuncMax, Func.Counts[I]);
2047 FuncSum += Func.Counts[I];
2048 }
2049
2050 if (FuncMax < ValueCutoff) {
2051 ++BelowCutoffFunctions;
2052 if (OnlyListBelow) {
2053 OS << " " << Func.Name << ": (Max = " << FuncMax
2054 << " Sum = " << FuncSum << ")\n";
2055 }
2056 continue;
2057 } else if (OnlyListBelow)
2058 continue;
2059
2060 if (TopN) {
2061 if (HottestFuncs.size() == TopN) {
2062 if (HottestFuncs.top().second < FuncMax) {
2063 HottestFuncs.pop();
2064 HottestFuncs.emplace(std::make_pair(std::string(Func.Name), FuncMax));
2065 }
2066 } else
2067 HottestFuncs.emplace(std::make_pair(std::string(Func.Name), FuncMax));
2068 }
2069
2070 if (Show) {
2071 if (!ShownFunctions)
2072 OS << "Counters:\n";
2073
2074 ++ShownFunctions;
2075
2076 OS << " " << Func.Name << ":\n"
2077 << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n"
2078 << " Counters: " << Func.Counts.size() << "\n";
2079 if (!IsIRInstr)
2080 OS << " Function count: " << Func.Counts[0] << "\n";
2081
2082 if (ShowIndirectCallTargets)
2083 OS << " Indirect Call Site Count: "
2084 << Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n";
2085
2086 uint32_t NumMemOPCalls = Func.getNumValueSites(IPVK_MemOPSize);
2087 if (ShowMemOPSizes && NumMemOPCalls > 0)
2088 OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls
2089 << "\n";
2090
2091 if (ShowCounts) {
2092 OS << " Block counts: [";
2093 size_t Start = (IsIRInstr ? 0 : 1);
2094 for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) {
2095 OS << (I == Start ? "" : ", ") << Func.Counts[I];
2096 }
2097 OS << "]\n";
2098 }
2099
2100 if (ShowIndirectCallTargets) {
2101 OS << " Indirect Target Results:\n";
2102 traverseAllValueSites(Func, IPVK_IndirectCallTarget,
2103 VPStats[IPVK_IndirectCallTarget], OS,
2104 &(Reader->getSymtab()));
2105 }
2106
2107 if (ShowMemOPSizes && NumMemOPCalls > 0) {
2108 OS << " Memory Intrinsic Size Results:\n";
2109 traverseAllValueSites(Func, IPVK_MemOPSize, VPStats[IPVK_MemOPSize], OS,
2110 nullptr);
2111 }
2112 }
2113 }
2114 if (Reader->hasError())
2115 exitWithError(Reader->getError(), Filename);
2116
2117 if (TextFormat)
2118 return 0;
2119 std::unique_ptr<ProfileSummary> PS(Builder.getSummary());
2120 bool IsIR = Reader->isIRLevelProfile();
2121 OS << "Instrumentation level: " << (IsIR ? "IR" : "Front-end");
2122 if (IsIR)
2123 OS << " entry_first = " << Reader->instrEntryBBEnabled();
2124 OS << "\n";
2125 if (ShowAllFunctions || !ShowFunction.empty())
2126 OS << "Functions shown: " << ShownFunctions << "\n";
2127 OS << "Total functions: " << PS->getNumFunctions() << "\n";
2128 if (ValueCutoff > 0) {
2129 OS << "Number of functions with maximum count (< " << ValueCutoff
2130 << "): " << BelowCutoffFunctions << "\n";
2131 OS << "Number of functions with maximum count (>= " << ValueCutoff
2132 << "): " << PS->getNumFunctions() - BelowCutoffFunctions << "\n";
2133 }
2134 OS << "Maximum function count: " << PS->getMaxFunctionCount() << "\n";
2135 OS << "Maximum internal block count: " << PS->getMaxInternalCount() << "\n";
2136
2137 if (TopN) {
2138 std::vector<std::pair<std::string, uint64_t>> SortedHottestFuncs;
2139 while (!HottestFuncs.empty()) {
2140 SortedHottestFuncs.emplace_back(HottestFuncs.top());
2141 HottestFuncs.pop();
2142 }
2143 OS << "Top " << TopN
2144 << " functions with the largest internal block counts: \n";
2145 for (auto &hotfunc : llvm::reverse(SortedHottestFuncs))
2146 OS << " " << hotfunc.first << ", max count = " << hotfunc.second << "\n";
2147 }
2148
2149 if (ShownFunctions && ShowIndirectCallTargets) {
2150 OS << "Statistics for indirect call sites profile:\n";
2151 showValueSitesStats(OS, IPVK_IndirectCallTarget,
2152 VPStats[IPVK_IndirectCallTarget]);
2153 }
2154
2155 if (ShownFunctions && ShowMemOPSizes) {
2156 OS << "Statistics for memory intrinsic calls sizes profile:\n";
2157 showValueSitesStats(OS, IPVK_MemOPSize, VPStats[IPVK_MemOPSize]);
2158 }
2159
2160 if (ShowDetailedSummary) {
2161 OS << "Total number of blocks: " << PS->getNumCounts() << "\n";
2162 OS << "Total count: " << PS->getTotalCount() << "\n";
2163 PS->printDetailedSummary(OS);
2164 }
2165 return 0;
2166 }
2167
showSectionInfo(sampleprof::SampleProfileReader * Reader,raw_fd_ostream & OS)2168 static void showSectionInfo(sampleprof::SampleProfileReader *Reader,
2169 raw_fd_ostream &OS) {
2170 if (!Reader->dumpSectionInfo(OS)) {
2171 WithColor::warning() << "-show-sec-info-only is only supported for "
2172 << "sample profile in extbinary format and is "
2173 << "ignored for other formats.\n";
2174 return;
2175 }
2176 }
2177
2178 namespace {
2179 struct HotFuncInfo {
2180 StringRef FuncName;
2181 uint64_t TotalCount;
2182 double TotalCountPercent;
2183 uint64_t MaxCount;
2184 uint64_t EntryCount;
2185
HotFuncInfo__anonf47fdd840d11::HotFuncInfo2186 HotFuncInfo()
2187 : FuncName(), TotalCount(0), TotalCountPercent(0.0f), MaxCount(0),
2188 EntryCount(0) {}
2189
HotFuncInfo__anonf47fdd840d11::HotFuncInfo2190 HotFuncInfo(StringRef FN, uint64_t TS, double TSP, uint64_t MS, uint64_t ES)
2191 : FuncName(FN), TotalCount(TS), TotalCountPercent(TSP), MaxCount(MS),
2192 EntryCount(ES) {}
2193 };
2194 } // namespace
2195
2196 // Print out detailed information about hot functions in PrintValues vector.
2197 // Users specify titles and offset of every columns through ColumnTitle and
2198 // ColumnOffset. The size of ColumnTitle and ColumnOffset need to be the same
2199 // and at least 4. Besides, users can optionally give a HotFuncMetric string to
2200 // print out or let it be an empty string.
dumpHotFunctionList(const std::vector<std::string> & ColumnTitle,const std::vector<int> & ColumnOffset,const std::vector<HotFuncInfo> & PrintValues,uint64_t HotFuncCount,uint64_t TotalFuncCount,uint64_t HotProfCount,uint64_t TotalProfCount,const std::string & HotFuncMetric,raw_fd_ostream & OS)2201 static void dumpHotFunctionList(const std::vector<std::string> &ColumnTitle,
2202 const std::vector<int> &ColumnOffset,
2203 const std::vector<HotFuncInfo> &PrintValues,
2204 uint64_t HotFuncCount, uint64_t TotalFuncCount,
2205 uint64_t HotProfCount, uint64_t TotalProfCount,
2206 const std::string &HotFuncMetric,
2207 raw_fd_ostream &OS) {
2208 assert(ColumnOffset.size() == ColumnTitle.size() &&
2209 "ColumnOffset and ColumnTitle should have the same size");
2210 assert(ColumnTitle.size() >= 4 &&
2211 "ColumnTitle should have at least 4 elements");
2212 assert(TotalFuncCount > 0 &&
2213 "There should be at least one function in the profile");
2214 double TotalProfPercent = 0;
2215 if (TotalProfCount > 0)
2216 TotalProfPercent = static_cast<double>(HotProfCount) / TotalProfCount * 100;
2217
2218 formatted_raw_ostream FOS(OS);
2219 FOS << HotFuncCount << " out of " << TotalFuncCount
2220 << " functions with profile ("
2221 << format("%.2f%%",
2222 (static_cast<double>(HotFuncCount) / TotalFuncCount * 100))
2223 << ") are considered hot functions";
2224 if (!HotFuncMetric.empty())
2225 FOS << " (" << HotFuncMetric << ")";
2226 FOS << ".\n";
2227 FOS << HotProfCount << " out of " << TotalProfCount << " profile counts ("
2228 << format("%.2f%%", TotalProfPercent) << ") are from hot functions.\n";
2229
2230 for (size_t I = 0; I < ColumnTitle.size(); ++I) {
2231 FOS.PadToColumn(ColumnOffset[I]);
2232 FOS << ColumnTitle[I];
2233 }
2234 FOS << "\n";
2235
2236 for (const HotFuncInfo &R : PrintValues) {
2237 FOS.PadToColumn(ColumnOffset[0]);
2238 FOS << R.TotalCount << " (" << format("%.2f%%", R.TotalCountPercent) << ")";
2239 FOS.PadToColumn(ColumnOffset[1]);
2240 FOS << R.MaxCount;
2241 FOS.PadToColumn(ColumnOffset[2]);
2242 FOS << R.EntryCount;
2243 FOS.PadToColumn(ColumnOffset[3]);
2244 FOS << R.FuncName << "\n";
2245 }
2246 return;
2247 }
2248
2249 static int
showHotFunctionList(const StringMap<sampleprof::FunctionSamples> & Profiles,ProfileSummary & PS,raw_fd_ostream & OS)2250 showHotFunctionList(const StringMap<sampleprof::FunctionSamples> &Profiles,
2251 ProfileSummary &PS, raw_fd_ostream &OS) {
2252 using namespace sampleprof;
2253
2254 const uint32_t HotFuncCutoff = 990000;
2255 auto &SummaryVector = PS.getDetailedSummary();
2256 uint64_t MinCountThreshold = 0;
2257 for (const ProfileSummaryEntry &SummaryEntry : SummaryVector) {
2258 if (SummaryEntry.Cutoff == HotFuncCutoff) {
2259 MinCountThreshold = SummaryEntry.MinCount;
2260 break;
2261 }
2262 }
2263
2264 // Traverse all functions in the profile and keep only hot functions.
2265 // The following loop also calculates the sum of total samples of all
2266 // functions.
2267 std::multimap<uint64_t, std::pair<const FunctionSamples *, const uint64_t>,
2268 std::greater<uint64_t>>
2269 HotFunc;
2270 uint64_t ProfileTotalSample = 0;
2271 uint64_t HotFuncSample = 0;
2272 uint64_t HotFuncCount = 0;
2273
2274 for (const auto &I : Profiles) {
2275 FuncSampleStats FuncStats;
2276 const FunctionSamples &FuncProf = I.second;
2277 ProfileTotalSample += FuncProf.getTotalSamples();
2278 getFuncSampleStats(FuncProf, FuncStats, MinCountThreshold);
2279
2280 if (isFunctionHot(FuncStats, MinCountThreshold)) {
2281 HotFunc.emplace(FuncProf.getTotalSamples(),
2282 std::make_pair(&(I.second), FuncStats.MaxSample));
2283 HotFuncSample += FuncProf.getTotalSamples();
2284 ++HotFuncCount;
2285 }
2286 }
2287
2288 std::vector<std::string> ColumnTitle{"Total sample (%)", "Max sample",
2289 "Entry sample", "Function name"};
2290 std::vector<int> ColumnOffset{0, 24, 42, 58};
2291 std::string Metric =
2292 std::string("max sample >= ") + std::to_string(MinCountThreshold);
2293 std::vector<HotFuncInfo> PrintValues;
2294 for (const auto &FuncPair : HotFunc) {
2295 const FunctionSamples &Func = *FuncPair.second.first;
2296 double TotalSamplePercent =
2297 (ProfileTotalSample > 0)
2298 ? (Func.getTotalSamples() * 100.0) / ProfileTotalSample
2299 : 0;
2300 PrintValues.emplace_back(
2301 HotFuncInfo(Func.getName(), Func.getTotalSamples(), TotalSamplePercent,
2302 FuncPair.second.second, Func.getEntrySamples()));
2303 }
2304 dumpHotFunctionList(ColumnTitle, ColumnOffset, PrintValues, HotFuncCount,
2305 Profiles.size(), HotFuncSample, ProfileTotalSample,
2306 Metric, OS);
2307
2308 return 0;
2309 }
2310
showSampleProfile(const std::string & Filename,bool ShowCounts,bool ShowAllFunctions,bool ShowDetailedSummary,const std::string & ShowFunction,bool ShowProfileSymbolList,bool ShowSectionInfoOnly,bool ShowHotFuncList,raw_fd_ostream & OS)2311 static int showSampleProfile(const std::string &Filename, bool ShowCounts,
2312 bool ShowAllFunctions, bool ShowDetailedSummary,
2313 const std::string &ShowFunction,
2314 bool ShowProfileSymbolList,
2315 bool ShowSectionInfoOnly, bool ShowHotFuncList,
2316 raw_fd_ostream &OS) {
2317 using namespace sampleprof;
2318 LLVMContext Context;
2319 auto ReaderOrErr = SampleProfileReader::create(Filename, Context);
2320 if (std::error_code EC = ReaderOrErr.getError())
2321 exitWithErrorCode(EC, Filename);
2322
2323 auto Reader = std::move(ReaderOrErr.get());
2324
2325 if (ShowSectionInfoOnly) {
2326 showSectionInfo(Reader.get(), OS);
2327 return 0;
2328 }
2329
2330 if (std::error_code EC = Reader->read())
2331 exitWithErrorCode(EC, Filename);
2332
2333 if (ShowAllFunctions || ShowFunction.empty())
2334 Reader->dump(OS);
2335 else
2336 Reader->dumpFunctionProfile(ShowFunction, OS);
2337
2338 if (ShowProfileSymbolList) {
2339 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList =
2340 Reader->getProfileSymbolList();
2341 ReaderList->dump(OS);
2342 }
2343
2344 if (ShowDetailedSummary) {
2345 auto &PS = Reader->getSummary();
2346 PS.printSummary(OS);
2347 PS.printDetailedSummary(OS);
2348 }
2349
2350 if (ShowHotFuncList)
2351 showHotFunctionList(Reader->getProfiles(), Reader->getSummary(), OS);
2352
2353 return 0;
2354 }
2355
show_main(int argc,const char * argv[])2356 static int show_main(int argc, const char *argv[]) {
2357 cl::opt<std::string> Filename(cl::Positional, cl::Required,
2358 cl::desc("<profdata-file>"));
2359
2360 cl::opt<bool> ShowCounts("counts", cl::init(false),
2361 cl::desc("Show counter values for shown functions"));
2362 cl::opt<bool> TextFormat(
2363 "text", cl::init(false),
2364 cl::desc("Show instr profile data in text dump format"));
2365 cl::opt<bool> ShowIndirectCallTargets(
2366 "ic-targets", cl::init(false),
2367 cl::desc("Show indirect call site target values for shown functions"));
2368 cl::opt<bool> ShowMemOPSizes(
2369 "memop-sizes", cl::init(false),
2370 cl::desc("Show the profiled sizes of the memory intrinsic calls "
2371 "for shown functions"));
2372 cl::opt<bool> ShowDetailedSummary("detailed-summary", cl::init(false),
2373 cl::desc("Show detailed profile summary"));
2374 cl::list<uint32_t> DetailedSummaryCutoffs(
2375 cl::CommaSeparated, "detailed-summary-cutoffs",
2376 cl::desc(
2377 "Cutoff percentages (times 10000) for generating detailed summary"),
2378 cl::value_desc("800000,901000,999999"));
2379 cl::opt<bool> ShowHotFuncList(
2380 "hot-func-list", cl::init(false),
2381 cl::desc("Show profile summary of a list of hot functions"));
2382 cl::opt<bool> ShowAllFunctions("all-functions", cl::init(false),
2383 cl::desc("Details for every function"));
2384 cl::opt<bool> ShowCS("showcs", cl::init(false),
2385 cl::desc("Show context sensitive counts"));
2386 cl::opt<std::string> ShowFunction("function",
2387 cl::desc("Details for matching functions"));
2388
2389 cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
2390 cl::init("-"), cl::desc("Output file"));
2391 cl::alias OutputFilenameA("o", cl::desc("Alias for --output"),
2392 cl::aliasopt(OutputFilename));
2393 cl::opt<ProfileKinds> ProfileKind(
2394 cl::desc("Profile kind:"), cl::init(instr),
2395 cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
2396 clEnumVal(sample, "Sample profile")));
2397 cl::opt<uint32_t> TopNFunctions(
2398 "topn", cl::init(0),
2399 cl::desc("Show the list of functions with the largest internal counts"));
2400 cl::opt<uint32_t> ValueCutoff(
2401 "value-cutoff", cl::init(0),
2402 cl::desc("Set the count value cutoff. Functions with the maximum count "
2403 "less than this value will not be printed out. (Default is 0)"));
2404 cl::opt<bool> OnlyListBelow(
2405 "list-below-cutoff", cl::init(false),
2406 cl::desc("Only output names of functions whose max count values are "
2407 "below the cutoff value"));
2408 cl::opt<bool> ShowProfileSymbolList(
2409 "show-prof-sym-list", cl::init(false),
2410 cl::desc("Show profile symbol list if it exists in the profile. "));
2411 cl::opt<bool> ShowSectionInfoOnly(
2412 "show-sec-info-only", cl::init(false),
2413 cl::desc("Show the information of each section in the sample profile. "
2414 "The flag is only usable when the sample profile is in "
2415 "extbinary format"));
2416
2417 cl::ParseCommandLineOptions(argc, argv, "LLVM profile data summary\n");
2418
2419 if (OutputFilename.empty())
2420 OutputFilename = "-";
2421
2422 if (!Filename.compare(OutputFilename)) {
2423 errs() << sys::path::filename(argv[0])
2424 << ": Input file name cannot be the same as the output file name!\n";
2425 return 1;
2426 }
2427
2428 std::error_code EC;
2429 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_Text);
2430 if (EC)
2431 exitWithErrorCode(EC, OutputFilename);
2432
2433 if (ShowAllFunctions && !ShowFunction.empty())
2434 WithColor::warning() << "-function argument ignored: showing all functions\n";
2435
2436 if (ProfileKind == instr)
2437 return showInstrProfile(Filename, ShowCounts, TopNFunctions,
2438 ShowIndirectCallTargets, ShowMemOPSizes,
2439 ShowDetailedSummary, DetailedSummaryCutoffs,
2440 ShowAllFunctions, ShowCS, ValueCutoff,
2441 OnlyListBelow, ShowFunction, TextFormat, OS);
2442 else
2443 return showSampleProfile(Filename, ShowCounts, ShowAllFunctions,
2444 ShowDetailedSummary, ShowFunction,
2445 ShowProfileSymbolList, ShowSectionInfoOnly,
2446 ShowHotFuncList, OS);
2447 }
2448
main(int argc,const char * argv[])2449 int main(int argc, const char *argv[]) {
2450 InitLLVM X(argc, argv);
2451
2452 StringRef ProgName(sys::path::filename(argv[0]));
2453 if (argc > 1) {
2454 int (*func)(int, const char *[]) = nullptr;
2455
2456 if (strcmp(argv[1], "merge") == 0)
2457 func = merge_main;
2458 else if (strcmp(argv[1], "show") == 0)
2459 func = show_main;
2460 else if (strcmp(argv[1], "overlap") == 0)
2461 func = overlap_main;
2462
2463 if (func) {
2464 std::string Invocation(ProgName.str() + " " + argv[1]);
2465 argv[1] = Invocation.c_str();
2466 return func(argc - 1, argv + 1);
2467 }
2468
2469 if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "-help") == 0 ||
2470 strcmp(argv[1], "--help") == 0) {
2471
2472 errs() << "OVERVIEW: LLVM profile data tools\n\n"
2473 << "USAGE: " << ProgName << " <command> [args...]\n"
2474 << "USAGE: " << ProgName << " <command> -help\n\n"
2475 << "See each individual command --help for more details.\n"
2476 << "Available commands: merge, show, overlap\n";
2477 return 0;
2478 }
2479 }
2480
2481 if (argc < 2)
2482 errs() << ProgName << ": No command specified!\n";
2483 else
2484 errs() << ProgName << ": Unknown command!\n";
2485
2486 errs() << "USAGE: " << ProgName << " <merge|show|overlap> [args...]\n";
2487 return 1;
2488 }
2489