1 //===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the class that reads LLVM sample profiles. It 10 // supports three file formats: text, binary and gcov. 11 // 12 // The textual representation is useful for debugging and testing purposes. The 13 // binary representation is more compact, resulting in smaller file sizes. 14 // 15 // The gcov encoding is the one generated by GCC's AutoFDO profile creation 16 // tool (https://github.com/google/autofdo) 17 // 18 // All three encodings can be used interchangeably as an input sample profile. 19 // 20 //===----------------------------------------------------------------------===// 21 22 #include "llvm/ProfileData/SampleProfReader.h" 23 #include "llvm/ADT/DenseMap.h" 24 #include "llvm/ADT/STLExtras.h" 25 #include "llvm/ADT/StringRef.h" 26 #include "llvm/IR/ProfileSummary.h" 27 #include "llvm/ProfileData/ProfileCommon.h" 28 #include "llvm/ProfileData/SampleProf.h" 29 #include "llvm/Support/ErrorOr.h" 30 #include "llvm/Support/LEB128.h" 31 #include "llvm/Support/LineIterator.h" 32 #include "llvm/Support/MD5.h" 33 #include "llvm/Support/MemoryBuffer.h" 34 #include "llvm/Support/raw_ostream.h" 35 #include <algorithm> 36 #include <cstddef> 37 #include <cstdint> 38 #include <limits> 39 #include <memory> 40 #include <system_error> 41 #include <vector> 42 43 using namespace llvm; 44 using namespace sampleprof; 45 46 /// Dump the function profile for \p FName. 47 /// 48 /// \param FName Name of the function to print. 49 /// \param OS Stream to emit the output to. 50 void SampleProfileReader::dumpFunctionProfile(StringRef FName, 51 raw_ostream &OS) { 52 OS << "Function: " << FName << ": " << Profiles[FName]; 53 } 54 55 /// Dump all the function profiles found on stream \p OS. 56 void SampleProfileReader::dump(raw_ostream &OS) { 57 for (const auto &I : Profiles) 58 dumpFunctionProfile(I.getKey(), OS); 59 } 60 61 /// Parse \p Input as function head. 62 /// 63 /// Parse one line of \p Input, and update function name in \p FName, 64 /// function's total sample count in \p NumSamples, function's entry 65 /// count in \p NumHeadSamples. 66 /// 67 /// \returns true if parsing is successful. 68 static bool ParseHead(const StringRef &Input, StringRef &FName, 69 uint64_t &NumSamples, uint64_t &NumHeadSamples) { 70 if (Input[0] == ' ') 71 return false; 72 size_t n2 = Input.rfind(':'); 73 size_t n1 = Input.rfind(':', n2 - 1); 74 FName = Input.substr(0, n1); 75 if (Input.substr(n1 + 1, n2 - n1 - 1).getAsInteger(10, NumSamples)) 76 return false; 77 if (Input.substr(n2 + 1).getAsInteger(10, NumHeadSamples)) 78 return false; 79 return true; 80 } 81 82 /// Returns true if line offset \p L is legal (only has 16 bits). 83 static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; } 84 85 /// Parse \p Input as line sample. 86 /// 87 /// \param Input input line. 88 /// \param IsCallsite true if the line represents an inlined callsite. 89 /// \param Depth the depth of the inline stack. 90 /// \param NumSamples total samples of the line/inlined callsite. 91 /// \param LineOffset line offset to the start of the function. 92 /// \param Discriminator discriminator of the line. 93 /// \param TargetCountMap map from indirect call target to count. 94 /// 95 /// returns true if parsing is successful. 96 static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth, 97 uint64_t &NumSamples, uint32_t &LineOffset, 98 uint32_t &Discriminator, StringRef &CalleeName, 99 DenseMap<StringRef, uint64_t> &TargetCountMap) { 100 for (Depth = 0; Input[Depth] == ' '; Depth++) 101 ; 102 if (Depth == 0) 103 return false; 104 105 size_t n1 = Input.find(':'); 106 StringRef Loc = Input.substr(Depth, n1 - Depth); 107 size_t n2 = Loc.find('.'); 108 if (n2 == StringRef::npos) { 109 if (Loc.getAsInteger(10, LineOffset) || !isOffsetLegal(LineOffset)) 110 return false; 111 Discriminator = 0; 112 } else { 113 if (Loc.substr(0, n2).getAsInteger(10, LineOffset)) 114 return false; 115 if (Loc.substr(n2 + 1).getAsInteger(10, Discriminator)) 116 return false; 117 } 118 119 StringRef Rest = Input.substr(n1 + 2); 120 if (Rest[0] >= '0' && Rest[0] <= '9') { 121 IsCallsite = false; 122 size_t n3 = Rest.find(' '); 123 if (n3 == StringRef::npos) { 124 if (Rest.getAsInteger(10, NumSamples)) 125 return false; 126 } else { 127 if (Rest.substr(0, n3).getAsInteger(10, NumSamples)) 128 return false; 129 } 130 // Find call targets and their sample counts. 131 // Note: In some cases, there are symbols in the profile which are not 132 // mangled. To accommodate such cases, use colon + integer pairs as the 133 // anchor points. 134 // An example: 135 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437 136 // ":1000" and ":437" are used as anchor points so the string above will 137 // be interpreted as 138 // target: _M_construct<char *> 139 // count: 1000 140 // target: string_view<std::allocator<char> > 141 // count: 437 142 while (n3 != StringRef::npos) { 143 n3 += Rest.substr(n3).find_first_not_of(' '); 144 Rest = Rest.substr(n3); 145 n3 = Rest.find_first_of(':'); 146 if (n3 == StringRef::npos || n3 == 0) 147 return false; 148 149 StringRef Target; 150 uint64_t count, n4; 151 while (true) { 152 // Get the segment after the current colon. 153 StringRef AfterColon = Rest.substr(n3 + 1); 154 // Get the target symbol before the current colon. 155 Target = Rest.substr(0, n3); 156 // Check if the word after the current colon is an integer. 157 n4 = AfterColon.find_first_of(' '); 158 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size(); 159 StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1); 160 if (!WordAfterColon.getAsInteger(10, count)) 161 break; 162 163 // Try to find the next colon. 164 uint64_t n5 = AfterColon.find_first_of(':'); 165 if (n5 == StringRef::npos) 166 return false; 167 n3 += n5 + 1; 168 } 169 170 // An anchor point is found. Save the {target, count} pair 171 TargetCountMap[Target] = count; 172 if (n4 == Rest.size()) 173 break; 174 // Change n3 to the next blank space after colon + integer pair. 175 n3 = n4; 176 } 177 } else { 178 IsCallsite = true; 179 size_t n3 = Rest.find_last_of(':'); 180 CalleeName = Rest.substr(0, n3); 181 if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples)) 182 return false; 183 } 184 return true; 185 } 186 187 /// Load samples from a text file. 188 /// 189 /// See the documentation at the top of the file for an explanation of 190 /// the expected format. 191 /// 192 /// \returns true if the file was loaded successfully, false otherwise. 193 std::error_code SampleProfileReaderText::read() { 194 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#'); 195 sampleprof_error Result = sampleprof_error::success; 196 197 InlineCallStack InlineStack; 198 199 for (; !LineIt.is_at_eof(); ++LineIt) { 200 if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#') 201 continue; 202 // Read the header of each function. 203 // 204 // Note that for function identifiers we are actually expecting 205 // mangled names, but we may not always get them. This happens when 206 // the compiler decides not to emit the function (e.g., it was inlined 207 // and removed). In this case, the binary will not have the linkage 208 // name for the function, so the profiler will emit the function's 209 // unmangled name, which may contain characters like ':' and '>' in its 210 // name (member functions, templates, etc). 211 // 212 // The only requirement we place on the identifier, then, is that it 213 // should not begin with a number. 214 if ((*LineIt)[0] != ' ') { 215 uint64_t NumSamples, NumHeadSamples; 216 StringRef FName; 217 if (!ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) { 218 reportError(LineIt.line_number(), 219 "Expected 'mangled_name:NUM:NUM', found " + *LineIt); 220 return sampleprof_error::malformed; 221 } 222 Profiles[FName] = FunctionSamples(); 223 FunctionSamples &FProfile = Profiles[FName]; 224 FProfile.setName(FName); 225 MergeResult(Result, FProfile.addTotalSamples(NumSamples)); 226 MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); 227 InlineStack.clear(); 228 InlineStack.push_back(&FProfile); 229 } else { 230 uint64_t NumSamples; 231 StringRef FName; 232 DenseMap<StringRef, uint64_t> TargetCountMap; 233 bool IsCallsite; 234 uint32_t Depth, LineOffset, Discriminator; 235 if (!ParseLine(*LineIt, IsCallsite, Depth, NumSamples, LineOffset, 236 Discriminator, FName, TargetCountMap)) { 237 reportError(LineIt.line_number(), 238 "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + 239 *LineIt); 240 return sampleprof_error::malformed; 241 } 242 if (IsCallsite) { 243 while (InlineStack.size() > Depth) { 244 InlineStack.pop_back(); 245 } 246 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt( 247 LineLocation(LineOffset, Discriminator))[FName]; 248 FSamples.setName(FName); 249 MergeResult(Result, FSamples.addTotalSamples(NumSamples)); 250 InlineStack.push_back(&FSamples); 251 } else { 252 while (InlineStack.size() > Depth) { 253 InlineStack.pop_back(); 254 } 255 FunctionSamples &FProfile = *InlineStack.back(); 256 for (const auto &name_count : TargetCountMap) { 257 MergeResult(Result, FProfile.addCalledTargetSamples( 258 LineOffset, Discriminator, name_count.first, 259 name_count.second)); 260 } 261 MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator, 262 NumSamples)); 263 } 264 } 265 } 266 if (Result == sampleprof_error::success) 267 computeSummary(); 268 269 return Result; 270 } 271 272 bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) { 273 bool result = false; 274 275 // Check that the first non-comment line is a valid function header. 276 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#'); 277 if (!LineIt.is_at_eof()) { 278 if ((*LineIt)[0] != ' ') { 279 uint64_t NumSamples, NumHeadSamples; 280 StringRef FName; 281 result = ParseHead(*LineIt, FName, NumSamples, NumHeadSamples); 282 } 283 } 284 285 return result; 286 } 287 288 template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() { 289 unsigned NumBytesRead = 0; 290 std::error_code EC; 291 uint64_t Val = decodeULEB128(Data, &NumBytesRead); 292 293 if (Val > std::numeric_limits<T>::max()) 294 EC = sampleprof_error::malformed; 295 else if (Data + NumBytesRead > End) 296 EC = sampleprof_error::truncated; 297 else 298 EC = sampleprof_error::success; 299 300 if (EC) { 301 reportError(0, EC.message()); 302 return EC; 303 } 304 305 Data += NumBytesRead; 306 return static_cast<T>(Val); 307 } 308 309 ErrorOr<StringRef> SampleProfileReaderBinary::readString() { 310 std::error_code EC; 311 StringRef Str(reinterpret_cast<const char *>(Data)); 312 if (Data + Str.size() + 1 > End) { 313 EC = sampleprof_error::truncated; 314 reportError(0, EC.message()); 315 return EC; 316 } 317 318 Data += Str.size() + 1; 319 return Str; 320 } 321 322 template <typename T> 323 ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() { 324 std::error_code EC; 325 326 if (Data + sizeof(T) > End) { 327 EC = sampleprof_error::truncated; 328 reportError(0, EC.message()); 329 return EC; 330 } 331 332 using namespace support; 333 T Val = endian::readNext<T, little, unaligned>(Data); 334 return Val; 335 } 336 337 template <typename T> 338 inline ErrorOr<uint32_t> SampleProfileReaderBinary::readStringIndex(T &Table) { 339 std::error_code EC; 340 auto Idx = readNumber<uint32_t>(); 341 if (std::error_code EC = Idx.getError()) 342 return EC; 343 if (*Idx >= Table.size()) 344 return sampleprof_error::truncated_name_table; 345 return *Idx; 346 } 347 348 ErrorOr<StringRef> SampleProfileReaderRawBinary::readStringFromTable() { 349 auto Idx = readStringIndex(NameTable); 350 if (std::error_code EC = Idx.getError()) 351 return EC; 352 353 return NameTable[*Idx]; 354 } 355 356 ErrorOr<StringRef> SampleProfileReaderCompactBinary::readStringFromTable() { 357 auto Idx = readStringIndex(NameTable); 358 if (std::error_code EC = Idx.getError()) 359 return EC; 360 361 return StringRef(NameTable[*Idx]); 362 } 363 364 std::error_code 365 SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { 366 auto NumSamples = readNumber<uint64_t>(); 367 if (std::error_code EC = NumSamples.getError()) 368 return EC; 369 FProfile.addTotalSamples(*NumSamples); 370 371 // Read the samples in the body. 372 auto NumRecords = readNumber<uint32_t>(); 373 if (std::error_code EC = NumRecords.getError()) 374 return EC; 375 376 for (uint32_t I = 0; I < *NumRecords; ++I) { 377 auto LineOffset = readNumber<uint64_t>(); 378 if (std::error_code EC = LineOffset.getError()) 379 return EC; 380 381 if (!isOffsetLegal(*LineOffset)) { 382 return std::error_code(); 383 } 384 385 auto Discriminator = readNumber<uint64_t>(); 386 if (std::error_code EC = Discriminator.getError()) 387 return EC; 388 389 auto NumSamples = readNumber<uint64_t>(); 390 if (std::error_code EC = NumSamples.getError()) 391 return EC; 392 393 auto NumCalls = readNumber<uint32_t>(); 394 if (std::error_code EC = NumCalls.getError()) 395 return EC; 396 397 for (uint32_t J = 0; J < *NumCalls; ++J) { 398 auto CalledFunction(readStringFromTable()); 399 if (std::error_code EC = CalledFunction.getError()) 400 return EC; 401 402 auto CalledFunctionSamples = readNumber<uint64_t>(); 403 if (std::error_code EC = CalledFunctionSamples.getError()) 404 return EC; 405 406 FProfile.addCalledTargetSamples(*LineOffset, *Discriminator, 407 *CalledFunction, *CalledFunctionSamples); 408 } 409 410 FProfile.addBodySamples(*LineOffset, *Discriminator, *NumSamples); 411 } 412 413 // Read all the samples for inlined function calls. 414 auto NumCallsites = readNumber<uint32_t>(); 415 if (std::error_code EC = NumCallsites.getError()) 416 return EC; 417 418 for (uint32_t J = 0; J < *NumCallsites; ++J) { 419 auto LineOffset = readNumber<uint64_t>(); 420 if (std::error_code EC = LineOffset.getError()) 421 return EC; 422 423 auto Discriminator = readNumber<uint64_t>(); 424 if (std::error_code EC = Discriminator.getError()) 425 return EC; 426 427 auto FName(readStringFromTable()); 428 if (std::error_code EC = FName.getError()) 429 return EC; 430 431 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt( 432 LineLocation(*LineOffset, *Discriminator))[*FName]; 433 CalleeProfile.setName(*FName); 434 if (std::error_code EC = readProfile(CalleeProfile)) 435 return EC; 436 } 437 438 return sampleprof_error::success; 439 } 440 441 std::error_code SampleProfileReaderBinary::readFuncProfile() { 442 auto NumHeadSamples = readNumber<uint64_t>(); 443 if (std::error_code EC = NumHeadSamples.getError()) 444 return EC; 445 446 auto FName(readStringFromTable()); 447 if (std::error_code EC = FName.getError()) 448 return EC; 449 450 Profiles[*FName] = FunctionSamples(); 451 FunctionSamples &FProfile = Profiles[*FName]; 452 FProfile.setName(*FName); 453 454 FProfile.addHeadSamples(*NumHeadSamples); 455 456 if (std::error_code EC = readProfile(FProfile)) 457 return EC; 458 return sampleprof_error::success; 459 } 460 461 std::error_code SampleProfileReaderBinary::read() { 462 while (!at_eof()) { 463 if (std::error_code EC = readFuncProfile()) 464 return EC; 465 } 466 467 return sampleprof_error::success; 468 } 469 470 std::error_code SampleProfileReaderCompactBinary::read() { 471 for (auto Name : FuncsToUse) { 472 auto GUID = std::to_string(MD5Hash(Name)); 473 auto iter = FuncOffsetTable.find(StringRef(GUID)); 474 if (iter == FuncOffsetTable.end()) 475 continue; 476 const uint8_t *SavedData = Data; 477 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) + 478 iter->second; 479 if (std::error_code EC = readFuncProfile()) 480 return EC; 481 Data = SavedData; 482 } 483 return sampleprof_error::success; 484 } 485 486 std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) { 487 if (Magic == SPMagic()) 488 return sampleprof_error::success; 489 return sampleprof_error::bad_magic; 490 } 491 492 std::error_code 493 SampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) { 494 if (Magic == SPMagic(SPF_Compact_Binary)) 495 return sampleprof_error::success; 496 return sampleprof_error::bad_magic; 497 } 498 499 std::error_code SampleProfileReaderRawBinary::readNameTable() { 500 auto Size = readNumber<uint32_t>(); 501 if (std::error_code EC = Size.getError()) 502 return EC; 503 NameTable.reserve(*Size); 504 for (uint32_t I = 0; I < *Size; ++I) { 505 auto Name(readString()); 506 if (std::error_code EC = Name.getError()) 507 return EC; 508 NameTable.push_back(*Name); 509 } 510 511 return sampleprof_error::success; 512 } 513 514 std::error_code SampleProfileReaderCompactBinary::readNameTable() { 515 auto Size = readNumber<uint64_t>(); 516 if (std::error_code EC = Size.getError()) 517 return EC; 518 NameTable.reserve(*Size); 519 for (uint32_t I = 0; I < *Size; ++I) { 520 auto FID = readNumber<uint64_t>(); 521 if (std::error_code EC = FID.getError()) 522 return EC; 523 NameTable.push_back(std::to_string(*FID)); 524 } 525 return sampleprof_error::success; 526 } 527 528 std::error_code SampleProfileReaderBinary::readHeader() { 529 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()); 530 End = Data + Buffer->getBufferSize(); 531 532 // Read and check the magic identifier. 533 auto Magic = readNumber<uint64_t>(); 534 if (std::error_code EC = Magic.getError()) 535 return EC; 536 else if (std::error_code EC = verifySPMagic(*Magic)) 537 return EC; 538 539 // Read the version number. 540 auto Version = readNumber<uint64_t>(); 541 if (std::error_code EC = Version.getError()) 542 return EC; 543 else if (*Version != SPVersion()) 544 return sampleprof_error::unsupported_version; 545 546 if (std::error_code EC = readSummary()) 547 return EC; 548 549 if (std::error_code EC = readNameTable()) 550 return EC; 551 return sampleprof_error::success; 552 } 553 554 std::error_code SampleProfileReaderCompactBinary::readHeader() { 555 SampleProfileReaderBinary::readHeader(); 556 if (std::error_code EC = readFuncOffsetTable()) 557 return EC; 558 return sampleprof_error::success; 559 } 560 561 std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() { 562 auto TableOffset = readUnencodedNumber<uint64_t>(); 563 if (std::error_code EC = TableOffset.getError()) 564 return EC; 565 566 const uint8_t *SavedData = Data; 567 const uint8_t *TableStart = 568 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) + 569 *TableOffset; 570 Data = TableStart; 571 572 auto Size = readNumber<uint64_t>(); 573 if (std::error_code EC = Size.getError()) 574 return EC; 575 576 FuncOffsetTable.reserve(*Size); 577 for (uint32_t I = 0; I < *Size; ++I) { 578 auto FName(readStringFromTable()); 579 if (std::error_code EC = FName.getError()) 580 return EC; 581 582 auto Offset = readNumber<uint64_t>(); 583 if (std::error_code EC = Offset.getError()) 584 return EC; 585 586 FuncOffsetTable[*FName] = *Offset; 587 } 588 End = TableStart; 589 Data = SavedData; 590 return sampleprof_error::success; 591 } 592 593 void SampleProfileReaderCompactBinary::collectFuncsToUse(const Module &M) { 594 FuncsToUse.clear(); 595 for (auto &F : M) { 596 StringRef CanonName = FunctionSamples::getCanonicalFnName(F); 597 FuncsToUse.insert(CanonName); 598 } 599 } 600 601 std::error_code SampleProfileReaderBinary::readSummaryEntry( 602 std::vector<ProfileSummaryEntry> &Entries) { 603 auto Cutoff = readNumber<uint64_t>(); 604 if (std::error_code EC = Cutoff.getError()) 605 return EC; 606 607 auto MinBlockCount = readNumber<uint64_t>(); 608 if (std::error_code EC = MinBlockCount.getError()) 609 return EC; 610 611 auto NumBlocks = readNumber<uint64_t>(); 612 if (std::error_code EC = NumBlocks.getError()) 613 return EC; 614 615 Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks); 616 return sampleprof_error::success; 617 } 618 619 std::error_code SampleProfileReaderBinary::readSummary() { 620 auto TotalCount = readNumber<uint64_t>(); 621 if (std::error_code EC = TotalCount.getError()) 622 return EC; 623 624 auto MaxBlockCount = readNumber<uint64_t>(); 625 if (std::error_code EC = MaxBlockCount.getError()) 626 return EC; 627 628 auto MaxFunctionCount = readNumber<uint64_t>(); 629 if (std::error_code EC = MaxFunctionCount.getError()) 630 return EC; 631 632 auto NumBlocks = readNumber<uint64_t>(); 633 if (std::error_code EC = NumBlocks.getError()) 634 return EC; 635 636 auto NumFunctions = readNumber<uint64_t>(); 637 if (std::error_code EC = NumFunctions.getError()) 638 return EC; 639 640 auto NumSummaryEntries = readNumber<uint64_t>(); 641 if (std::error_code EC = NumSummaryEntries.getError()) 642 return EC; 643 644 std::vector<ProfileSummaryEntry> Entries; 645 for (unsigned i = 0; i < *NumSummaryEntries; i++) { 646 std::error_code EC = readSummaryEntry(Entries); 647 if (EC != sampleprof_error::success) 648 return EC; 649 } 650 Summary = llvm::make_unique<ProfileSummary>( 651 ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0, 652 *MaxFunctionCount, *NumBlocks, *NumFunctions); 653 654 return sampleprof_error::success; 655 } 656 657 bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) { 658 const uint8_t *Data = 659 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart()); 660 uint64_t Magic = decodeULEB128(Data); 661 return Magic == SPMagic(); 662 } 663 664 bool SampleProfileReaderCompactBinary::hasFormat(const MemoryBuffer &Buffer) { 665 const uint8_t *Data = 666 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart()); 667 uint64_t Magic = decodeULEB128(Data); 668 return Magic == SPMagic(SPF_Compact_Binary); 669 } 670 671 std::error_code SampleProfileReaderGCC::skipNextWord() { 672 uint32_t dummy; 673 if (!GcovBuffer.readInt(dummy)) 674 return sampleprof_error::truncated; 675 return sampleprof_error::success; 676 } 677 678 template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() { 679 if (sizeof(T) <= sizeof(uint32_t)) { 680 uint32_t Val; 681 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max()) 682 return static_cast<T>(Val); 683 } else if (sizeof(T) <= sizeof(uint64_t)) { 684 uint64_t Val; 685 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max()) 686 return static_cast<T>(Val); 687 } 688 689 std::error_code EC = sampleprof_error::malformed; 690 reportError(0, EC.message()); 691 return EC; 692 } 693 694 ErrorOr<StringRef> SampleProfileReaderGCC::readString() { 695 StringRef Str; 696 if (!GcovBuffer.readString(Str)) 697 return sampleprof_error::truncated; 698 return Str; 699 } 700 701 std::error_code SampleProfileReaderGCC::readHeader() { 702 // Read the magic identifier. 703 if (!GcovBuffer.readGCDAFormat()) 704 return sampleprof_error::unrecognized_format; 705 706 // Read the version number. Note - the GCC reader does not validate this 707 // version, but the profile creator generates v704. 708 GCOV::GCOVVersion version; 709 if (!GcovBuffer.readGCOVVersion(version)) 710 return sampleprof_error::unrecognized_format; 711 712 if (version != GCOV::V704) 713 return sampleprof_error::unsupported_version; 714 715 // Skip the empty integer. 716 if (std::error_code EC = skipNextWord()) 717 return EC; 718 719 return sampleprof_error::success; 720 } 721 722 std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) { 723 uint32_t Tag; 724 if (!GcovBuffer.readInt(Tag)) 725 return sampleprof_error::truncated; 726 727 if (Tag != Expected) 728 return sampleprof_error::malformed; 729 730 if (std::error_code EC = skipNextWord()) 731 return EC; 732 733 return sampleprof_error::success; 734 } 735 736 std::error_code SampleProfileReaderGCC::readNameTable() { 737 if (std::error_code EC = readSectionTag(GCOVTagAFDOFileNames)) 738 return EC; 739 740 uint32_t Size; 741 if (!GcovBuffer.readInt(Size)) 742 return sampleprof_error::truncated; 743 744 for (uint32_t I = 0; I < Size; ++I) { 745 StringRef Str; 746 if (!GcovBuffer.readString(Str)) 747 return sampleprof_error::truncated; 748 Names.push_back(Str); 749 } 750 751 return sampleprof_error::success; 752 } 753 754 std::error_code SampleProfileReaderGCC::readFunctionProfiles() { 755 if (std::error_code EC = readSectionTag(GCOVTagAFDOFunction)) 756 return EC; 757 758 uint32_t NumFunctions; 759 if (!GcovBuffer.readInt(NumFunctions)) 760 return sampleprof_error::truncated; 761 762 InlineCallStack Stack; 763 for (uint32_t I = 0; I < NumFunctions; ++I) 764 if (std::error_code EC = readOneFunctionProfile(Stack, true, 0)) 765 return EC; 766 767 computeSummary(); 768 return sampleprof_error::success; 769 } 770 771 std::error_code SampleProfileReaderGCC::readOneFunctionProfile( 772 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) { 773 uint64_t HeadCount = 0; 774 if (InlineStack.size() == 0) 775 if (!GcovBuffer.readInt64(HeadCount)) 776 return sampleprof_error::truncated; 777 778 uint32_t NameIdx; 779 if (!GcovBuffer.readInt(NameIdx)) 780 return sampleprof_error::truncated; 781 782 StringRef Name(Names[NameIdx]); 783 784 uint32_t NumPosCounts; 785 if (!GcovBuffer.readInt(NumPosCounts)) 786 return sampleprof_error::truncated; 787 788 uint32_t NumCallsites; 789 if (!GcovBuffer.readInt(NumCallsites)) 790 return sampleprof_error::truncated; 791 792 FunctionSamples *FProfile = nullptr; 793 if (InlineStack.size() == 0) { 794 // If this is a top function that we have already processed, do not 795 // update its profile again. This happens in the presence of 796 // function aliases. Since these aliases share the same function 797 // body, there will be identical replicated profiles for the 798 // original function. In this case, we simply not bother updating 799 // the profile of the original function. 800 FProfile = &Profiles[Name]; 801 FProfile->addHeadSamples(HeadCount); 802 if (FProfile->getTotalSamples() > 0) 803 Update = false; 804 } else { 805 // Otherwise, we are reading an inlined instance. The top of the 806 // inline stack contains the profile of the caller. Insert this 807 // callee in the caller's CallsiteMap. 808 FunctionSamples *CallerProfile = InlineStack.front(); 809 uint32_t LineOffset = Offset >> 16; 810 uint32_t Discriminator = Offset & 0xffff; 811 FProfile = &CallerProfile->functionSamplesAt( 812 LineLocation(LineOffset, Discriminator))[Name]; 813 } 814 FProfile->setName(Name); 815 816 for (uint32_t I = 0; I < NumPosCounts; ++I) { 817 uint32_t Offset; 818 if (!GcovBuffer.readInt(Offset)) 819 return sampleprof_error::truncated; 820 821 uint32_t NumTargets; 822 if (!GcovBuffer.readInt(NumTargets)) 823 return sampleprof_error::truncated; 824 825 uint64_t Count; 826 if (!GcovBuffer.readInt64(Count)) 827 return sampleprof_error::truncated; 828 829 // The line location is encoded in the offset as: 830 // high 16 bits: line offset to the start of the function. 831 // low 16 bits: discriminator. 832 uint32_t LineOffset = Offset >> 16; 833 uint32_t Discriminator = Offset & 0xffff; 834 835 InlineCallStack NewStack; 836 NewStack.push_back(FProfile); 837 NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end()); 838 if (Update) { 839 // Walk up the inline stack, adding the samples on this line to 840 // the total sample count of the callers in the chain. 841 for (auto CallerProfile : NewStack) 842 CallerProfile->addTotalSamples(Count); 843 844 // Update the body samples for the current profile. 845 FProfile->addBodySamples(LineOffset, Discriminator, Count); 846 } 847 848 // Process the list of functions called at an indirect call site. 849 // These are all the targets that a function pointer (or virtual 850 // function) resolved at runtime. 851 for (uint32_t J = 0; J < NumTargets; J++) { 852 uint32_t HistVal; 853 if (!GcovBuffer.readInt(HistVal)) 854 return sampleprof_error::truncated; 855 856 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN) 857 return sampleprof_error::malformed; 858 859 uint64_t TargetIdx; 860 if (!GcovBuffer.readInt64(TargetIdx)) 861 return sampleprof_error::truncated; 862 StringRef TargetName(Names[TargetIdx]); 863 864 uint64_t TargetCount; 865 if (!GcovBuffer.readInt64(TargetCount)) 866 return sampleprof_error::truncated; 867 868 if (Update) 869 FProfile->addCalledTargetSamples(LineOffset, Discriminator, 870 TargetName, TargetCount); 871 } 872 } 873 874 // Process all the inlined callers into the current function. These 875 // are all the callsites that were inlined into this function. 876 for (uint32_t I = 0; I < NumCallsites; I++) { 877 // The offset is encoded as: 878 // high 16 bits: line offset to the start of the function. 879 // low 16 bits: discriminator. 880 uint32_t Offset; 881 if (!GcovBuffer.readInt(Offset)) 882 return sampleprof_error::truncated; 883 InlineCallStack NewStack; 884 NewStack.push_back(FProfile); 885 NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end()); 886 if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset)) 887 return EC; 888 } 889 890 return sampleprof_error::success; 891 } 892 893 /// Read a GCC AutoFDO profile. 894 /// 895 /// This format is generated by the Linux Perf conversion tool at 896 /// https://github.com/google/autofdo. 897 std::error_code SampleProfileReaderGCC::read() { 898 // Read the string table. 899 if (std::error_code EC = readNameTable()) 900 return EC; 901 902 // Read the source profile. 903 if (std::error_code EC = readFunctionProfiles()) 904 return EC; 905 906 return sampleprof_error::success; 907 } 908 909 bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) { 910 StringRef Magic(reinterpret_cast<const char *>(Buffer.getBufferStart())); 911 return Magic == "adcg*704"; 912 } 913 914 std::error_code SampleProfileReaderItaniumRemapper::read() { 915 // If the underlying data is in compact format, we can't remap it because 916 // we don't know what the original function names were. 917 if (getFormat() == SPF_Compact_Binary) { 918 Ctx.diagnose(DiagnosticInfoSampleProfile( 919 Buffer->getBufferIdentifier(), 920 "Profile data remapping cannot be applied to profile data " 921 "in compact format (original mangled names are not available).", 922 DS_Warning)); 923 return sampleprof_error::success; 924 } 925 926 if (Error E = Remappings.read(*Buffer)) { 927 handleAllErrors( 928 std::move(E), [&](const SymbolRemappingParseError &ParseError) { 929 reportError(ParseError.getLineNum(), ParseError.getMessage()); 930 }); 931 return sampleprof_error::malformed; 932 } 933 934 for (auto &Sample : getProfiles()) 935 if (auto Key = Remappings.insert(Sample.first())) 936 SampleMap.insert({Key, &Sample.second}); 937 938 return sampleprof_error::success; 939 } 940 941 FunctionSamples * 942 SampleProfileReaderItaniumRemapper::getSamplesFor(StringRef Fname) { 943 if (auto Key = Remappings.lookup(Fname)) 944 return SampleMap.lookup(Key); 945 return SampleProfileReader::getSamplesFor(Fname); 946 } 947 948 /// Prepare a memory buffer for the contents of \p Filename. 949 /// 950 /// \returns an error code indicating the status of the buffer. 951 static ErrorOr<std::unique_ptr<MemoryBuffer>> 952 setupMemoryBuffer(const Twine &Filename) { 953 auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename); 954 if (std::error_code EC = BufferOrErr.getError()) 955 return EC; 956 auto Buffer = std::move(BufferOrErr.get()); 957 958 // Sanity check the file. 959 if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<uint32_t>::max()) 960 return sampleprof_error::too_large; 961 962 return std::move(Buffer); 963 } 964 965 /// Create a sample profile reader based on the format of the input file. 966 /// 967 /// \param Filename The file to open. 968 /// 969 /// \param C The LLVM context to use to emit diagnostics. 970 /// 971 /// \returns an error code indicating the status of the created reader. 972 ErrorOr<std::unique_ptr<SampleProfileReader>> 973 SampleProfileReader::create(const Twine &Filename, LLVMContext &C) { 974 auto BufferOrError = setupMemoryBuffer(Filename); 975 if (std::error_code EC = BufferOrError.getError()) 976 return EC; 977 return create(BufferOrError.get(), C); 978 } 979 980 /// Create a sample profile remapper from the given input, to remap the 981 /// function names in the given profile data. 982 /// 983 /// \param Filename The file to open. 984 /// 985 /// \param C The LLVM context to use to emit diagnostics. 986 /// 987 /// \param Underlying The underlying profile data reader to remap. 988 /// 989 /// \returns an error code indicating the status of the created reader. 990 ErrorOr<std::unique_ptr<SampleProfileReader>> 991 SampleProfileReaderItaniumRemapper::create( 992 const Twine &Filename, LLVMContext &C, 993 std::unique_ptr<SampleProfileReader> Underlying) { 994 auto BufferOrError = setupMemoryBuffer(Filename); 995 if (std::error_code EC = BufferOrError.getError()) 996 return EC; 997 return llvm::make_unique<SampleProfileReaderItaniumRemapper>( 998 std::move(BufferOrError.get()), C, std::move(Underlying)); 999 } 1000 1001 /// Create a sample profile reader based on the format of the input data. 1002 /// 1003 /// \param B The memory buffer to create the reader from (assumes ownership). 1004 /// 1005 /// \param C The LLVM context to use to emit diagnostics. 1006 /// 1007 /// \returns an error code indicating the status of the created reader. 1008 ErrorOr<std::unique_ptr<SampleProfileReader>> 1009 SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C) { 1010 std::unique_ptr<SampleProfileReader> Reader; 1011 if (SampleProfileReaderRawBinary::hasFormat(*B)) 1012 Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C)); 1013 else if (SampleProfileReaderCompactBinary::hasFormat(*B)) 1014 Reader.reset(new SampleProfileReaderCompactBinary(std::move(B), C)); 1015 else if (SampleProfileReaderGCC::hasFormat(*B)) 1016 Reader.reset(new SampleProfileReaderGCC(std::move(B), C)); 1017 else if (SampleProfileReaderText::hasFormat(*B)) 1018 Reader.reset(new SampleProfileReaderText(std::move(B), C)); 1019 else 1020 return sampleprof_error::unrecognized_format; 1021 1022 FunctionSamples::Format = Reader->getFormat(); 1023 if (std::error_code EC = Reader->readHeader()) 1024 return EC; 1025 1026 return std::move(Reader); 1027 } 1028 1029 // For text and GCC file formats, we compute the summary after reading the 1030 // profile. Binary format has the profile summary in its header. 1031 void SampleProfileReader::computeSummary() { 1032 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); 1033 for (const auto &I : Profiles) { 1034 const FunctionSamples &Profile = I.second; 1035 Builder.addRecord(Profile); 1036 } 1037 Summary = Builder.getSummary(); 1038 } 1039