1 //===- InstrProf.cpp - Instrumented profiling format support --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains support for clang's instrumentation based PGO and 10 // coverage. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/ProfileData/InstrProf.h" 15 #include "llvm/ADT/ArrayRef.h" 16 #include "llvm/ADT/SetVector.h" 17 #include "llvm/ADT/SmallString.h" 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/ADT/StringExtras.h" 20 #include "llvm/ADT/StringRef.h" 21 #include "llvm/Config/config.h" 22 #include "llvm/IR/Constant.h" 23 #include "llvm/IR/Constants.h" 24 #include "llvm/IR/Function.h" 25 #include "llvm/IR/GlobalValue.h" 26 #include "llvm/IR/GlobalVariable.h" 27 #include "llvm/IR/Instruction.h" 28 #include "llvm/IR/LLVMContext.h" 29 #include "llvm/IR/MDBuilder.h" 30 #include "llvm/IR/Mangler.h" 31 #include "llvm/IR/Metadata.h" 32 #include "llvm/IR/Module.h" 33 #include "llvm/IR/Type.h" 34 #include "llvm/ProfileData/InstrProfReader.h" 35 #include "llvm/Support/Casting.h" 36 #include "llvm/Support/CommandLine.h" 37 #include "llvm/Support/Compiler.h" 38 #include "llvm/Support/Compression.h" 39 #include "llvm/Support/Endian.h" 40 #include "llvm/Support/Error.h" 41 #include "llvm/Support/ErrorHandling.h" 42 #include "llvm/Support/LEB128.h" 43 #include "llvm/Support/MathExtras.h" 44 #include "llvm/Support/Path.h" 45 #include "llvm/Support/SwapByteOrder.h" 46 #include "llvm/Support/VirtualFileSystem.h" 47 #include "llvm/TargetParser/Triple.h" 48 #include <algorithm> 49 #include <cassert> 50 #include <cstddef> 51 #include <cstdint> 52 #include <cstring> 53 #include <memory> 54 #include <string> 55 #include <system_error> 56 #include <type_traits> 57 #include <utility> 58 #include <vector> 59 60 using namespace llvm; 61 62 static cl::opt<bool> StaticFuncFullModulePrefix( 63 "static-func-full-module-prefix", cl::init(true), cl::Hidden, 64 cl::desc("Use full module build paths in the profile counter names for " 65 "static functions.")); 66 67 // This option is tailored to users that have different top-level directory in 68 // profile-gen and profile-use compilation. Users need to specific the number 69 // of levels to strip. A value larger than the number of directories in the 70 // source file will strip all the directory names and only leave the basename. 71 // 72 // Note current ThinLTO module importing for the indirect-calls assumes 73 // the source directory name not being stripped. A non-zero option value here 74 // can potentially prevent some inter-module indirect-call-promotions. 75 static cl::opt<unsigned> StaticFuncStripDirNamePrefix( 76 "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, 77 cl::desc("Strip specified level of directory name from source path in " 78 "the profile counter name for static functions.")); 79 80 static std::string getInstrProfErrString(instrprof_error Err, 81 const std::string &ErrMsg = "") { 82 std::string Msg; 83 raw_string_ostream OS(Msg); 84 85 switch (Err) { 86 case instrprof_error::success: 87 OS << "success"; 88 break; 89 case instrprof_error::eof: 90 OS << "end of File"; 91 break; 92 case instrprof_error::unrecognized_format: 93 OS << "unrecognized instrumentation profile encoding format"; 94 break; 95 case instrprof_error::bad_magic: 96 OS << "invalid instrumentation profile data (bad magic)"; 97 break; 98 case instrprof_error::bad_header: 99 OS << "invalid instrumentation profile data (file header is corrupt)"; 100 break; 101 case instrprof_error::unsupported_version: 102 OS << "unsupported instrumentation profile format version"; 103 break; 104 case instrprof_error::unsupported_hash_type: 105 OS << "unsupported instrumentation profile hash type"; 106 break; 107 case instrprof_error::too_large: 108 OS << "too much profile data"; 109 break; 110 case instrprof_error::truncated: 111 OS << "truncated profile data"; 112 break; 113 case instrprof_error::malformed: 114 OS << "malformed instrumentation profile data"; 115 break; 116 case instrprof_error::missing_correlation_info: 117 OS << "debug info/binary for correlation is required"; 118 break; 119 case instrprof_error::unexpected_correlation_info: 120 OS << "debug info/binary for correlation is not necessary"; 121 break; 122 case instrprof_error::unable_to_correlate_profile: 123 OS << "unable to correlate profile"; 124 break; 125 case instrprof_error::invalid_prof: 126 OS << "invalid profile created. Please file a bug " 127 "at: " BUG_REPORT_URL 128 " and include the profraw files that caused this error."; 129 break; 130 case instrprof_error::unknown_function: 131 OS << "no profile data available for function"; 132 break; 133 case instrprof_error::hash_mismatch: 134 OS << "function control flow change detected (hash mismatch)"; 135 break; 136 case instrprof_error::count_mismatch: 137 OS << "function basic block count change detected (counter mismatch)"; 138 break; 139 case instrprof_error::bitmap_mismatch: 140 OS << "function bitmap size change detected (bitmap size mismatch)"; 141 break; 142 case instrprof_error::counter_overflow: 143 OS << "counter overflow"; 144 break; 145 case instrprof_error::value_site_count_mismatch: 146 OS << "function value site count change detected (counter mismatch)"; 147 break; 148 case instrprof_error::compress_failed: 149 OS << "failed to compress data (zlib)"; 150 break; 151 case instrprof_error::uncompress_failed: 152 OS << "failed to uncompress data (zlib)"; 153 break; 154 case instrprof_error::empty_raw_profile: 155 OS << "empty raw profile file"; 156 break; 157 case instrprof_error::zlib_unavailable: 158 OS << "profile uses zlib compression but the profile reader was built " 159 "without zlib support"; 160 break; 161 case instrprof_error::raw_profile_version_mismatch: 162 OS << "raw profile version mismatch"; 163 break; 164 case instrprof_error::counter_value_too_large: 165 OS << "excessively large counter value suggests corrupted profile data"; 166 break; 167 } 168 169 // If optional error message is not empty, append it to the message. 170 if (!ErrMsg.empty()) 171 OS << ": " << ErrMsg; 172 173 return OS.str(); 174 } 175 176 namespace { 177 178 // FIXME: This class is only here to support the transition to llvm::Error. It 179 // will be removed once this transition is complete. Clients should prefer to 180 // deal with the Error value directly, rather than converting to error_code. 181 class InstrProfErrorCategoryType : public std::error_category { 182 const char *name() const noexcept override { return "llvm.instrprof"; } 183 184 std::string message(int IE) const override { 185 return getInstrProfErrString(static_cast<instrprof_error>(IE)); 186 } 187 }; 188 189 } // end anonymous namespace 190 191 const std::error_category &llvm::instrprof_category() { 192 static InstrProfErrorCategoryType ErrorCategory; 193 return ErrorCategory; 194 } 195 196 namespace { 197 198 const char *InstrProfSectNameCommon[] = { 199 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ 200 SectNameCommon, 201 #include "llvm/ProfileData/InstrProfData.inc" 202 }; 203 204 const char *InstrProfSectNameCoff[] = { 205 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ 206 SectNameCoff, 207 #include "llvm/ProfileData/InstrProfData.inc" 208 }; 209 210 const char *InstrProfSectNamePrefix[] = { 211 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ 212 Prefix, 213 #include "llvm/ProfileData/InstrProfData.inc" 214 }; 215 216 } // namespace 217 218 namespace llvm { 219 220 cl::opt<bool> DoInstrProfNameCompression( 221 "enable-name-compression", 222 cl::desc("Enable name/filename string compression"), cl::init(true)); 223 224 std::string getInstrProfSectionName(InstrProfSectKind IPSK, 225 Triple::ObjectFormatType OF, 226 bool AddSegmentInfo) { 227 std::string SectName; 228 229 if (OF == Triple::MachO && AddSegmentInfo) 230 SectName = InstrProfSectNamePrefix[IPSK]; 231 232 if (OF == Triple::COFF) 233 SectName += InstrProfSectNameCoff[IPSK]; 234 else 235 SectName += InstrProfSectNameCommon[IPSK]; 236 237 if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo) 238 SectName += ",regular,live_support"; 239 240 return SectName; 241 } 242 243 std::string InstrProfError::message() const { 244 return getInstrProfErrString(Err, Msg); 245 } 246 247 char InstrProfError::ID = 0; 248 249 std::string getPGOFuncName(StringRef RawFuncName, 250 GlobalValue::LinkageTypes Linkage, 251 StringRef FileName, 252 uint64_t Version LLVM_ATTRIBUTE_UNUSED) { 253 return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName); 254 } 255 256 // Strip NumPrefix level of directory name from PathNameStr. If the number of 257 // directory separators is less than NumPrefix, strip all the directories and 258 // leave base file name only. 259 static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) { 260 uint32_t Count = NumPrefix; 261 uint32_t Pos = 0, LastPos = 0; 262 for (auto & CI : PathNameStr) { 263 ++Pos; 264 if (llvm::sys::path::is_separator(CI)) { 265 LastPos = Pos; 266 --Count; 267 } 268 if (Count == 0) 269 break; 270 } 271 return PathNameStr.substr(LastPos); 272 } 273 274 static StringRef getStrippedSourceFileName(const GlobalObject &GO) { 275 StringRef FileName(GO.getParent()->getSourceFileName()); 276 uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1; 277 if (StripLevel < StaticFuncStripDirNamePrefix) 278 StripLevel = StaticFuncStripDirNamePrefix; 279 if (StripLevel) 280 FileName = stripDirPrefix(FileName, StripLevel); 281 return FileName; 282 } 283 284 // The PGO name has the format [<filepath>;]<linkage-name> where <filepath>; is 285 // provided if linkage is local and <linkage-name> is the mangled function 286 // name. The filepath is used to discriminate possibly identical function names. 287 // ; is used because it is unlikely to be found in either <filepath> or 288 // <linkage-name>. 289 // 290 // Older compilers used getPGOFuncName() which has the format 291 // [<filepath>:]<function-name>. <filepath> is used to discriminate between 292 // possibly identical function names when linkage is local and <function-name> 293 // simply comes from F.getName(). This caused trouble for Objective-C functions 294 // which commonly have :'s in their names. Also, since <function-name> is not 295 // mangled, they cannot be passed to Mach-O linkers via -order_file. We still 296 // need to compute this name to lookup functions from profiles built by older 297 // compilers. 298 static std::string 299 getIRPGONameForGlobalObject(const GlobalObject &GO, 300 GlobalValue::LinkageTypes Linkage, 301 StringRef FileName) { 302 SmallString<64> Name; 303 if (llvm::GlobalValue::isLocalLinkage(Linkage)) { 304 Name.append(FileName.empty() ? "<unknown>" : FileName); 305 Name.append(";"); 306 } 307 Mangler().getNameWithPrefix(Name, &GO, /*CannotUsePrivateLabel=*/true); 308 return Name.str().str(); 309 } 310 311 static std::optional<std::string> lookupPGONameFromMetadata(MDNode *MD) { 312 if (MD != nullptr) { 313 StringRef S = cast<MDString>(MD->getOperand(0))->getString(); 314 return S.str(); 315 } 316 return {}; 317 } 318 319 // Returns the PGO object name. This function has some special handling 320 // when called in LTO optimization. The following only applies when calling in 321 // LTO passes (when \c InLTO is true): LTO's internalization privatizes many 322 // global linkage symbols. This happens after value profile annotation, but 323 // those internal linkage functions should not have a source prefix. 324 // Additionally, for ThinLTO mode, exported internal functions are promoted 325 // and renamed. We need to ensure that the original internal PGO name is 326 // used when computing the GUID that is compared against the profiled GUIDs. 327 // To differentiate compiler generated internal symbols from original ones, 328 // PGOFuncName meta data are created and attached to the original internal 329 // symbols in the value profile annotation step 330 // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta 331 // data, its original linkage must be non-internal. 332 static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, 333 MDNode *PGONameMetadata) { 334 if (!InLTO) { 335 auto FileName = getStrippedSourceFileName(GO); 336 return getIRPGONameForGlobalObject(GO, GO.getLinkage(), FileName); 337 } 338 339 // In LTO mode (when InLTO is true), first check if there is a meta data. 340 if (auto IRPGOFuncName = lookupPGONameFromMetadata(PGONameMetadata)) 341 return *IRPGOFuncName; 342 343 // If there is no meta data, the function must be a global before the value 344 // profile annotation pass. Its current linkage may be internal if it is 345 // internalized in LTO mode. 346 return getIRPGONameForGlobalObject(GO, GlobalValue::ExternalLinkage, ""); 347 } 348 349 // Returns the IRPGO function name and does special handling when called 350 // in LTO optimization. See the comments of `getIRPGOObjectName` for details. 351 std::string getIRPGOFuncName(const Function &F, bool InLTO) { 352 return getIRPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F)); 353 } 354 355 // This is similar to `getIRPGOFuncName` except that this function calls 356 // 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls 357 // 'getIRPGONameForGlobalObject'. See the difference between two callees in the 358 // comments of `getIRPGONameForGlobalObject`. 359 std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { 360 if (!InLTO) { 361 auto FileName = getStrippedSourceFileName(F); 362 return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version); 363 } 364 365 // In LTO mode (when InLTO is true), first check if there is a meta data. 366 if (auto PGOFuncName = lookupPGONameFromMetadata(getPGOFuncNameMetadata(F))) 367 return *PGOFuncName; 368 369 // If there is no meta data, the function must be a global before the value 370 // profile annotation pass. Its current linkage may be internal if it is 371 // internalized in LTO mode. 372 return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); 373 } 374 375 // See getIRPGOFuncName() for a discription of the format. 376 std::pair<StringRef, StringRef> 377 getParsedIRPGOFuncName(StringRef IRPGOFuncName) { 378 auto [FileName, FuncName] = IRPGOFuncName.split(';'); 379 if (FuncName.empty()) 380 return std::make_pair(StringRef(), IRPGOFuncName); 381 return std::make_pair(FileName, FuncName); 382 } 383 384 StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { 385 if (FileName.empty()) 386 return PGOFuncName; 387 // Drop the file name including ':'. See also getPGOFuncName. 388 if (PGOFuncName.starts_with(FileName)) 389 PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1); 390 return PGOFuncName; 391 } 392 393 // \p FuncName is the string used as profile lookup key for the function. A 394 // symbol is created to hold the name. Return the legalized symbol name. 395 std::string getPGOFuncNameVarName(StringRef FuncName, 396 GlobalValue::LinkageTypes Linkage) { 397 std::string VarName = std::string(getInstrProfNameVarPrefix()); 398 VarName += FuncName; 399 400 if (!GlobalValue::isLocalLinkage(Linkage)) 401 return VarName; 402 403 // Now fix up illegal chars in local VarName that may upset the assembler. 404 const char InvalidChars[] = "-:;<>/\"'"; 405 size_t found = VarName.find_first_of(InvalidChars); 406 while (found != std::string::npos) { 407 VarName[found] = '_'; 408 found = VarName.find_first_of(InvalidChars, found + 1); 409 } 410 return VarName; 411 } 412 413 GlobalVariable *createPGOFuncNameVar(Module &M, 414 GlobalValue::LinkageTypes Linkage, 415 StringRef PGOFuncName) { 416 // We generally want to match the function's linkage, but available_externally 417 // and extern_weak both have the wrong semantics, and anything that doesn't 418 // need to link across compilation units doesn't need to be visible at all. 419 if (Linkage == GlobalValue::ExternalWeakLinkage) 420 Linkage = GlobalValue::LinkOnceAnyLinkage; 421 else if (Linkage == GlobalValue::AvailableExternallyLinkage) 422 Linkage = GlobalValue::LinkOnceODRLinkage; 423 else if (Linkage == GlobalValue::InternalLinkage || 424 Linkage == GlobalValue::ExternalLinkage) 425 Linkage = GlobalValue::PrivateLinkage; 426 427 auto *Value = 428 ConstantDataArray::getString(M.getContext(), PGOFuncName, false); 429 auto FuncNameVar = 430 new GlobalVariable(M, Value->getType(), true, Linkage, Value, 431 getPGOFuncNameVarName(PGOFuncName, Linkage)); 432 433 // Hide the symbol so that we correctly get a copy for each executable. 434 if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage())) 435 FuncNameVar->setVisibility(GlobalValue::HiddenVisibility); 436 437 return FuncNameVar; 438 } 439 440 GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) { 441 return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName); 442 } 443 444 Error InstrProfSymtab::create(Module &M, bool InLTO) { 445 for (Function &F : M) { 446 // Function may not have a name: like using asm("") to overwrite the name. 447 // Ignore in this case. 448 if (!F.hasName()) 449 continue; 450 if (Error E = addFuncWithName(F, getIRPGOFuncName(F, InLTO))) 451 return E; 452 // Also use getPGOFuncName() so that we can find records from older profiles 453 if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO))) 454 return E; 455 } 456 Sorted = false; 457 finalizeSymtab(); 458 return Error::success(); 459 } 460 461 /// \c NameStrings is a string composed of one of more possibly encoded 462 /// sub-strings. The substrings are separated by 0 or more zero bytes. This 463 /// method decodes the string and calls `NameCallback` for each substring. 464 static Error 465 readAndDecodeStrings(StringRef NameStrings, 466 std::function<Error(StringRef)> NameCallback) { 467 const uint8_t *P = NameStrings.bytes_begin(); 468 const uint8_t *EndP = NameStrings.bytes_end(); 469 while (P < EndP) { 470 uint32_t N; 471 uint64_t UncompressedSize = decodeULEB128(P, &N); 472 P += N; 473 uint64_t CompressedSize = decodeULEB128(P, &N); 474 P += N; 475 bool isCompressed = (CompressedSize != 0); 476 SmallVector<uint8_t, 128> UncompressedNameStrings; 477 StringRef NameStrings; 478 if (isCompressed) { 479 if (!llvm::compression::zlib::isAvailable()) 480 return make_error<InstrProfError>(instrprof_error::zlib_unavailable); 481 482 if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize), 483 UncompressedNameStrings, 484 UncompressedSize)) { 485 consumeError(std::move(E)); 486 return make_error<InstrProfError>(instrprof_error::uncompress_failed); 487 } 488 P += CompressedSize; 489 NameStrings = toStringRef(UncompressedNameStrings); 490 } else { 491 NameStrings = 492 StringRef(reinterpret_cast<const char *>(P), UncompressedSize); 493 P += UncompressedSize; 494 } 495 // Now parse the name strings. 496 SmallVector<StringRef, 0> Names; 497 NameStrings.split(Names, getInstrProfNameSeparator()); 498 for (StringRef &Name : Names) 499 if (Error E = NameCallback(Name)) 500 return E; 501 502 while (P < EndP && *P == 0) 503 P++; 504 } 505 return Error::success(); 506 } 507 508 Error InstrProfSymtab::create(StringRef NameStrings) { 509 return readAndDecodeStrings( 510 NameStrings, 511 std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1)); 512 } 513 514 Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) { 515 if (Error E = addFuncName(PGOFuncName)) 516 return E; 517 MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); 518 // In ThinLTO, local function may have been promoted to global and have 519 // suffix ".llvm." added to the function name. We need to add the 520 // stripped function name to the symbol table so that we can find a match 521 // from profile. 522 // 523 // We may have other suffixes similar as ".llvm." which are needed to 524 // be stripped before the matching, but ".__uniq." suffix which is used 525 // to differentiate internal linkage functions in different modules 526 // should be kept. Now this is the only suffix with the pattern ".xxx" 527 // which is kept before matching. 528 const std::string UniqSuffix = ".__uniq."; 529 auto pos = PGOFuncName.find(UniqSuffix); 530 // Search '.' after ".__uniq." if ".__uniq." exists, otherwise 531 // search '.' from the beginning. 532 if (pos != std::string::npos) 533 pos += UniqSuffix.length(); 534 else 535 pos = 0; 536 pos = PGOFuncName.find('.', pos); 537 if (pos != std::string::npos && pos != 0) { 538 StringRef OtherFuncName = PGOFuncName.substr(0, pos); 539 if (Error E = addFuncName(OtherFuncName)) 540 return E; 541 MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F); 542 } 543 return Error::success(); 544 } 545 546 uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) { 547 finalizeSymtab(); 548 auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) { 549 return A.first < Address; 550 }); 551 // Raw function pointer collected by value profiler may be from 552 // external functions that are not instrumented. They won't have 553 // mapping data to be used by the deserializer. Force the value to 554 // be 0 in this case. 555 if (It != AddrToMD5Map.end() && It->first == Address) 556 return (uint64_t)It->second; 557 return 0; 558 } 559 560 void InstrProfSymtab::dumpNames(raw_ostream &OS) const { 561 SmallVector<StringRef, 0> Sorted(NameTab.keys()); 562 llvm::sort(Sorted); 563 for (StringRef S : Sorted) 564 OS << S << '\n'; 565 } 566 567 Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs, 568 bool doCompression, std::string &Result) { 569 assert(!NameStrs.empty() && "No name data to emit"); 570 571 uint8_t Header[20], *P = Header; 572 std::string UncompressedNameStrings = 573 join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator()); 574 575 assert(StringRef(UncompressedNameStrings) 576 .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && 577 "PGO name is invalid (contains separator token)"); 578 579 unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P); 580 P += EncLen; 581 582 auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) { 583 EncLen = encodeULEB128(CompressedLen, P); 584 P += EncLen; 585 char *HeaderStr = reinterpret_cast<char *>(&Header[0]); 586 unsigned HeaderLen = P - &Header[0]; 587 Result.append(HeaderStr, HeaderLen); 588 Result += InputStr; 589 return Error::success(); 590 }; 591 592 if (!doCompression) { 593 return WriteStringToResult(0, UncompressedNameStrings); 594 } 595 596 SmallVector<uint8_t, 128> CompressedNameStrings; 597 compression::zlib::compress(arrayRefFromStringRef(UncompressedNameStrings), 598 CompressedNameStrings, 599 compression::zlib::BestSizeCompression); 600 601 return WriteStringToResult(CompressedNameStrings.size(), 602 toStringRef(CompressedNameStrings)); 603 } 604 605 StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) { 606 auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer()); 607 StringRef NameStr = 608 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString(); 609 return NameStr; 610 } 611 612 Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars, 613 std::string &Result, bool doCompression) { 614 std::vector<std::string> NameStrs; 615 for (auto *NameVar : NameVars) { 616 NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar))); 617 } 618 return collectGlobalObjectNameStrings( 619 NameStrs, compression::zlib::isAvailable() && doCompression, Result); 620 } 621 622 void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const { 623 uint64_t FuncSum = 0; 624 Sum.NumEntries += Counts.size(); 625 for (uint64_t Count : Counts) 626 FuncSum += Count; 627 Sum.CountSum += FuncSum; 628 629 for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) { 630 uint64_t KindSum = 0; 631 uint32_t NumValueSites = getNumValueSites(VK); 632 for (size_t I = 0; I < NumValueSites; ++I) { 633 uint32_t NV = getNumValueDataForSite(VK, I); 634 std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I); 635 for (uint32_t V = 0; V < NV; V++) 636 KindSum += VD[V].Count; 637 } 638 Sum.ValueCounts[VK] += KindSum; 639 } 640 } 641 642 void InstrProfValueSiteRecord::overlap(InstrProfValueSiteRecord &Input, 643 uint32_t ValueKind, 644 OverlapStats &Overlap, 645 OverlapStats &FuncLevelOverlap) { 646 this->sortByTargetValues(); 647 Input.sortByTargetValues(); 648 double Score = 0.0f, FuncLevelScore = 0.0f; 649 auto I = ValueData.begin(); 650 auto IE = ValueData.end(); 651 auto J = Input.ValueData.begin(); 652 auto JE = Input.ValueData.end(); 653 while (I != IE && J != JE) { 654 if (I->Value == J->Value) { 655 Score += OverlapStats::score(I->Count, J->Count, 656 Overlap.Base.ValueCounts[ValueKind], 657 Overlap.Test.ValueCounts[ValueKind]); 658 FuncLevelScore += OverlapStats::score( 659 I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind], 660 FuncLevelOverlap.Test.ValueCounts[ValueKind]); 661 ++I; 662 } else if (I->Value < J->Value) { 663 ++I; 664 continue; 665 } 666 ++J; 667 } 668 Overlap.Overlap.ValueCounts[ValueKind] += Score; 669 FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore; 670 } 671 672 // Return false on mismatch. 673 void InstrProfRecord::overlapValueProfData(uint32_t ValueKind, 674 InstrProfRecord &Other, 675 OverlapStats &Overlap, 676 OverlapStats &FuncLevelOverlap) { 677 uint32_t ThisNumValueSites = getNumValueSites(ValueKind); 678 assert(ThisNumValueSites == Other.getNumValueSites(ValueKind)); 679 if (!ThisNumValueSites) 680 return; 681 682 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = 683 getOrCreateValueSitesForKind(ValueKind); 684 MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords = 685 Other.getValueSitesForKind(ValueKind); 686 for (uint32_t I = 0; I < ThisNumValueSites; I++) 687 ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap, 688 FuncLevelOverlap); 689 } 690 691 void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap, 692 OverlapStats &FuncLevelOverlap, 693 uint64_t ValueCutoff) { 694 // FuncLevel CountSum for other should already computed and nonzero. 695 assert(FuncLevelOverlap.Test.CountSum >= 1.0f); 696 accumulateCounts(FuncLevelOverlap.Base); 697 bool Mismatch = (Counts.size() != Other.Counts.size()); 698 699 // Check if the value profiles mismatch. 700 if (!Mismatch) { 701 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { 702 uint32_t ThisNumValueSites = getNumValueSites(Kind); 703 uint32_t OtherNumValueSites = Other.getNumValueSites(Kind); 704 if (ThisNumValueSites != OtherNumValueSites) { 705 Mismatch = true; 706 break; 707 } 708 } 709 } 710 if (Mismatch) { 711 Overlap.addOneMismatch(FuncLevelOverlap.Test); 712 return; 713 } 714 715 // Compute overlap for value counts. 716 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 717 overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap); 718 719 double Score = 0.0; 720 uint64_t MaxCount = 0; 721 // Compute overlap for edge counts. 722 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) { 723 Score += OverlapStats::score(Counts[I], Other.Counts[I], 724 Overlap.Base.CountSum, Overlap.Test.CountSum); 725 MaxCount = std::max(Other.Counts[I], MaxCount); 726 } 727 Overlap.Overlap.CountSum += Score; 728 Overlap.Overlap.NumEntries += 1; 729 730 if (MaxCount >= ValueCutoff) { 731 double FuncScore = 0.0; 732 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) 733 FuncScore += OverlapStats::score(Counts[I], Other.Counts[I], 734 FuncLevelOverlap.Base.CountSum, 735 FuncLevelOverlap.Test.CountSum); 736 FuncLevelOverlap.Overlap.CountSum = FuncScore; 737 FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size(); 738 FuncLevelOverlap.Valid = true; 739 } 740 } 741 742 void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, 743 uint64_t Weight, 744 function_ref<void(instrprof_error)> Warn) { 745 this->sortByTargetValues(); 746 Input.sortByTargetValues(); 747 auto I = ValueData.begin(); 748 auto IE = ValueData.end(); 749 for (const InstrProfValueData &J : Input.ValueData) { 750 while (I != IE && I->Value < J.Value) 751 ++I; 752 if (I != IE && I->Value == J.Value) { 753 bool Overflowed; 754 I->Count = SaturatingMultiplyAdd(J.Count, Weight, I->Count, &Overflowed); 755 if (Overflowed) 756 Warn(instrprof_error::counter_overflow); 757 ++I; 758 continue; 759 } 760 ValueData.insert(I, J); 761 } 762 } 763 764 void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D, 765 function_ref<void(instrprof_error)> Warn) { 766 for (InstrProfValueData &I : ValueData) { 767 bool Overflowed; 768 I.Count = SaturatingMultiply(I.Count, N, &Overflowed) / D; 769 if (Overflowed) 770 Warn(instrprof_error::counter_overflow); 771 } 772 } 773 774 // Merge Value Profile data from Src record to this record for ValueKind. 775 // Scale merged value counts by \p Weight. 776 void InstrProfRecord::mergeValueProfData( 777 uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight, 778 function_ref<void(instrprof_error)> Warn) { 779 uint32_t ThisNumValueSites = getNumValueSites(ValueKind); 780 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind); 781 if (ThisNumValueSites != OtherNumValueSites) { 782 Warn(instrprof_error::value_site_count_mismatch); 783 return; 784 } 785 if (!ThisNumValueSites) 786 return; 787 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = 788 getOrCreateValueSitesForKind(ValueKind); 789 MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords = 790 Src.getValueSitesForKind(ValueKind); 791 for (uint32_t I = 0; I < ThisNumValueSites; I++) 792 ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn); 793 } 794 795 void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight, 796 function_ref<void(instrprof_error)> Warn) { 797 // If the number of counters doesn't match we either have bad data 798 // or a hash collision. 799 if (Counts.size() != Other.Counts.size()) { 800 Warn(instrprof_error::count_mismatch); 801 return; 802 } 803 804 // Special handling of the first count as the PseudoCount. 805 CountPseudoKind OtherKind = Other.getCountPseudoKind(); 806 CountPseudoKind ThisKind = getCountPseudoKind(); 807 if (OtherKind != NotPseudo || ThisKind != NotPseudo) { 808 // We don't allow the merge of a profile with pseudo counts and 809 // a normal profile (i.e. without pesudo counts). 810 // Profile supplimenation should be done after the profile merge. 811 if (OtherKind == NotPseudo || ThisKind == NotPseudo) { 812 Warn(instrprof_error::count_mismatch); 813 return; 814 } 815 if (OtherKind == PseudoHot || ThisKind == PseudoHot) 816 setPseudoCount(PseudoHot); 817 else 818 setPseudoCount(PseudoWarm); 819 return; 820 } 821 822 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) { 823 bool Overflowed; 824 uint64_t Value = 825 SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed); 826 if (Value > getInstrMaxCountValue()) { 827 Value = getInstrMaxCountValue(); 828 Overflowed = true; 829 } 830 Counts[I] = Value; 831 if (Overflowed) 832 Warn(instrprof_error::counter_overflow); 833 } 834 835 // If the number of bitmap bytes doesn't match we either have bad data 836 // or a hash collision. 837 if (BitmapBytes.size() != Other.BitmapBytes.size()) { 838 Warn(instrprof_error::bitmap_mismatch); 839 return; 840 } 841 842 // Bitmap bytes are merged by simply ORing them together. 843 for (size_t I = 0, E = Other.BitmapBytes.size(); I < E; ++I) { 844 BitmapBytes[I] = Other.BitmapBytes[I] | BitmapBytes[I]; 845 } 846 847 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 848 mergeValueProfData(Kind, Other, Weight, Warn); 849 } 850 851 void InstrProfRecord::scaleValueProfData( 852 uint32_t ValueKind, uint64_t N, uint64_t D, 853 function_ref<void(instrprof_error)> Warn) { 854 for (auto &R : getValueSitesForKind(ValueKind)) 855 R.scale(N, D, Warn); 856 } 857 858 void InstrProfRecord::scale(uint64_t N, uint64_t D, 859 function_ref<void(instrprof_error)> Warn) { 860 assert(D != 0 && "D cannot be 0"); 861 for (auto &Count : this->Counts) { 862 bool Overflowed; 863 Count = SaturatingMultiply(Count, N, &Overflowed) / D; 864 if (Count > getInstrMaxCountValue()) { 865 Count = getInstrMaxCountValue(); 866 Overflowed = true; 867 } 868 if (Overflowed) 869 Warn(instrprof_error::counter_overflow); 870 } 871 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 872 scaleValueProfData(Kind, N, D, Warn); 873 } 874 875 // Map indirect call target name hash to name string. 876 uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, 877 InstrProfSymtab *SymTab) { 878 if (!SymTab) 879 return Value; 880 881 if (ValueKind == IPVK_IndirectCallTarget) 882 return SymTab->getFunctionHashFromAddress(Value); 883 884 return Value; 885 } 886 887 void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site, 888 InstrProfValueData *VData, uint32_t N, 889 InstrProfSymtab *ValueMap) { 890 for (uint32_t I = 0; I < N; I++) { 891 VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap); 892 } 893 std::vector<InstrProfValueSiteRecord> &ValueSites = 894 getOrCreateValueSitesForKind(ValueKind); 895 if (N == 0) 896 ValueSites.emplace_back(); 897 else 898 ValueSites.emplace_back(VData, VData + N); 899 } 900 901 std::vector<BPFunctionNode> TemporalProfTraceTy::createBPFunctionNodes( 902 ArrayRef<TemporalProfTraceTy> Traces) { 903 using IDT = BPFunctionNode::IDT; 904 using UtilityNodeT = BPFunctionNode::UtilityNodeT; 905 // Collect all function IDs ordered by their smallest timestamp. This will be 906 // used as the initial FunctionNode order. 907 SetVector<IDT> FunctionIds; 908 size_t LargestTraceSize = 0; 909 for (auto &Trace : Traces) 910 LargestTraceSize = 911 std::max(LargestTraceSize, Trace.FunctionNameRefs.size()); 912 for (size_t Timestamp = 0; Timestamp < LargestTraceSize; Timestamp++) 913 for (auto &Trace : Traces) 914 if (Timestamp < Trace.FunctionNameRefs.size()) 915 FunctionIds.insert(Trace.FunctionNameRefs[Timestamp]); 916 917 int N = std::ceil(std::log2(LargestTraceSize)); 918 919 // TODO: We need to use the Trace.Weight field to give more weight to more 920 // important utilities 921 DenseMap<IDT, SmallVector<UtilityNodeT, 4>> FuncGroups; 922 for (size_t TraceIdx = 0; TraceIdx < Traces.size(); TraceIdx++) { 923 auto &Trace = Traces[TraceIdx].FunctionNameRefs; 924 for (size_t Timestamp = 0; Timestamp < Trace.size(); Timestamp++) { 925 for (int I = std::floor(std::log2(Timestamp + 1)); I < N; I++) { 926 auto &FunctionId = Trace[Timestamp]; 927 UtilityNodeT GroupId = TraceIdx * N + I; 928 FuncGroups[FunctionId].push_back(GroupId); 929 } 930 } 931 } 932 933 std::vector<BPFunctionNode> Nodes; 934 for (auto &Id : FunctionIds) { 935 auto &UNs = FuncGroups[Id]; 936 llvm::sort(UNs); 937 UNs.erase(std::unique(UNs.begin(), UNs.end()), UNs.end()); 938 Nodes.emplace_back(Id, UNs); 939 } 940 return Nodes; 941 } 942 943 #define INSTR_PROF_COMMON_API_IMPL 944 #include "llvm/ProfileData/InstrProfData.inc" 945 946 /*! 947 * ValueProfRecordClosure Interface implementation for InstrProfRecord 948 * class. These C wrappers are used as adaptors so that C++ code can be 949 * invoked as callbacks. 950 */ 951 uint32_t getNumValueKindsInstrProf(const void *Record) { 952 return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds(); 953 } 954 955 uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) { 956 return reinterpret_cast<const InstrProfRecord *>(Record) 957 ->getNumValueSites(VKind); 958 } 959 960 uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) { 961 return reinterpret_cast<const InstrProfRecord *>(Record) 962 ->getNumValueData(VKind); 963 } 964 965 uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, 966 uint32_t S) { 967 return reinterpret_cast<const InstrProfRecord *>(R) 968 ->getNumValueDataForSite(VK, S); 969 } 970 971 void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, 972 uint32_t K, uint32_t S) { 973 reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S); 974 } 975 976 ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { 977 ValueProfData *VD = 978 (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData()); 979 memset(VD, 0, TotalSizeInBytes); 980 return VD; 981 } 982 983 static ValueProfRecordClosure InstrProfRecordClosure = { 984 nullptr, 985 getNumValueKindsInstrProf, 986 getNumValueSitesInstrProf, 987 getNumValueDataInstrProf, 988 getNumValueDataForSiteInstrProf, 989 nullptr, 990 getValueForSiteInstrProf, 991 allocValueProfDataInstrProf}; 992 993 // Wrapper implementation using the closure mechanism. 994 uint32_t ValueProfData::getSize(const InstrProfRecord &Record) { 995 auto Closure = InstrProfRecordClosure; 996 Closure.Record = &Record; 997 return getValueProfDataSize(&Closure); 998 } 999 1000 // Wrapper implementation using the closure mechanism. 1001 std::unique_ptr<ValueProfData> 1002 ValueProfData::serializeFrom(const InstrProfRecord &Record) { 1003 InstrProfRecordClosure.Record = &Record; 1004 1005 std::unique_ptr<ValueProfData> VPD( 1006 serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr)); 1007 return VPD; 1008 } 1009 1010 void ValueProfRecord::deserializeTo(InstrProfRecord &Record, 1011 InstrProfSymtab *SymTab) { 1012 Record.reserveSites(Kind, NumValueSites); 1013 1014 InstrProfValueData *ValueData = getValueProfRecordValueData(this); 1015 for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) { 1016 uint8_t ValueDataCount = this->SiteCountArray[VSite]; 1017 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab); 1018 ValueData += ValueDataCount; 1019 } 1020 } 1021 1022 // For writing/serializing, Old is the host endianness, and New is 1023 // byte order intended on disk. For Reading/deserialization, Old 1024 // is the on-disk source endianness, and New is the host endianness. 1025 void ValueProfRecord::swapBytes(llvm::endianness Old, llvm::endianness New) { 1026 using namespace support; 1027 1028 if (Old == New) 1029 return; 1030 1031 if (llvm::endianness::native != Old) { 1032 sys::swapByteOrder<uint32_t>(NumValueSites); 1033 sys::swapByteOrder<uint32_t>(Kind); 1034 } 1035 uint32_t ND = getValueProfRecordNumValueData(this); 1036 InstrProfValueData *VD = getValueProfRecordValueData(this); 1037 1038 // No need to swap byte array: SiteCountArrray. 1039 for (uint32_t I = 0; I < ND; I++) { 1040 sys::swapByteOrder<uint64_t>(VD[I].Value); 1041 sys::swapByteOrder<uint64_t>(VD[I].Count); 1042 } 1043 if (llvm::endianness::native == Old) { 1044 sys::swapByteOrder<uint32_t>(NumValueSites); 1045 sys::swapByteOrder<uint32_t>(Kind); 1046 } 1047 } 1048 1049 void ValueProfData::deserializeTo(InstrProfRecord &Record, 1050 InstrProfSymtab *SymTab) { 1051 if (NumValueKinds == 0) 1052 return; 1053 1054 ValueProfRecord *VR = getFirstValueProfRecord(this); 1055 for (uint32_t K = 0; K < NumValueKinds; K++) { 1056 VR->deserializeTo(Record, SymTab); 1057 VR = getValueProfRecordNext(VR); 1058 } 1059 } 1060 1061 template <class T> 1062 static T swapToHostOrder(const unsigned char *&D, llvm::endianness Orig) { 1063 using namespace support; 1064 1065 if (Orig == llvm::endianness::little) 1066 return endian::readNext<T, llvm::endianness::little, unaligned>(D); 1067 else 1068 return endian::readNext<T, llvm::endianness::big, unaligned>(D); 1069 } 1070 1071 static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) { 1072 return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize)) 1073 ValueProfData()); 1074 } 1075 1076 Error ValueProfData::checkIntegrity() { 1077 if (NumValueKinds > IPVK_Last + 1) 1078 return make_error<InstrProfError>( 1079 instrprof_error::malformed, "number of value profile kinds is invalid"); 1080 // Total size needs to be multiple of quadword size. 1081 if (TotalSize % sizeof(uint64_t)) 1082 return make_error<InstrProfError>( 1083 instrprof_error::malformed, "total size is not multiples of quardword"); 1084 1085 ValueProfRecord *VR = getFirstValueProfRecord(this); 1086 for (uint32_t K = 0; K < this->NumValueKinds; K++) { 1087 if (VR->Kind > IPVK_Last) 1088 return make_error<InstrProfError>(instrprof_error::malformed, 1089 "value kind is invalid"); 1090 VR = getValueProfRecordNext(VR); 1091 if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize) 1092 return make_error<InstrProfError>( 1093 instrprof_error::malformed, 1094 "value profile address is greater than total size"); 1095 } 1096 return Error::success(); 1097 } 1098 1099 Expected<std::unique_ptr<ValueProfData>> 1100 ValueProfData::getValueProfData(const unsigned char *D, 1101 const unsigned char *const BufferEnd, 1102 llvm::endianness Endianness) { 1103 using namespace support; 1104 1105 if (D + sizeof(ValueProfData) > BufferEnd) 1106 return make_error<InstrProfError>(instrprof_error::truncated); 1107 1108 const unsigned char *Header = D; 1109 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness); 1110 if (D + TotalSize > BufferEnd) 1111 return make_error<InstrProfError>(instrprof_error::too_large); 1112 1113 std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize); 1114 memcpy(VPD.get(), D, TotalSize); 1115 // Byte swap. 1116 VPD->swapBytesToHost(Endianness); 1117 1118 Error E = VPD->checkIntegrity(); 1119 if (E) 1120 return std::move(E); 1121 1122 return std::move(VPD); 1123 } 1124 1125 void ValueProfData::swapBytesToHost(llvm::endianness Endianness) { 1126 using namespace support; 1127 1128 if (Endianness == llvm::endianness::native) 1129 return; 1130 1131 sys::swapByteOrder<uint32_t>(TotalSize); 1132 sys::swapByteOrder<uint32_t>(NumValueKinds); 1133 1134 ValueProfRecord *VR = getFirstValueProfRecord(this); 1135 for (uint32_t K = 0; K < NumValueKinds; K++) { 1136 VR->swapBytes(Endianness, llvm::endianness::native); 1137 VR = getValueProfRecordNext(VR); 1138 } 1139 } 1140 1141 void ValueProfData::swapBytesFromHost(llvm::endianness Endianness) { 1142 using namespace support; 1143 1144 if (Endianness == llvm::endianness::native) 1145 return; 1146 1147 ValueProfRecord *VR = getFirstValueProfRecord(this); 1148 for (uint32_t K = 0; K < NumValueKinds; K++) { 1149 ValueProfRecord *NVR = getValueProfRecordNext(VR); 1150 VR->swapBytes(llvm::endianness::native, Endianness); 1151 VR = NVR; 1152 } 1153 sys::swapByteOrder<uint32_t>(TotalSize); 1154 sys::swapByteOrder<uint32_t>(NumValueKinds); 1155 } 1156 1157 void annotateValueSite(Module &M, Instruction &Inst, 1158 const InstrProfRecord &InstrProfR, 1159 InstrProfValueKind ValueKind, uint32_t SiteIdx, 1160 uint32_t MaxMDCount) { 1161 uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx); 1162 if (!NV) 1163 return; 1164 1165 uint64_t Sum = 0; 1166 std::unique_ptr<InstrProfValueData[]> VD = 1167 InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum); 1168 1169 ArrayRef<InstrProfValueData> VDs(VD.get(), NV); 1170 annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount); 1171 } 1172 1173 void annotateValueSite(Module &M, Instruction &Inst, 1174 ArrayRef<InstrProfValueData> VDs, 1175 uint64_t Sum, InstrProfValueKind ValueKind, 1176 uint32_t MaxMDCount) { 1177 LLVMContext &Ctx = M.getContext(); 1178 MDBuilder MDHelper(Ctx); 1179 SmallVector<Metadata *, 3> Vals; 1180 // Tag 1181 Vals.push_back(MDHelper.createString("VP")); 1182 // Value Kind 1183 Vals.push_back(MDHelper.createConstant( 1184 ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind))); 1185 // Total Count 1186 Vals.push_back( 1187 MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum))); 1188 1189 // Value Profile Data 1190 uint32_t MDCount = MaxMDCount; 1191 for (auto &VD : VDs) { 1192 Vals.push_back(MDHelper.createConstant( 1193 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value))); 1194 Vals.push_back(MDHelper.createConstant( 1195 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count))); 1196 if (--MDCount == 0) 1197 break; 1198 } 1199 Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); 1200 } 1201 1202 bool getValueProfDataFromInst(const Instruction &Inst, 1203 InstrProfValueKind ValueKind, 1204 uint32_t MaxNumValueData, 1205 InstrProfValueData ValueData[], 1206 uint32_t &ActualNumValueData, uint64_t &TotalC, 1207 bool GetNoICPValue) { 1208 MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof); 1209 if (!MD) 1210 return false; 1211 1212 unsigned NOps = MD->getNumOperands(); 1213 1214 if (NOps < 5) 1215 return false; 1216 1217 // Operand 0 is a string tag "VP": 1218 MDString *Tag = cast<MDString>(MD->getOperand(0)); 1219 if (!Tag) 1220 return false; 1221 1222 if (!Tag->getString().equals("VP")) 1223 return false; 1224 1225 // Now check kind: 1226 ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1)); 1227 if (!KindInt) 1228 return false; 1229 if (KindInt->getZExtValue() != ValueKind) 1230 return false; 1231 1232 // Get total count 1233 ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2)); 1234 if (!TotalCInt) 1235 return false; 1236 TotalC = TotalCInt->getZExtValue(); 1237 1238 ActualNumValueData = 0; 1239 1240 for (unsigned I = 3; I < NOps; I += 2) { 1241 if (ActualNumValueData >= MaxNumValueData) 1242 break; 1243 ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I)); 1244 ConstantInt *Count = 1245 mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1)); 1246 if (!Value || !Count) 1247 return false; 1248 uint64_t CntValue = Count->getZExtValue(); 1249 if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM)) 1250 continue; 1251 ValueData[ActualNumValueData].Value = Value->getZExtValue(); 1252 ValueData[ActualNumValueData].Count = CntValue; 1253 ActualNumValueData++; 1254 } 1255 return true; 1256 } 1257 1258 MDNode *getPGOFuncNameMetadata(const Function &F) { 1259 return F.getMetadata(getPGOFuncNameMetadataName()); 1260 } 1261 1262 void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { 1263 // Only for internal linkage functions. 1264 if (PGOFuncName == F.getName()) 1265 return; 1266 // Don't create duplicated meta-data. 1267 if (getPGOFuncNameMetadata(F)) 1268 return; 1269 LLVMContext &C = F.getContext(); 1270 MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName)); 1271 F.setMetadata(getPGOFuncNameMetadataName(), N); 1272 } 1273 1274 bool needsComdatForCounter(const Function &F, const Module &M) { 1275 if (F.hasComdat()) 1276 return true; 1277 1278 if (!Triple(M.getTargetTriple()).supportsCOMDAT()) 1279 return false; 1280 1281 // See createPGOFuncNameVar for more details. To avoid link errors, profile 1282 // counters for function with available_externally linkage needs to be changed 1283 // to linkonce linkage. On ELF based systems, this leads to weak symbols to be 1284 // created. Without using comdat, duplicate entries won't be removed by the 1285 // linker leading to increased data segement size and raw profile size. Even 1286 // worse, since the referenced counter from profile per-function data object 1287 // will be resolved to the common strong definition, the profile counts for 1288 // available_externally functions will end up being duplicated in raw profile 1289 // data. This can result in distorted profile as the counts of those dups 1290 // will be accumulated by the profile merger. 1291 GlobalValue::LinkageTypes Linkage = F.getLinkage(); 1292 if (Linkage != GlobalValue::ExternalWeakLinkage && 1293 Linkage != GlobalValue::AvailableExternallyLinkage) 1294 return false; 1295 1296 return true; 1297 } 1298 1299 // Check if INSTR_PROF_RAW_VERSION_VAR is defined. 1300 bool isIRPGOFlagSet(const Module *M) { 1301 auto IRInstrVar = 1302 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); 1303 if (!IRInstrVar || IRInstrVar->hasLocalLinkage()) 1304 return false; 1305 1306 // For CSPGO+LTO, this variable might be marked as non-prevailing and we only 1307 // have the decl. 1308 if (IRInstrVar->isDeclaration()) 1309 return true; 1310 1311 // Check if the flag is set. 1312 if (!IRInstrVar->hasInitializer()) 1313 return false; 1314 1315 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer()); 1316 if (!InitVal) 1317 return false; 1318 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0; 1319 } 1320 1321 // Check if we can safely rename this Comdat function. 1322 bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) { 1323 if (F.getName().empty()) 1324 return false; 1325 if (!needsComdatForCounter(F, *(F.getParent()))) 1326 return false; 1327 // Unsafe to rename the address-taken function (which can be used in 1328 // function comparison). 1329 if (CheckAddressTaken && F.hasAddressTaken()) 1330 return false; 1331 // Only safe to do if this function may be discarded if it is not used 1332 // in the compilation unit. 1333 if (!GlobalValue::isDiscardableIfUnused(F.getLinkage())) 1334 return false; 1335 1336 // For AvailableExternallyLinkage functions. 1337 if (!F.hasComdat()) { 1338 assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage); 1339 return true; 1340 } 1341 return true; 1342 } 1343 1344 // Create the variable for the profile file name. 1345 void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) { 1346 if (InstrProfileOutput.empty()) 1347 return; 1348 Constant *ProfileNameConst = 1349 ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true); 1350 GlobalVariable *ProfileNameVar = new GlobalVariable( 1351 M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage, 1352 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)); 1353 ProfileNameVar->setVisibility(GlobalValue::HiddenVisibility); 1354 Triple TT(M.getTargetTriple()); 1355 if (TT.supportsCOMDAT()) { 1356 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage); 1357 ProfileNameVar->setComdat(M.getOrInsertComdat( 1358 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)))); 1359 } 1360 } 1361 1362 Error OverlapStats::accumulateCounts(const std::string &BaseFilename, 1363 const std::string &TestFilename, 1364 bool IsCS) { 1365 auto getProfileSum = [IsCS](const std::string &Filename, 1366 CountSumOrPercent &Sum) -> Error { 1367 // This function is only used from llvm-profdata that doesn't use any kind 1368 // of VFS. Just create a default RealFileSystem to read profiles. 1369 auto FS = vfs::getRealFileSystem(); 1370 auto ReaderOrErr = InstrProfReader::create(Filename, *FS); 1371 if (Error E = ReaderOrErr.takeError()) { 1372 return E; 1373 } 1374 auto Reader = std::move(ReaderOrErr.get()); 1375 Reader->accumulateCounts(Sum, IsCS); 1376 return Error::success(); 1377 }; 1378 auto Ret = getProfileSum(BaseFilename, Base); 1379 if (Ret) 1380 return Ret; 1381 Ret = getProfileSum(TestFilename, Test); 1382 if (Ret) 1383 return Ret; 1384 this->BaseFilename = &BaseFilename; 1385 this->TestFilename = &TestFilename; 1386 Valid = true; 1387 return Error::success(); 1388 } 1389 1390 void OverlapStats::addOneMismatch(const CountSumOrPercent &MismatchFunc) { 1391 Mismatch.NumEntries += 1; 1392 Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum; 1393 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { 1394 if (Test.ValueCounts[I] >= 1.0f) 1395 Mismatch.ValueCounts[I] += 1396 MismatchFunc.ValueCounts[I] / Test.ValueCounts[I]; 1397 } 1398 } 1399 1400 void OverlapStats::addOneUnique(const CountSumOrPercent &UniqueFunc) { 1401 Unique.NumEntries += 1; 1402 Unique.CountSum += UniqueFunc.CountSum / Test.CountSum; 1403 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { 1404 if (Test.ValueCounts[I] >= 1.0f) 1405 Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I]; 1406 } 1407 } 1408 1409 void OverlapStats::dump(raw_fd_ostream &OS) const { 1410 if (!Valid) 1411 return; 1412 1413 const char *EntryName = 1414 (Level == ProgramLevel ? "functions" : "edge counters"); 1415 if (Level == ProgramLevel) { 1416 OS << "Profile overlap infomation for base_profile: " << *BaseFilename 1417 << " and test_profile: " << *TestFilename << "\nProgram level:\n"; 1418 } else { 1419 OS << "Function level:\n" 1420 << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n"; 1421 } 1422 1423 OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n"; 1424 if (Mismatch.NumEntries) 1425 OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries 1426 << "\n"; 1427 if (Unique.NumEntries) 1428 OS << " # of " << EntryName 1429 << " only in test_profile: " << Unique.NumEntries << "\n"; 1430 1431 OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100) 1432 << "\n"; 1433 if (Mismatch.NumEntries) 1434 OS << " Mismatched count percentage (Edge): " 1435 << format("%.3f%%", Mismatch.CountSum * 100) << "\n"; 1436 if (Unique.NumEntries) 1437 OS << " Percentage of Edge profile only in test_profile: " 1438 << format("%.3f%%", Unique.CountSum * 100) << "\n"; 1439 OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum) 1440 << "\n" 1441 << " Edge profile test count sum: " << format("%.0f", Test.CountSum) 1442 << "\n"; 1443 1444 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { 1445 if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f) 1446 continue; 1447 char ProfileKindName[20]; 1448 switch (I) { 1449 case IPVK_IndirectCallTarget: 1450 strncpy(ProfileKindName, "IndirectCall", 19); 1451 break; 1452 case IPVK_MemOPSize: 1453 strncpy(ProfileKindName, "MemOP", 19); 1454 break; 1455 default: 1456 snprintf(ProfileKindName, 19, "VP[%d]", I); 1457 break; 1458 } 1459 OS << " " << ProfileKindName 1460 << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100) 1461 << "\n"; 1462 if (Mismatch.NumEntries) 1463 OS << " Mismatched count percentage (" << ProfileKindName 1464 << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n"; 1465 if (Unique.NumEntries) 1466 OS << " Percentage of " << ProfileKindName 1467 << " profile only in test_profile: " 1468 << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n"; 1469 OS << " " << ProfileKindName 1470 << " profile base count sum: " << format("%.0f", Base.ValueCounts[I]) 1471 << "\n" 1472 << " " << ProfileKindName 1473 << " profile test count sum: " << format("%.0f", Test.ValueCounts[I]) 1474 << "\n"; 1475 } 1476 } 1477 1478 namespace IndexedInstrProf { 1479 // A C++14 compatible version of the offsetof macro. 1480 template <typename T1, typename T2> 1481 inline size_t constexpr offsetOf(T1 T2::*Member) { 1482 constexpr T2 Object{}; 1483 return size_t(&(Object.*Member)) - size_t(&Object); 1484 } 1485 1486 static inline uint64_t read(const unsigned char *Buffer, size_t Offset) { 1487 return *reinterpret_cast<const uint64_t *>(Buffer + Offset); 1488 } 1489 1490 uint64_t Header::formatVersion() const { 1491 using namespace support; 1492 return endian::byte_swap<uint64_t, llvm::endianness::little>(Version); 1493 } 1494 1495 Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) { 1496 using namespace support; 1497 static_assert(std::is_standard_layout_v<Header>, 1498 "The header should be standard layout type since we use offset " 1499 "of fields to read."); 1500 Header H; 1501 1502 H.Magic = read(Buffer, offsetOf(&Header::Magic)); 1503 // Check the magic number. 1504 uint64_t Magic = 1505 endian::byte_swap<uint64_t, llvm::endianness::little>(H.Magic); 1506 if (Magic != IndexedInstrProf::Magic) 1507 return make_error<InstrProfError>(instrprof_error::bad_magic); 1508 1509 // Read the version. 1510 H.Version = read(Buffer, offsetOf(&Header::Version)); 1511 if (GET_VERSION(H.formatVersion()) > 1512 IndexedInstrProf::ProfVersion::CurrentVersion) 1513 return make_error<InstrProfError>(instrprof_error::unsupported_version); 1514 1515 switch (GET_VERSION(H.formatVersion())) { 1516 // When a new field is added in the header add a case statement here to 1517 // populate it. 1518 static_assert( 1519 IndexedInstrProf::ProfVersion::CurrentVersion == Version11, 1520 "Please update the reading code below if a new field has been added, " 1521 "if not add a case statement to fall through to the latest version."); 1522 case 11ull: 1523 [[fallthrough]]; 1524 case 10ull: 1525 H.TemporalProfTracesOffset = 1526 read(Buffer, offsetOf(&Header::TemporalProfTracesOffset)); 1527 [[fallthrough]]; 1528 case 9ull: 1529 H.BinaryIdOffset = read(Buffer, offsetOf(&Header::BinaryIdOffset)); 1530 [[fallthrough]]; 1531 case 8ull: 1532 H.MemProfOffset = read(Buffer, offsetOf(&Header::MemProfOffset)); 1533 [[fallthrough]]; 1534 default: // Version7 (when the backwards compatible header was introduced). 1535 H.HashType = read(Buffer, offsetOf(&Header::HashType)); 1536 H.HashOffset = read(Buffer, offsetOf(&Header::HashOffset)); 1537 } 1538 1539 return H; 1540 } 1541 1542 size_t Header::size() const { 1543 switch (GET_VERSION(formatVersion())) { 1544 // When a new field is added to the header add a case statement here to 1545 // compute the size as offset of the new field + size of the new field. This 1546 // relies on the field being added to the end of the list. 1547 static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11, 1548 "Please update the size computation below if a new field has " 1549 "been added to the header, if not add a case statement to " 1550 "fall through to the latest version."); 1551 case 11ull: 1552 [[fallthrough]]; 1553 case 10ull: 1554 return offsetOf(&Header::TemporalProfTracesOffset) + 1555 sizeof(Header::TemporalProfTracesOffset); 1556 case 9ull: 1557 return offsetOf(&Header::BinaryIdOffset) + sizeof(Header::BinaryIdOffset); 1558 case 8ull: 1559 return offsetOf(&Header::MemProfOffset) + sizeof(Header::MemProfOffset); 1560 default: // Version7 (when the backwards compatible header was introduced). 1561 return offsetOf(&Header::HashOffset) + sizeof(Header::HashOffset); 1562 } 1563 } 1564 1565 } // namespace IndexedInstrProf 1566 1567 } // end namespace llvm 1568