1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2|* 3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4|* See https://llvm.org/LICENSE.txt for license information. 5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6|* 7\*===----------------------------------------------------------------------===*/ 8/* 9 * This is the main file that defines all the data structure, signature, 10 * constant literals that are shared across profiling runtime library, 11 * compiler (instrumentation), and host tools (reader/writer). The entities 12 * defined in this file affect the profile runtime ABI, the raw profile format, 13 * or both. 14 * 15 * The file has two identical copies. The primary copy lives in LLVM and 16 * the other one sits in compiler-rt/lib/profile directory. To make changes 17 * in this file, first modify the primary copy and copy it over to compiler-rt. 18 * Testing of any change in this file can start only after the two copies are 19 * synced up. 20 * 21 * The first part of the file includes macros that defines types, names, and 22 * initializers for the member fields of the core data structures. The field 23 * declarations for one structure is enabled by defining the field activation 24 * macro associated with that structure. Only one field activation record 25 * can be defined at one time and the rest definitions will be filtered out by 26 * the preprocessor. 27 * 28 * Examples of how the template is used to instantiate structure definition: 29 * 1. To declare a structure: 30 * 31 * struct ProfData { 32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 33 * Type Name; 34 * #include "llvm/ProfileData/InstrProfData.inc" 35 * }; 36 * 37 * 2. To construct LLVM type arrays for the struct type: 38 * 39 * Type *DataTypes[] = { 40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 41 * LLVMType, 42 * #include "llvm/ProfileData/InstrProfData.inc" 43 * }; 44 * 45 * 4. To construct constant array for the initializers: 46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 47 * Initializer, 48 * Constant *ConstantVals[] = { 49 * #include "llvm/ProfileData/InstrProfData.inc" 50 * }; 51 * 52 * 53 * The second part of the file includes definitions all other entities that 54 * are related to runtime ABI and format. When no field activation macro is 55 * defined, this file can be included to introduce the definitions. 56 * 57\*===----------------------------------------------------------------------===*/ 58 59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in 60 * the compiler runtime. */ 61#ifndef INSTR_PROF_VISIBILITY 62#define INSTR_PROF_VISIBILITY 63#endif 64 65/* INSTR_PROF_DATA start. */ 66/* Definition of member fields of the per-function control structure. */ 67#ifndef INSTR_PROF_DATA 68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 69#else 70#define INSTR_PROF_DATA_DEFINED 71#endif 72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) 75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 77 Inc->getHash()->getZExtValue())) 78INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) 79INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) 80/* This is used to map function pointers for the indirect call targets to 81 * function name hashes during the conversion from raw to merged profile 82 * data. 83 */ 84INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \ 85 FunctionAddr) 86INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \ 87 ValuesPtrExpr) 88INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 89 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 90INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 91 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ 92INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ 93 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) 94#undef INSTR_PROF_DATA 95/* INSTR_PROF_DATA end. */ 96 97 98/* This is an internal data structure used by value profiler. It 99 * is defined here to allow serialization code sharing by LLVM 100 * to be used in unit test. 101 * 102 * typedef struct ValueProfNode { 103 * // InstrProfValueData VData; 104 * uint64_t Value; 105 * uint64_t Count; 106 * struct ValueProfNode *Next; 107 * } ValueProfNode; 108 */ 109/* INSTR_PROF_VALUE_NODE start. */ 110#ifndef INSTR_PROF_VALUE_NODE 111#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) 112#else 113#define INSTR_PROF_DATA_DEFINED 114#endif 115INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ 116 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 117INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ 118 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 119INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \ 120 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) 121#undef INSTR_PROF_VALUE_NODE 122/* INSTR_PROF_VALUE_NODE end. */ 123 124/* INSTR_PROF_RAW_HEADER start */ 125/* Definition of member fields of the raw profile header data structure. */ 126/* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating 127 raw profile format. */ 128#ifndef INSTR_PROF_RAW_HEADER 129#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 130#else 131#define INSTR_PROF_DATA_DEFINED 132#endif 133INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 134INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 135INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) 136INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) 137INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) 138INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) 139INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) 140INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) 141INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) 142INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 143INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, 144 (uintptr_t)CountersBegin - (uintptr_t)DataBegin) 145INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, 146 (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) 147INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 148INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 149#undef INSTR_PROF_RAW_HEADER 150/* INSTR_PROF_RAW_HEADER end */ 151 152/* VALUE_PROF_FUNC_PARAM start */ 153/* Definition of parameter types of the runtime API used to do value profiling 154 * for a given value site. 155 */ 156#ifndef VALUE_PROF_FUNC_PARAM 157#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 158#define INSTR_PROF_COMMA 159#else 160#define INSTR_PROF_DATA_DEFINED 161#define INSTR_PROF_COMMA , 162#endif 163VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 164 INSTR_PROF_COMMA 165VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA 166VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 167#undef VALUE_PROF_FUNC_PARAM 168#undef INSTR_PROF_COMMA 169/* VALUE_PROF_FUNC_PARAM end */ 170 171/* VALUE_PROF_KIND start */ 172#ifndef VALUE_PROF_KIND 173#define VALUE_PROF_KIND(Enumerator, Value, Descr) 174#else 175#define INSTR_PROF_DATA_DEFINED 176#endif 177/* For indirect function call value profiling, the addresses of the target 178 * functions are profiled by the instrumented code. The target addresses are 179 * written in the raw profile data and converted to target function name's MD5 180 * hash by the profile reader during deserialization. Typically, this happens 181 * when the raw profile data is read during profile merging. 182 * 183 * For this remapping the ProfData is used. ProfData contains both the function 184 * name hash and the function address. 185 */ 186VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") 187/* For memory intrinsic functions size profiling. */ 188VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") 189/* These two kinds must be the last to be 190 * declared. This is to make sure the string 191 * array created with the template can be 192 * indexed with the kind value. 193 */ 194VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") 195VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") 196 197#undef VALUE_PROF_KIND 198/* VALUE_PROF_KIND end */ 199 200#undef COVMAP_V2_OR_V3 201#ifdef COVMAP_V2 202#define COVMAP_V2_OR_V3 203#endif 204#ifdef COVMAP_V3 205#define COVMAP_V2_OR_V3 206#endif 207 208/* COVMAP_FUNC_RECORD start */ 209/* Definition of member fields of the function record structure in coverage 210 * map. 211 */ 212#ifndef COVMAP_FUNC_RECORD 213#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 214#else 215#define INSTR_PROF_DATA_DEFINED 216#endif 217#ifdef COVMAP_V1 218COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ 219 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 220 llvm::PointerType::getUnqual(Ctx))) 221COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 222 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 223 NameValue.size())) 224#endif 225#ifdef COVMAP_V2_OR_V3 226COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 227 llvm::ConstantInt::get( \ 228 llvm::Type::getInt64Ty(Ctx), NameHash)) 229#endif 230COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 231 llvm::ConstantInt::get( \ 232 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) 233COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 234 llvm::ConstantInt::get( \ 235 llvm::Type::getInt64Ty(Ctx), FuncHash)) 236#ifdef COVMAP_V3 237COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ 238 llvm::ConstantInt::get( \ 239 llvm::Type::getInt64Ty(Ctx), FilenamesRef)) 240COVMAP_FUNC_RECORD(const char, \ 241 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ 242 CoverageMapping.size()), \ 243 CoverageMapping, 244 llvm::ConstantDataArray::getRaw( \ 245 CoverageMapping, CoverageMapping.size(), \ 246 llvm::Type::getInt8Ty(Ctx))) 247#endif 248#undef COVMAP_FUNC_RECORD 249/* COVMAP_FUNC_RECORD end. */ 250 251/* COVMAP_HEADER start */ 252/* Definition of member fields of coverage map header. 253 */ 254#ifndef COVMAP_HEADER 255#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 256#else 257#define INSTR_PROF_DATA_DEFINED 258#endif 259COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 260 llvm::ConstantInt::get(Int32Ty, NRecords)) 261COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 262 llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 263COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 264 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 265COVMAP_HEADER(uint32_t, Int32Ty, Version, \ 266 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 267#undef COVMAP_HEADER 268/* COVMAP_HEADER end. */ 269 270 271#ifdef INSTR_PROF_SECT_ENTRY 272#define INSTR_PROF_DATA_DEFINED 273INSTR_PROF_SECT_ENTRY(IPSK_data, \ 274 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ 275 INSTR_PROF_DATA_COFF, "__DATA,") 276INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ 277 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ 278 INSTR_PROF_CNTS_COFF, "__DATA,") 279INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ 280 INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ 281 INSTR_PROF_BITS_COFF, "__DATA,") 282INSTR_PROF_SECT_ENTRY(IPSK_name, \ 283 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ 284 INSTR_PROF_NAME_COFF, "__DATA,") 285INSTR_PROF_SECT_ENTRY(IPSK_vals, \ 286 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ 287 INSTR_PROF_VALS_COFF, "__DATA,") 288INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ 289 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ 290 INSTR_PROF_VNODES_COFF, "__DATA,") 291INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ 292 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ 293 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") 294INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ 295 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ 296 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,") 297INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ 298 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ 299 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") 300INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ 301 INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ 302 INSTR_PROF_COVDATA_COFF, "__LLVM_COV,") 303INSTR_PROF_SECT_ENTRY(IPSK_covname, \ 304 INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ 305 INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") 306 307#undef INSTR_PROF_SECT_ENTRY 308#endif 309 310 311#ifdef INSTR_PROF_VALUE_PROF_DATA 312#define INSTR_PROF_DATA_DEFINED 313 314#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 315/*! 316 * This is the header of the data structure that defines the on-disk 317 * layout of the value profile data of a particular kind for one function. 318 */ 319typedef struct ValueProfRecord { 320 /* The kind of the value profile record. */ 321 uint32_t Kind; 322 /* 323 * The number of value profile sites. It is guaranteed to be non-zero; 324 * otherwise the record for this kind won't be emitted. 325 */ 326 uint32_t NumValueSites; 327 /* 328 * The first element of the array that stores the number of profiled 329 * values for each value site. The size of the array is NumValueSites. 330 * Since NumValueSites is greater than zero, there is at least one 331 * element in the array. 332 */ 333 uint8_t SiteCountArray[1]; 334 335 /* 336 * The fake declaration is for documentation purpose only. 337 * Align the start of next field to be on 8 byte boundaries. 338 uint8_t Padding[X]; 339 */ 340 341 /* The array of value profile data. The size of the array is the sum 342 * of all elements in SiteCountArray[]. 343 InstrProfValueData ValueData[]; 344 */ 345 346#ifdef __cplusplus 347 /*! 348 * Return the number of value sites. 349 */ 350 uint32_t getNumValueSites() const { return NumValueSites; } 351 /*! 352 * Read data from this record and save it to Record. 353 */ 354 void deserializeTo(InstrProfRecord &Record, 355 InstrProfSymtab *SymTab); 356 /* 357 * In-place byte swap: 358 * Do byte swap for this instance. \c Old is the original order before 359 * the swap, and \c New is the New byte order. 360 */ 361 void swapBytes(llvm::endianness Old, llvm::endianness New); 362#endif 363} ValueProfRecord; 364 365/*! 366 * Per-function header/control data structure for value profiling 367 * data in indexed format. 368 */ 369typedef struct ValueProfData { 370 /* 371 * Total size in bytes including this field. It must be a multiple 372 * of sizeof(uint64_t). 373 */ 374 uint32_t TotalSize; 375 /* 376 *The number of value profile kinds that has value profile data. 377 * In this implementation, a value profile kind is considered to 378 * have profile data if the number of value profile sites for the 379 * kind is not zero. More aggressively, the implementation can 380 * choose to check the actual data value: if none of the value sites 381 * has any profiled values, the kind can be skipped. 382 */ 383 uint32_t NumValueKinds; 384 385 /* 386 * Following are a sequence of variable length records. The prefix/header 387 * of each record is defined by ValueProfRecord type. The number of 388 * records is NumValueKinds. 389 * ValueProfRecord Record_1; 390 * ValueProfRecord Record_N; 391 */ 392 393#if __cplusplus 394 /*! 395 * Return the total size in bytes of the on-disk value profile data 396 * given the data stored in Record. 397 */ 398 static uint32_t getSize(const InstrProfRecord &Record); 399 /*! 400 * Return a pointer to \c ValueProfData instance ready to be streamed. 401 */ 402 static std::unique_ptr<ValueProfData> 403 serializeFrom(const InstrProfRecord &Record); 404 /*! 405 * Check the integrity of the record. 406 */ 407 Error checkIntegrity(); 408 /*! 409 * Return a pointer to \c ValueProfileData instance ready to be read. 410 * All data in the instance are properly byte swapped. The input 411 * data is assumed to be in little endian order. 412 */ 413 static Expected<std::unique_ptr<ValueProfData>> 414 getValueProfData(const unsigned char *SrcBuffer, 415 const unsigned char *const SrcBufferEnd, 416 llvm::endianness SrcDataEndianness); 417 /*! 418 * Swap byte order from \c Endianness order to host byte order. 419 */ 420 void swapBytesToHost(llvm::endianness Endianness); 421 /*! 422 * Swap byte order from host byte order to \c Endianness order. 423 */ 424 void swapBytesFromHost(llvm::endianness Endianness); 425 /*! 426 * Return the total size of \c ValueProfileData. 427 */ 428 uint32_t getSize() const { return TotalSize; } 429 /*! 430 * Read data from this data and save it to \c Record. 431 */ 432 void deserializeTo(InstrProfRecord &Record, 433 InstrProfSymtab *SymTab); 434 void operator delete(void *ptr) { ::operator delete(ptr); } 435#endif 436} ValueProfData; 437 438/* 439 * The closure is designed to abstact away two types of value profile data: 440 * - InstrProfRecord which is the primary data structure used to 441 * represent profile data in host tools (reader, writer, and profile-use) 442 * - value profile runtime data structure suitable to be used by C 443 * runtime library. 444 * 445 * Both sources of data need to serialize to disk/memory-buffer in common 446 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 447 * writer to share the same format and code with indexed profile writer. 448 * 449 * For documentation of the member methods below, refer to corresponding methods 450 * in class InstrProfRecord. 451 */ 452typedef struct ValueProfRecordClosure { 453 const void *Record; 454 uint32_t (*GetNumValueKinds)(const void *Record); 455 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 456 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 457 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 458 459 /* 460 * After extracting the value profile data from the value profile record, 461 * this method is used to map the in-memory value to on-disk value. If 462 * the method is null, value will be written out untranslated. 463 */ 464 uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 465 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 466 uint32_t S); 467 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 468} ValueProfRecordClosure; 469 470INSTR_PROF_VISIBILITY ValueProfRecord * 471getFirstValueProfRecord(ValueProfData *VPD); 472INSTR_PROF_VISIBILITY ValueProfRecord * 473getValueProfRecordNext(ValueProfRecord *VPR); 474INSTR_PROF_VISIBILITY InstrProfValueData * 475getValueProfRecordValueData(ValueProfRecord *VPR); 476INSTR_PROF_VISIBILITY uint32_t 477getValueProfRecordHeaderSize(uint32_t NumValueSites); 478 479#undef INSTR_PROF_VALUE_PROF_DATA 480#endif /* INSTR_PROF_VALUE_PROF_DATA */ 481 482 483#ifdef INSTR_PROF_COMMON_API_IMPL 484#define INSTR_PROF_DATA_DEFINED 485#ifdef __cplusplus 486#define INSTR_PROF_INLINE inline 487#define INSTR_PROF_NULLPTR nullptr 488#else 489#define INSTR_PROF_INLINE 490#define INSTR_PROF_NULLPTR NULL 491#endif 492 493#ifndef offsetof 494#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 495#endif 496 497/*! 498 * Return the \c ValueProfRecord header size including the 499 * padding bytes. 500 */ 501INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 502uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 503 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 504 sizeof(uint8_t) * NumValueSites; 505 /* Round the size to multiple of 8 bytes. */ 506 Size = (Size + 7) & ~7; 507 return Size; 508} 509 510/*! 511 * Return the total size of the value profile record including the 512 * header and the value data. 513 */ 514INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 515uint32_t getValueProfRecordSize(uint32_t NumValueSites, 516 uint32_t NumValueData) { 517 return getValueProfRecordHeaderSize(NumValueSites) + 518 sizeof(InstrProfValueData) * NumValueData; 519} 520 521/*! 522 * Return the pointer to the start of value data array. 523 */ 524INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 525InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 526 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 527 This->NumValueSites)); 528} 529 530/*! 531 * Return the total number of value data for \c This record. 532 */ 533INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 534uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 535 uint32_t NumValueData = 0; 536 uint32_t I; 537 for (I = 0; I < This->NumValueSites; I++) 538 NumValueData += This->SiteCountArray[I]; 539 return NumValueData; 540} 541 542/*! 543 * Use this method to advance to the next \c This \c ValueProfRecord. 544 */ 545INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 546ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 547 uint32_t NumValueData = getValueProfRecordNumValueData(This); 548 return (ValueProfRecord *)((char *)This + 549 getValueProfRecordSize(This->NumValueSites, 550 NumValueData)); 551} 552 553/*! 554 * Return the first \c ValueProfRecord instance. 555 */ 556INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 557ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 558 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 559} 560 561/* Closure based interfaces. */ 562 563/*! 564 * Return the total size in bytes of the on-disk value profile data 565 * given the data stored in Record. 566 */ 567INSTR_PROF_VISIBILITY uint32_t 568getValueProfDataSize(ValueProfRecordClosure *Closure) { 569 uint32_t Kind; 570 uint32_t TotalSize = sizeof(ValueProfData); 571 const void *Record = Closure->Record; 572 573 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 574 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 575 if (!NumValueSites) 576 continue; 577 TotalSize += getValueProfRecordSize(NumValueSites, 578 Closure->GetNumValueData(Record, Kind)); 579 } 580 return TotalSize; 581} 582 583/*! 584 * Extract value profile data of a function for the profile kind \c ValueKind 585 * from the \c Closure and serialize the data into \c This record instance. 586 */ 587INSTR_PROF_VISIBILITY void 588serializeValueProfRecordFrom(ValueProfRecord *This, 589 ValueProfRecordClosure *Closure, 590 uint32_t ValueKind, uint32_t NumValueSites) { 591 uint32_t S; 592 const void *Record = Closure->Record; 593 This->Kind = ValueKind; 594 This->NumValueSites = NumValueSites; 595 InstrProfValueData *DstVD = getValueProfRecordValueData(This); 596 597 for (S = 0; S < NumValueSites; S++) { 598 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 599 This->SiteCountArray[S] = ND; 600 Closure->GetValueForSite(Record, DstVD, ValueKind, S); 601 DstVD += ND; 602 } 603} 604 605/*! 606 * Extract value profile data of a function from the \c Closure 607 * and serialize the data into \c DstData if it is not NULL or heap 608 * memory allocated by the \c Closure's allocator method. If \c 609 * DstData is not null, the caller is expected to set the TotalSize 610 * in DstData. 611 */ 612INSTR_PROF_VISIBILITY ValueProfData * 613serializeValueProfDataFrom(ValueProfRecordClosure *Closure, 614 ValueProfData *DstData) { 615 uint32_t Kind; 616 uint32_t TotalSize = 617 DstData ? DstData->TotalSize : getValueProfDataSize(Closure); 618 619 ValueProfData *VPD = 620 DstData ? DstData : Closure->AllocValueProfData(TotalSize); 621 622 VPD->TotalSize = TotalSize; 623 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 624 ValueProfRecord *VR = getFirstValueProfRecord(VPD); 625 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 626 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 627 if (!NumValueSites) 628 continue; 629 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 630 VR = getValueProfRecordNext(VR); 631 } 632 return VPD; 633} 634 635#undef INSTR_PROF_COMMON_API_IMPL 636#endif /* INSTR_PROF_COMMON_API_IMPL */ 637 638/*============================================================================*/ 639 640#ifndef INSTR_PROF_DATA_DEFINED 641 642#ifndef INSTR_PROF_DATA_INC 643#define INSTR_PROF_DATA_INC 644 645/* Helper macros. */ 646#define INSTR_PROF_SIMPLE_QUOTE(x) #x 647#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 648#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 649#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 650 651/* Magic number to detect file format and endianness. 652 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 653 * so that utilities, like strings, don't grab it as a string. 129 is also 654 * invalid UTF-8, and high enough to be interesting. 655 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 656 * for 32-bit platforms. 657 */ 658#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 659 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 660 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 661#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 662 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 663 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 664 665/* Raw profile format version (start from 1). */ 666#define INSTR_PROF_RAW_VERSION 9 667/* Indexed profile format version (start from 1). */ 668#define INSTR_PROF_INDEX_VERSION 11 669/* Coverage mapping format version (start from 0). */ 670#define INSTR_PROF_COVMAP_VERSION 6 671 672/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the 673 * version for other variants of profile. We set the 8th most significant bit 674 * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation 675 * generated profile, and 0 if this is a Clang FE generated profile. 676 * 1 in bit 57 indicates there are context-sensitive records in the profile. 677 * The 59th bit indicates whether to use debug info to correlate profiles. 678 * The 60th bit indicates single byte coverage instrumentation. 679 * The 61st bit indicates function entry instrumentation only. 680 * The 62nd bit indicates whether memory profile information is present. 681 * The 63rd bit indicates if this is a temporal profile. 682 */ 683#define VARIANT_MASKS_ALL 0xffffffff00000000ULL 684#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 685#define VARIANT_MASK_IR_PROF (0x1ULL << 56) 686#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) 687#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) 688#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59) 689#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60) 690#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61) 691#define VARIANT_MASK_MEMPROF (0x1ULL << 62) 692#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63) 693#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version 694#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime 695#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias 696#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp 697 698/* The variable that holds the name of the profile data 699 * specified via command line. */ 700#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename 701 702/* section name strings common to all targets other 703 than WIN32 */ 704#define INSTR_PROF_DATA_COMMON __llvm_prf_data 705#define INSTR_PROF_NAME_COMMON __llvm_prf_names 706#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts 707#define INSTR_PROF_BITS_COMMON __llvm_prf_bits 708#define INSTR_PROF_VALS_COMMON __llvm_prf_vals 709#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds 710#define INSTR_PROF_COVMAP_COMMON __llvm_covmap 711#define INSTR_PROF_COVFUN_COMMON __llvm_covfun 712#define INSTR_PROF_COVDATA_COMMON __llvm_covdata 713#define INSTR_PROF_COVNAME_COMMON __llvm_covnames 714#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile 715/* Windows section names. Because these section names contain dollar characters, 716 * they must be quoted. 717 */ 718#define INSTR_PROF_DATA_COFF ".lprfd$M" 719#define INSTR_PROF_NAME_COFF ".lprfn$M" 720#define INSTR_PROF_CNTS_COFF ".lprfc$M" 721#define INSTR_PROF_BITS_COFF ".lprfb$M" 722#define INSTR_PROF_VALS_COFF ".lprfv$M" 723#define INSTR_PROF_VNODES_COFF ".lprfnd$M" 724#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" 725#define INSTR_PROF_COVFUN_COFF ".lcovfun$M" 726/* Since cov data and cov names sections are not allocated, we don't need to 727 * access them at runtime. 728 */ 729#define INSTR_PROF_COVDATA_COFF ".lcovd" 730#define INSTR_PROF_COVNAME_COFF ".lcovn" 731#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" 732 733#ifdef _WIN32 734/* Runtime section names and name strings. */ 735#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF 736#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF 737#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF 738#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF 739/* Array of pointers. Each pointer points to a list 740 * of value nodes associated with one value site. 741 */ 742#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF 743/* Value profile nodes section. */ 744#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF 745#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF 746#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF 747#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF 748#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF 749#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF 750#else 751/* Runtime section names and name strings. */ 752#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) 753#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) 754#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) 755#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) 756/* Array of pointers. Each pointer points to a list 757 * of value nodes associated with one value site. 758 */ 759#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) 760/* Value profile nodes section. */ 761#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) 762#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) 763#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) 764#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) 765#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) 766/* Order file instrumentation. */ 767#define INSTR_PROF_ORDERFILE_SECT_NAME \ 768 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) 769#endif 770 771#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer 772#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ 773 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) 774#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx 775#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ 776 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) 777 778/* Macros to define start/stop section symbol for a given 779 * section on Linux. For instance 780 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 781 * expand to __start___llvm_prof_data 782 */ 783#define INSTR_PROF_SECT_START(Sect) \ 784 INSTR_PROF_CONCAT(__start_,Sect) 785#define INSTR_PROF_SECT_STOP(Sect) \ 786 INSTR_PROF_CONCAT(__stop_,Sect) 787 788/* Value Profiling API linkage name. */ 789#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 790#define INSTR_PROF_VALUE_PROF_FUNC_STR \ 791 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 792#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop 793#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ 794 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) 795 796/* InstrProfile per-function control data alignment. */ 797#define INSTR_PROF_DATA_ALIGNMENT 8 798 799/* The data structure that represents a tracked value by the 800 * value profiler. 801 */ 802typedef struct InstrProfValueData { 803 /* Profiled value. */ 804 uint64_t Value; 805 /* Number of times the value appears in the training run. */ 806 uint64_t Count; 807} InstrProfValueData; 808 809#endif /* INSTR_PROF_DATA_INC */ 810 811#ifndef INSTR_ORDER_FILE_INC 812/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ 813#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 814#define INSTR_ORDER_FILE_BUFFER_BITS 17 815#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff 816#endif /* INSTR_ORDER_FILE_INC */ 817#else 818#undef INSTR_PROF_DATA_DEFINED 819#endif 820 821#undef COVMAP_V2_OR_V3 822 823#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API 824 825#ifdef __cplusplus 826#define INSTR_PROF_INLINE inline 827#else 828#define INSTR_PROF_INLINE 829#endif 830 831/* The value range buckets (22 buckets) for the memop size value profiling looks 832 * like: 833 * 834 * [0, 0] 835 * [1, 1] 836 * [2, 2] 837 * [3, 3] 838 * [4, 4] 839 * [5, 5] 840 * [6, 6] 841 * [7, 7] 842 * [8, 8] 843 * [9, 15] 844 * [16, 16] 845 * [17, 31] 846 * [32, 32] 847 * [33, 63] 848 * [64, 64] 849 * [65, 127] 850 * [128, 128] 851 * [129, 255] 852 * [256, 256] 853 * [257, 511] 854 * [512, 512] 855 * [513, UINT64_MAX] 856 * 857 * Each range has a 'representative value' which is the lower end value of the 858 * range and used to store in the runtime profile data records and the VP 859 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. 860 */ 861#define INSTR_PROF_NUM_BUCKETS 22 862 863/* 864 * Clz and Popcount. This code was copied from 865 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and 866 * llvm/include/llvm/Support/MathExtras.h. 867 */ 868#if defined(_MSC_VER) && !defined(__clang__) 869 870#include <intrin.h> 871INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 872int InstProfClzll(unsigned long long X) { 873 unsigned long LeadZeroIdx = 0; 874#if !defined(_M_ARM64) && !defined(_M_X64) 875 // Scan the high 32 bits. 876 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) 877 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset 878 // from the MSB. 879 // Scan the low 32 bits. 880 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) 881 return (int)(63 - LeadZeroIdx); 882#else 883 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; 884#endif 885 return 64; 886} 887INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 888int InstProfPopcountll(unsigned long long X) { 889 // This code originates from https://reviews.llvm.org/rG30626254510f. 890 unsigned long long v = X; 891 v = v - ((v >> 1) & 0x5555555555555555ULL); 892 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); 893 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; 894 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); 895} 896 897#else 898 899INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 900int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } 901INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 902int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } 903 904#endif /* defined(_MSC_VER) && !defined(__clang__) */ 905 906/* Map an (observed) memop size value to the representative value of its range. 907 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ 908INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t 909InstrProfGetRangeRepValue(uint64_t Value) { 910 if (Value <= 8) 911 // The first ranges are individually tracked. Use the value as is. 912 return Value; 913 else if (Value >= 513) 914 // The last range is mapped to its lowest value. 915 return 513; 916 else if (InstProfPopcountll(Value) == 1) 917 // If it's a power of two, use it as is. 918 return Value; 919 else 920 // Otherwise, take to the previous power of two + 1. 921 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1; 922} 923 924/* Return true if the range that an (observed) memop size value belongs to has 925 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> 926 * false, 64 -> true, 100 -> false, 513 -> false. */ 927INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned 928InstrProfIsSingleValRange(uint64_t Value) { 929 if (Value <= 8) 930 // The first ranges are individually tracked. 931 return 1; 932 else if (InstProfPopcountll(Value) == 1) 933 // If it's a power of two, there's only one value. 934 return 1; 935 else 936 // Otherwise, there's more than one value in the range. 937 return 0; 938} 939 940#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ 941