1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2|* 3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4|* See https://llvm.org/LICENSE.txt for license information. 5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6|* 7\*===----------------------------------------------------------------------===*/ 8/* 9 * This is the main file that defines all the data structure, signature, 10 * constant literals that are shared across profiling runtime library, 11 * compiler (instrumentation), and host tools (reader/writer). The entities 12 * defined in this file affect the profile runtime ABI, the raw profile format, 13 * or both. 14 * 15 * The file has two identical copies. The primary copy lives in LLVM and 16 * the other one sits in compiler-rt/lib/profile directory. To make changes 17 * in this file, first modify the primary copy and copy it over to compiler-rt. 18 * Testing of any change in this file can start only after the two copies are 19 * synced up. 20 * 21 * The first part of the file includes macros that defines types, names, and 22 * initializers for the member fields of the core data structures. The field 23 * declarations for one structure is enabled by defining the field activation 24 * macro associated with that structure. Only one field activation record 25 * can be defined at one time and the rest definitions will be filtered out by 26 * the preprocessor. 27 * 28 * Examples of how the template is used to instantiate structure definition: 29 * 1. To declare a structure: 30 * 31 * struct ProfData { 32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 33 * Type Name; 34 * #include "llvm/ProfileData/InstrProfData.inc" 35 * }; 36 * 37 * 2. To construct LLVM type arrays for the struct type: 38 * 39 * Type *DataTypes[] = { 40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 41 * LLVMType, 42 * #include "llvm/ProfileData/InstrProfData.inc" 43 * }; 44 * 45 * 4. To construct constant array for the initializers: 46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 47 * Initializer, 48 * Constant *ConstantVals[] = { 49 * #include "llvm/ProfileData/InstrProfData.inc" 50 * }; 51 * 52 * 53 * The second part of the file includes definitions all other entities that 54 * are related to runtime ABI and format. When no field activation macro is 55 * defined, this file can be included to introduce the definitions. 56 * 57\*===----------------------------------------------------------------------===*/ 58 59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in 60 * the compiler runtime. */ 61#ifndef INSTR_PROF_VISIBILITY 62#define INSTR_PROF_VISIBILITY 63#endif 64 65/* INSTR_PROF_DATA start. */ 66/* Definition of member fields of the per-function control structure. */ 67#ifndef INSTR_PROF_DATA 68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 69#else 70#define INSTR_PROF_DATA_DEFINED 71#endif 72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) 75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 77 Inc->getHash()->getZExtValue())) 78INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ 79 ConstantExpr::getBitCast(CounterPtr, \ 80 llvm::Type::getInt64PtrTy(Ctx))) 81/* This is used to map function pointers for the indirect call targets to 82 * function name hashes during the conversion from raw to merged profile 83 * data. 84 */ 85INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ 86 FunctionAddr) 87INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ 88 ValuesPtrExpr) 89INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 90 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 91INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 92 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) 93#undef INSTR_PROF_DATA 94/* INSTR_PROF_DATA end. */ 95 96 97/* This is an internal data structure used by value profiler. It 98 * is defined here to allow serialization code sharing by LLVM 99 * to be used in unit test. 100 * 101 * typedef struct ValueProfNode { 102 * // InstrProfValueData VData; 103 * uint64_t Value; 104 * uint64_t Count; 105 * struct ValueProfNode *Next; 106 * } ValueProfNode; 107 */ 108/* INSTR_PROF_VALUE_NODE start. */ 109#ifndef INSTR_PROF_VALUE_NODE 110#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) 111#else 112#define INSTR_PROF_DATA_DEFINED 113#endif 114INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ 115 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 116INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ 117 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 118INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ 119 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) 120#undef INSTR_PROF_VALUE_NODE 121/* INSTR_PROF_VALUE_NODE end. */ 122 123/* INSTR_PROF_RAW_HEADER start */ 124/* Definition of member fields of the raw profile header data structure. */ 125#ifndef INSTR_PROF_RAW_HEADER 126#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 127#else 128#define INSTR_PROF_DATA_DEFINED 129#endif 130INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 131INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 132INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) 133INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) 134INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) 135INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) 136INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) 137INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 138INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) 139INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 140INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 141#undef INSTR_PROF_RAW_HEADER 142/* INSTR_PROF_RAW_HEADER end */ 143 144/* VALUE_PROF_FUNC_PARAM start */ 145/* Definition of parameter types of the runtime API used to do value profiling 146 * for a given value site. 147 */ 148#ifndef VALUE_PROF_FUNC_PARAM 149#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 150#define INSTR_PROF_COMMA 151#else 152#define INSTR_PROF_DATA_DEFINED 153#define INSTR_PROF_COMMA , 154#endif 155VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 156 INSTR_PROF_COMMA 157VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA 158VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 159#undef VALUE_PROF_FUNC_PARAM 160#undef INSTR_PROF_COMMA 161/* VALUE_PROF_FUNC_PARAM end */ 162 163/* VALUE_PROF_KIND start */ 164#ifndef VALUE_PROF_KIND 165#define VALUE_PROF_KIND(Enumerator, Value, Descr) 166#else 167#define INSTR_PROF_DATA_DEFINED 168#endif 169/* For indirect function call value profiling, the addresses of the target 170 * functions are profiled by the instrumented code. The target addresses are 171 * written in the raw profile data and converted to target function name's MD5 172 * hash by the profile reader during deserialization. Typically, this happens 173 * when the raw profile data is read during profile merging. 174 * 175 * For this remapping the ProfData is used. ProfData contains both the function 176 * name hash and the function address. 177 */ 178VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") 179/* For memory intrinsic functions size profiling. */ 180VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") 181/* These two kinds must be the last to be 182 * declared. This is to make sure the string 183 * array created with the template can be 184 * indexed with the kind value. 185 */ 186VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") 187VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") 188 189#undef VALUE_PROF_KIND 190/* VALUE_PROF_KIND end */ 191 192#undef COVMAP_V2_OR_V3 193#ifdef COVMAP_V2 194#define COVMAP_V2_OR_V3 195#endif 196#ifdef COVMAP_V3 197#define COVMAP_V2_OR_V3 198#endif 199 200/* COVMAP_FUNC_RECORD start */ 201/* Definition of member fields of the function record structure in coverage 202 * map. 203 */ 204#ifndef COVMAP_FUNC_RECORD 205#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 206#else 207#define INSTR_PROF_DATA_DEFINED 208#endif 209#ifdef COVMAP_V1 210COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ 211 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 212 llvm::Type::getInt8PtrTy(Ctx))) 213COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 214 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 215 NameValue.size())) 216#endif 217#ifdef COVMAP_V2_OR_V3 218COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 219 llvm::ConstantInt::get( \ 220 llvm::Type::getInt64Ty(Ctx), NameHash)) 221#endif 222COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 223 llvm::ConstantInt::get( \ 224 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) 225COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 226 llvm::ConstantInt::get( \ 227 llvm::Type::getInt64Ty(Ctx), FuncHash)) 228#ifdef COVMAP_V3 229COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ 230 llvm::ConstantInt::get( \ 231 llvm::Type::getInt64Ty(Ctx), FilenamesRef)) 232COVMAP_FUNC_RECORD(const char, \ 233 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ 234 CoverageMapping.size()), \ 235 CoverageMapping, 236 llvm::ConstantDataArray::getRaw( \ 237 CoverageMapping, CoverageMapping.size(), \ 238 llvm::Type::getInt8Ty(Ctx))) 239#endif 240#undef COVMAP_FUNC_RECORD 241/* COVMAP_FUNC_RECORD end. */ 242 243/* COVMAP_HEADER start */ 244/* Definition of member fields of coverage map header. 245 */ 246#ifndef COVMAP_HEADER 247#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 248#else 249#define INSTR_PROF_DATA_DEFINED 250#endif 251COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 252 llvm::ConstantInt::get(Int32Ty, NRecords)) 253COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 254 llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 255COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 256 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 257COVMAP_HEADER(uint32_t, Int32Ty, Version, \ 258 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 259#undef COVMAP_HEADER 260/* COVMAP_HEADER end. */ 261 262 263#ifdef INSTR_PROF_SECT_ENTRY 264#define INSTR_PROF_DATA_DEFINED 265INSTR_PROF_SECT_ENTRY(IPSK_data, \ 266 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ 267 INSTR_PROF_DATA_COFF, "__DATA,") 268INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ 269 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ 270 INSTR_PROF_CNTS_COFF, "__DATA,") 271INSTR_PROF_SECT_ENTRY(IPSK_name, \ 272 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ 273 INSTR_PROF_NAME_COFF, "__DATA,") 274INSTR_PROF_SECT_ENTRY(IPSK_vals, \ 275 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ 276 INSTR_PROF_VALS_COFF, "__DATA,") 277INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ 278 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ 279 INSTR_PROF_VNODES_COFF, "__DATA,") 280INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ 281 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ 282 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") 283INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ 284 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ 285 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,") 286INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ 287 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ 288 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") 289 290#undef INSTR_PROF_SECT_ENTRY 291#endif 292 293 294#ifdef INSTR_PROF_VALUE_PROF_DATA 295#define INSTR_PROF_DATA_DEFINED 296 297#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 298/*! 299 * This is the header of the data structure that defines the on-disk 300 * layout of the value profile data of a particular kind for one function. 301 */ 302typedef struct ValueProfRecord { 303 /* The kind of the value profile record. */ 304 uint32_t Kind; 305 /* 306 * The number of value profile sites. It is guaranteed to be non-zero; 307 * otherwise the record for this kind won't be emitted. 308 */ 309 uint32_t NumValueSites; 310 /* 311 * The first element of the array that stores the number of profiled 312 * values for each value site. The size of the array is NumValueSites. 313 * Since NumValueSites is greater than zero, there is at least one 314 * element in the array. 315 */ 316 uint8_t SiteCountArray[1]; 317 318 /* 319 * The fake declaration is for documentation purpose only. 320 * Align the start of next field to be on 8 byte boundaries. 321 uint8_t Padding[X]; 322 */ 323 324 /* The array of value profile data. The size of the array is the sum 325 * of all elements in SiteCountArray[]. 326 InstrProfValueData ValueData[]; 327 */ 328 329#ifdef __cplusplus 330 /*! 331 * Return the number of value sites. 332 */ 333 uint32_t getNumValueSites() const { return NumValueSites; } 334 /*! 335 * Read data from this record and save it to Record. 336 */ 337 void deserializeTo(InstrProfRecord &Record, 338 InstrProfSymtab *SymTab); 339 /* 340 * In-place byte swap: 341 * Do byte swap for this instance. \c Old is the original order before 342 * the swap, and \c New is the New byte order. 343 */ 344 void swapBytes(support::endianness Old, support::endianness New); 345#endif 346} ValueProfRecord; 347 348/*! 349 * Per-function header/control data structure for value profiling 350 * data in indexed format. 351 */ 352typedef struct ValueProfData { 353 /* 354 * Total size in bytes including this field. It must be a multiple 355 * of sizeof(uint64_t). 356 */ 357 uint32_t TotalSize; 358 /* 359 *The number of value profile kinds that has value profile data. 360 * In this implementation, a value profile kind is considered to 361 * have profile data if the number of value profile sites for the 362 * kind is not zero. More aggressively, the implementation can 363 * choose to check the actual data value: if none of the value sites 364 * has any profiled values, the kind can be skipped. 365 */ 366 uint32_t NumValueKinds; 367 368 /* 369 * Following are a sequence of variable length records. The prefix/header 370 * of each record is defined by ValueProfRecord type. The number of 371 * records is NumValueKinds. 372 * ValueProfRecord Record_1; 373 * ValueProfRecord Record_N; 374 */ 375 376#if __cplusplus 377 /*! 378 * Return the total size in bytes of the on-disk value profile data 379 * given the data stored in Record. 380 */ 381 static uint32_t getSize(const InstrProfRecord &Record); 382 /*! 383 * Return a pointer to \c ValueProfData instance ready to be streamed. 384 */ 385 static std::unique_ptr<ValueProfData> 386 serializeFrom(const InstrProfRecord &Record); 387 /*! 388 * Check the integrity of the record. 389 */ 390 Error checkIntegrity(); 391 /*! 392 * Return a pointer to \c ValueProfileData instance ready to be read. 393 * All data in the instance are properly byte swapped. The input 394 * data is assumed to be in little endian order. 395 */ 396 static Expected<std::unique_ptr<ValueProfData>> 397 getValueProfData(const unsigned char *SrcBuffer, 398 const unsigned char *const SrcBufferEnd, 399 support::endianness SrcDataEndianness); 400 /*! 401 * Swap byte order from \c Endianness order to host byte order. 402 */ 403 void swapBytesToHost(support::endianness Endianness); 404 /*! 405 * Swap byte order from host byte order to \c Endianness order. 406 */ 407 void swapBytesFromHost(support::endianness Endianness); 408 /*! 409 * Return the total size of \c ValueProfileData. 410 */ 411 uint32_t getSize() const { return TotalSize; } 412 /*! 413 * Read data from this data and save it to \c Record. 414 */ 415 void deserializeTo(InstrProfRecord &Record, 416 InstrProfSymtab *SymTab); 417 void operator delete(void *ptr) { ::operator delete(ptr); } 418#endif 419} ValueProfData; 420 421/* 422 * The closure is designed to abstact away two types of value profile data: 423 * - InstrProfRecord which is the primary data structure used to 424 * represent profile data in host tools (reader, writer, and profile-use) 425 * - value profile runtime data structure suitable to be used by C 426 * runtime library. 427 * 428 * Both sources of data need to serialize to disk/memory-buffer in common 429 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 430 * writer to share the same format and code with indexed profile writer. 431 * 432 * For documentation of the member methods below, refer to corresponding methods 433 * in class InstrProfRecord. 434 */ 435typedef struct ValueProfRecordClosure { 436 const void *Record; 437 uint32_t (*GetNumValueKinds)(const void *Record); 438 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 439 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 440 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 441 442 /* 443 * After extracting the value profile data from the value profile record, 444 * this method is used to map the in-memory value to on-disk value. If 445 * the method is null, value will be written out untranslated. 446 */ 447 uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 448 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 449 uint32_t S); 450 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 451} ValueProfRecordClosure; 452 453INSTR_PROF_VISIBILITY ValueProfRecord * 454getFirstValueProfRecord(ValueProfData *VPD); 455INSTR_PROF_VISIBILITY ValueProfRecord * 456getValueProfRecordNext(ValueProfRecord *VPR); 457INSTR_PROF_VISIBILITY InstrProfValueData * 458getValueProfRecordValueData(ValueProfRecord *VPR); 459INSTR_PROF_VISIBILITY uint32_t 460getValueProfRecordHeaderSize(uint32_t NumValueSites); 461 462#undef INSTR_PROF_VALUE_PROF_DATA 463#endif /* INSTR_PROF_VALUE_PROF_DATA */ 464 465 466#ifdef INSTR_PROF_COMMON_API_IMPL 467#define INSTR_PROF_DATA_DEFINED 468#ifdef __cplusplus 469#define INSTR_PROF_INLINE inline 470#define INSTR_PROF_NULLPTR nullptr 471#else 472#define INSTR_PROF_INLINE 473#define INSTR_PROF_NULLPTR NULL 474#endif 475 476#ifndef offsetof 477#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 478#endif 479 480/*! 481 * Return the \c ValueProfRecord header size including the 482 * padding bytes. 483 */ 484INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 485uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 486 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 487 sizeof(uint8_t) * NumValueSites; 488 /* Round the size to multiple of 8 bytes. */ 489 Size = (Size + 7) & ~7; 490 return Size; 491} 492 493/*! 494 * Return the total size of the value profile record including the 495 * header and the value data. 496 */ 497INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 498uint32_t getValueProfRecordSize(uint32_t NumValueSites, 499 uint32_t NumValueData) { 500 return getValueProfRecordHeaderSize(NumValueSites) + 501 sizeof(InstrProfValueData) * NumValueData; 502} 503 504/*! 505 * Return the pointer to the start of value data array. 506 */ 507INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 508InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 509 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 510 This->NumValueSites)); 511} 512 513/*! 514 * Return the total number of value data for \c This record. 515 */ 516INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 517uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 518 uint32_t NumValueData = 0; 519 uint32_t I; 520 for (I = 0; I < This->NumValueSites; I++) 521 NumValueData += This->SiteCountArray[I]; 522 return NumValueData; 523} 524 525/*! 526 * Use this method to advance to the next \c This \c ValueProfRecord. 527 */ 528INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 529ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 530 uint32_t NumValueData = getValueProfRecordNumValueData(This); 531 return (ValueProfRecord *)((char *)This + 532 getValueProfRecordSize(This->NumValueSites, 533 NumValueData)); 534} 535 536/*! 537 * Return the first \c ValueProfRecord instance. 538 */ 539INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 540ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 541 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 542} 543 544/* Closure based interfaces. */ 545 546/*! 547 * Return the total size in bytes of the on-disk value profile data 548 * given the data stored in Record. 549 */ 550INSTR_PROF_VISIBILITY uint32_t 551getValueProfDataSize(ValueProfRecordClosure *Closure) { 552 uint32_t Kind; 553 uint32_t TotalSize = sizeof(ValueProfData); 554 const void *Record = Closure->Record; 555 556 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 557 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 558 if (!NumValueSites) 559 continue; 560 TotalSize += getValueProfRecordSize(NumValueSites, 561 Closure->GetNumValueData(Record, Kind)); 562 } 563 return TotalSize; 564} 565 566/*! 567 * Extract value profile data of a function for the profile kind \c ValueKind 568 * from the \c Closure and serialize the data into \c This record instance. 569 */ 570INSTR_PROF_VISIBILITY void 571serializeValueProfRecordFrom(ValueProfRecord *This, 572 ValueProfRecordClosure *Closure, 573 uint32_t ValueKind, uint32_t NumValueSites) { 574 uint32_t S; 575 const void *Record = Closure->Record; 576 This->Kind = ValueKind; 577 This->NumValueSites = NumValueSites; 578 InstrProfValueData *DstVD = getValueProfRecordValueData(This); 579 580 for (S = 0; S < NumValueSites; S++) { 581 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 582 This->SiteCountArray[S] = ND; 583 Closure->GetValueForSite(Record, DstVD, ValueKind, S); 584 DstVD += ND; 585 } 586} 587 588/*! 589 * Extract value profile data of a function from the \c Closure 590 * and serialize the data into \c DstData if it is not NULL or heap 591 * memory allocated by the \c Closure's allocator method. If \c 592 * DstData is not null, the caller is expected to set the TotalSize 593 * in DstData. 594 */ 595INSTR_PROF_VISIBILITY ValueProfData * 596serializeValueProfDataFrom(ValueProfRecordClosure *Closure, 597 ValueProfData *DstData) { 598 uint32_t Kind; 599 uint32_t TotalSize = 600 DstData ? DstData->TotalSize : getValueProfDataSize(Closure); 601 602 ValueProfData *VPD = 603 DstData ? DstData : Closure->AllocValueProfData(TotalSize); 604 605 VPD->TotalSize = TotalSize; 606 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 607 ValueProfRecord *VR = getFirstValueProfRecord(VPD); 608 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 609 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 610 if (!NumValueSites) 611 continue; 612 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 613 VR = getValueProfRecordNext(VR); 614 } 615 return VPD; 616} 617 618#undef INSTR_PROF_COMMON_API_IMPL 619#endif /* INSTR_PROF_COMMON_API_IMPL */ 620 621/*============================================================================*/ 622 623#ifndef INSTR_PROF_DATA_DEFINED 624 625#ifndef INSTR_PROF_DATA_INC 626#define INSTR_PROF_DATA_INC 627 628/* Helper macros. */ 629#define INSTR_PROF_SIMPLE_QUOTE(x) #x 630#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 631#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 632#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 633 634/* Magic number to detect file format and endianness. 635 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 636 * so that utilities, like strings, don't grab it as a string. 129 is also 637 * invalid UTF-8, and high enough to be interesting. 638 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 639 * for 32-bit platforms. 640 */ 641#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 642 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 643 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 644#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 645 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 646 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 647 648/* Raw profile format version (start from 1). */ 649#define INSTR_PROF_RAW_VERSION 7 650/* Indexed profile format version (start from 1). */ 651#define INSTR_PROF_INDEX_VERSION 7 652/* Coverage mapping format version (start from 0). */ 653#define INSTR_PROF_COVMAP_VERSION 5 654 655/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the 656 * version for other variants of profile. We set the lowest bit of the upper 8 657 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton 658 * generated profile, and 0 if this is a Clang FE generated profile. 659 * 1 in bit 57 indicates there are context-sensitive records in the profile. 660 */ 661#define VARIANT_MASKS_ALL 0xff00000000000000ULL 662#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 663#define VARIANT_MASK_IR_PROF (0x1ULL << 56) 664#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) 665#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) 666#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version 667#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime 668#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias 669 670/* The variable that holds the name of the profile data 671 * specified via command line. */ 672#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename 673 674/* section name strings common to all targets other 675 than WIN32 */ 676#define INSTR_PROF_DATA_COMMON __llvm_prf_data 677#define INSTR_PROF_NAME_COMMON __llvm_prf_names 678#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts 679#define INSTR_PROF_VALS_COMMON __llvm_prf_vals 680#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds 681#define INSTR_PROF_COVMAP_COMMON __llvm_covmap 682#define INSTR_PROF_COVFUN_COMMON __llvm_covfun 683#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile 684/* Windows section names. Because these section names contain dollar characters, 685 * they must be quoted. 686 */ 687#define INSTR_PROF_DATA_COFF ".lprfd$M" 688#define INSTR_PROF_NAME_COFF ".lprfn$M" 689#define INSTR_PROF_CNTS_COFF ".lprfc$M" 690#define INSTR_PROF_VALS_COFF ".lprfv$M" 691#define INSTR_PROF_VNODES_COFF ".lprfnd$M" 692#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" 693#define INSTR_PROF_COVFUN_COFF ".lcovfun$M" 694#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" 695 696#ifdef _WIN32 697/* Runtime section names and name strings. */ 698#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF 699#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF 700#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF 701/* Array of pointers. Each pointer points to a list 702 * of value nodes associated with one value site. 703 */ 704#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF 705/* Value profile nodes section. */ 706#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF 707#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF 708#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF 709#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF 710#else 711/* Runtime section names and name strings. */ 712#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) 713#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) 714#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) 715/* Array of pointers. Each pointer points to a list 716 * of value nodes associated with one value site. 717 */ 718#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) 719/* Value profile nodes section. */ 720#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) 721#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) 722#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) 723/* Order file instrumentation. */ 724#define INSTR_PROF_ORDERFILE_SECT_NAME \ 725 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) 726#endif 727 728#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer 729#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ 730 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) 731#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx 732#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ 733 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) 734 735/* Macros to define start/stop section symbol for a given 736 * section on Linux. For instance 737 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 738 * expand to __start___llvm_prof_data 739 */ 740#define INSTR_PROF_SECT_START(Sect) \ 741 INSTR_PROF_CONCAT(__start_,Sect) 742#define INSTR_PROF_SECT_STOP(Sect) \ 743 INSTR_PROF_CONCAT(__stop_,Sect) 744 745/* Value Profiling API linkage name. */ 746#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 747#define INSTR_PROF_VALUE_PROF_FUNC_STR \ 748 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 749#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop 750#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ 751 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) 752 753/* InstrProfile per-function control data alignment. */ 754#define INSTR_PROF_DATA_ALIGNMENT 8 755 756/* The data structure that represents a tracked value by the 757 * value profiler. 758 */ 759typedef struct InstrProfValueData { 760 /* Profiled value. */ 761 uint64_t Value; 762 /* Number of times the value appears in the training run. */ 763 uint64_t Count; 764} InstrProfValueData; 765 766#endif /* INSTR_PROF_DATA_INC */ 767 768#ifndef INSTR_ORDER_FILE_INC 769/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ 770#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 771#define INSTR_ORDER_FILE_BUFFER_BITS 17 772#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff 773#endif /* INSTR_ORDER_FILE_INC */ 774#else 775#undef INSTR_PROF_DATA_DEFINED 776#endif 777 778#undef COVMAP_V2_OR_V3 779 780#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API 781 782#ifdef __cplusplus 783#define INSTR_PROF_INLINE inline 784#else 785#define INSTR_PROF_INLINE 786#endif 787 788/* The value range buckets (22 buckets) for the memop size value profiling looks 789 * like: 790 * 791 * [0, 0] 792 * [1, 1] 793 * [2, 2] 794 * [3, 3] 795 * [4, 4] 796 * [5, 5] 797 * [6, 6] 798 * [7, 7] 799 * [8, 8] 800 * [9, 15] 801 * [16, 16] 802 * [17, 31] 803 * [32, 32] 804 * [33, 63] 805 * [64, 64] 806 * [65, 127] 807 * [128, 128] 808 * [129, 255] 809 * [256, 256] 810 * [257, 511] 811 * [512, 512] 812 * [513, UINT64_MAX] 813 * 814 * Each range has a 'representative value' which is the lower end value of the 815 * range and used to store in the runtime profile data records and the VP 816 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. 817 */ 818#define INSTR_PROF_NUM_BUCKETS 22 819 820/* 821 * Clz and Popcount. This code was copied from 822 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and 823 * llvm/include/llvm/Support/MathExtras.h. 824 */ 825#if defined(_MSC_VER) && !defined(__clang__) 826 827#include <intrin.h> 828INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 829int InstProfClzll(unsigned long long X) { 830 unsigned long LeadZeroIdx = 0; 831#if !defined(_M_ARM64) && !defined(_M_X64) 832 // Scan the high 32 bits. 833 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) 834 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset 835 // from the MSB. 836 // Scan the low 32 bits. 837 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) 838 return (int)(63 - LeadZeroIdx); 839#else 840 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; 841#endif 842 return 64; 843} 844INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 845int InstProfPopcountll(unsigned long long X) { 846 // This code originates from https://reviews.llvm.org/rG30626254510f. 847 unsigned long long v = X; 848 v = v - ((v >> 1) & 0x5555555555555555ULL); 849 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); 850 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; 851 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); 852} 853 854#else 855 856INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 857int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } 858INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 859int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } 860 861#endif /* defined(_MSC_VER) && !defined(__clang__) */ 862 863/* Map an (observed) memop size value to the representative value of its range. 864 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ 865INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t 866InstrProfGetRangeRepValue(uint64_t Value) { 867 if (Value <= 8) 868 // The first ranges are individually tracked. Use the value as is. 869 return Value; 870 else if (Value >= 513) 871 // The last range is mapped to its lowest value. 872 return 513; 873 else if (InstProfPopcountll(Value) == 1) 874 // If it's a power of two, use it as is. 875 return Value; 876 else 877 // Otherwise, take to the previous power of two + 1. 878 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1; 879} 880 881/* Return true if the range that an (observed) memop size value belongs to has 882 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> 883 * false, 64 -> true, 100 -> false, 513 -> false. */ 884INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned 885InstrProfIsSingleValRange(uint64_t Value) { 886 if (Value <= 8) 887 // The first ranges are individually tracked. 888 return 1; 889 else if (InstProfPopcountll(Value) == 1) 890 // If it's a power of two, there's only one value. 891 return 1; 892 else 893 // Otherwise, there's more than one value in the range. 894 return 0; 895} 896 897#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ 898