1 //===- SampleProfWriter.h - Write LLVM sample profile data ------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains definitions needed for writing sample profiles. 10 // 11 //===----------------------------------------------------------------------===// 12 #ifndef LLVM_PROFILEDATA_SAMPLEPROFWRITER_H 13 #define LLVM_PROFILEDATA_SAMPLEPROFWRITER_H 14 15 #include "llvm/ADT/MapVector.h" 16 #include "llvm/ADT/StringRef.h" 17 #include "llvm/IR/ProfileSummary.h" 18 #include "llvm/ProfileData/SampleProf.h" 19 #include "llvm/Support/ErrorOr.h" 20 #include "llvm/Support/raw_ostream.h" 21 #include <cstdint> 22 #include <memory> 23 #include <set> 24 #include <system_error> 25 26 namespace llvm { 27 namespace sampleprof { 28 29 enum SectionLayout { 30 DefaultLayout, 31 // The layout splits profile with context information from profile without 32 // context information. When Thinlto is enabled, ThinLTO postlink phase only 33 // has to load profile with context information and can skip the other part. 34 CtxSplitLayout, 35 NumOfLayout, 36 }; 37 38 /// Sample-based profile writer. Base class. 39 class SampleProfileWriter { 40 public: 41 virtual ~SampleProfileWriter() = default; 42 43 /// Write sample profiles in \p S. 44 /// 45 /// \returns status code of the file update operation. 46 virtual std::error_code writeSample(const FunctionSamples &S) = 0; 47 48 /// Write all the sample profiles in the given map of samples. 49 /// 50 /// \returns status code of the file update operation. 51 virtual std::error_code write(const SampleProfileMap &ProfileMap); 52 53 raw_ostream &getOutputStream() { return *OutputStream; } 54 55 /// Profile writer factory. 56 /// 57 /// Create a new file writer based on the value of \p Format. 58 static ErrorOr<std::unique_ptr<SampleProfileWriter>> 59 create(StringRef Filename, SampleProfileFormat Format); 60 61 /// Create a new stream writer based on the value of \p Format. 62 /// For testing. 63 static ErrorOr<std::unique_ptr<SampleProfileWriter>> 64 create(std::unique_ptr<raw_ostream> &OS, SampleProfileFormat Format); 65 66 virtual void setProfileSymbolList(ProfileSymbolList *PSL) {} 67 virtual void setToCompressAllSections() {} 68 virtual void setUseMD5() {} 69 virtual void setPartialProfile() {} 70 virtual void resetSecLayout(SectionLayout SL) {} 71 72 protected: 73 SampleProfileWriter(std::unique_ptr<raw_ostream> &OS) 74 : OutputStream(std::move(OS)) {} 75 76 /// Write a file header for the profile file. 77 virtual std::error_code writeHeader(const SampleProfileMap &ProfileMap) = 0; 78 79 // Write function profiles to the profile file. 80 virtual std::error_code writeFuncProfiles(const SampleProfileMap &ProfileMap); 81 82 /// Output stream where to emit the profile to. 83 std::unique_ptr<raw_ostream> OutputStream; 84 85 /// Profile summary. 86 std::unique_ptr<ProfileSummary> Summary; 87 88 /// Compute summary for this profile. 89 void computeSummary(const SampleProfileMap &ProfileMap); 90 91 /// Profile format. 92 SampleProfileFormat Format = SPF_None; 93 }; 94 95 /// Sample-based profile writer (text format). 96 class SampleProfileWriterText : public SampleProfileWriter { 97 public: 98 std::error_code writeSample(const FunctionSamples &S) override; 99 100 protected: 101 SampleProfileWriterText(std::unique_ptr<raw_ostream> &OS) 102 : SampleProfileWriter(OS), Indent(0) {} 103 104 std::error_code writeHeader(const SampleProfileMap &ProfileMap) override { 105 return sampleprof_error::success; 106 } 107 108 private: 109 /// Indent level to use when writing. 110 /// 111 /// This is used when printing inlined callees. 112 unsigned Indent; 113 114 friend ErrorOr<std::unique_ptr<SampleProfileWriter>> 115 SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, 116 SampleProfileFormat Format); 117 }; 118 119 /// Sample-based profile writer (binary format). 120 class SampleProfileWriterBinary : public SampleProfileWriter { 121 public: 122 SampleProfileWriterBinary(std::unique_ptr<raw_ostream> &OS) 123 : SampleProfileWriter(OS) {} 124 125 virtual std::error_code writeSample(const FunctionSamples &S) override; 126 127 protected: 128 virtual MapVector<StringRef, uint32_t> &getNameTable() { return NameTable; } 129 virtual std::error_code writeMagicIdent(SampleProfileFormat Format); 130 virtual std::error_code writeNameTable(); 131 virtual std::error_code 132 writeHeader(const SampleProfileMap &ProfileMap) override; 133 std::error_code writeSummary(); 134 virtual std::error_code writeContextIdx(const SampleContext &Context); 135 std::error_code writeNameIdx(StringRef FName); 136 std::error_code writeBody(const FunctionSamples &S); 137 inline void stablizeNameTable(MapVector<StringRef, uint32_t> &NameTable, 138 std::set<StringRef> &V); 139 140 MapVector<StringRef, uint32_t> NameTable; 141 142 void addName(StringRef FName); 143 virtual void addContext(const SampleContext &Context); 144 void addNames(const FunctionSamples &S); 145 146 private: 147 friend ErrorOr<std::unique_ptr<SampleProfileWriter>> 148 SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, 149 SampleProfileFormat Format); 150 }; 151 152 class SampleProfileWriterRawBinary : public SampleProfileWriterBinary { 153 using SampleProfileWriterBinary::SampleProfileWriterBinary; 154 }; 155 156 const std::array<SmallVector<SecHdrTableEntry, 8>, NumOfLayout> 157 ExtBinaryHdrLayoutTable = { 158 // Note that SecFuncOffsetTable section is written after SecLBRProfile 159 // in the profile, but is put before SecLBRProfile in SectionHdrLayout. 160 // This is because sample reader follows the order in SectionHdrLayout 161 // to read each section. To read function profiles on demand, sample 162 // reader need to get the offset of each function profile first. 163 // 164 // DefaultLayout 165 SmallVector<SecHdrTableEntry, 8>({{SecProfSummary, 0, 0, 0, 0}, 166 {SecNameTable, 0, 0, 0, 0}, 167 {SecCSNameTable, 0, 0, 0, 0}, 168 {SecFuncOffsetTable, 0, 0, 0, 0}, 169 {SecLBRProfile, 0, 0, 0, 0}, 170 {SecProfileSymbolList, 0, 0, 0, 0}, 171 {SecFuncMetadata, 0, 0, 0, 0}}), 172 // CtxSplitLayout 173 SmallVector<SecHdrTableEntry, 8>({{SecProfSummary, 0, 0, 0, 0}, 174 {SecNameTable, 0, 0, 0, 0}, 175 // profile with context 176 // for next two sections 177 {SecFuncOffsetTable, 0, 0, 0, 0}, 178 {SecLBRProfile, 0, 0, 0, 0}, 179 // profile without context 180 // for next two sections 181 {SecFuncOffsetTable, 0, 0, 0, 0}, 182 {SecLBRProfile, 0, 0, 0, 0}, 183 {SecProfileSymbolList, 0, 0, 0, 0}, 184 {SecFuncMetadata, 0, 0, 0, 0}}), 185 }; 186 187 class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary { 188 using SampleProfileWriterBinary::SampleProfileWriterBinary; 189 public: 190 virtual std::error_code write(const SampleProfileMap &ProfileMap) override; 191 192 virtual void setToCompressAllSections() override; 193 void setToCompressSection(SecType Type); 194 virtual std::error_code writeSample(const FunctionSamples &S) override; 195 196 // Set to use MD5 to represent string in NameTable. 197 virtual void setUseMD5() override { 198 UseMD5 = true; 199 addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagMD5Name); 200 // MD5 will be stored as plain uint64_t instead of variable-length 201 // quantity format in NameTable section. 202 addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagFixedLengthMD5); 203 } 204 205 // Set the profile to be partial. It means the profile is for 206 // common/shared code. The common profile is usually merged from 207 // profiles collected from running other targets. 208 virtual void setPartialProfile() override { 209 addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagPartial); 210 } 211 212 virtual void setProfileSymbolList(ProfileSymbolList *PSL) override { 213 ProfSymList = PSL; 214 }; 215 216 virtual void resetSecLayout(SectionLayout SL) override { 217 verifySecLayout(SL); 218 #ifndef NDEBUG 219 // Make sure resetSecLayout is called before any flag setting. 220 for (auto &Entry : SectionHdrLayout) { 221 assert(Entry.Flags == 0 && 222 "resetSecLayout has to be called before any flag setting"); 223 } 224 #endif 225 SecLayout = SL; 226 SectionHdrLayout = ExtBinaryHdrLayoutTable[SL]; 227 } 228 229 protected: 230 uint64_t markSectionStart(SecType Type, uint32_t LayoutIdx); 231 std::error_code addNewSection(SecType Sec, uint32_t LayoutIdx, 232 uint64_t SectionStart); 233 template <class SecFlagType> 234 void addSectionFlag(SecType Type, SecFlagType Flag) { 235 for (auto &Entry : SectionHdrLayout) { 236 if (Entry.Type == Type) 237 addSecFlag(Entry, Flag); 238 } 239 } 240 template <class SecFlagType> 241 void addSectionFlag(uint32_t SectionIdx, SecFlagType Flag) { 242 addSecFlag(SectionHdrLayout[SectionIdx], Flag); 243 } 244 245 virtual void addContext(const SampleContext &Context) override; 246 247 // placeholder for subclasses to dispatch their own section writers. 248 virtual std::error_code writeCustomSection(SecType Type) = 0; 249 // Verify the SecLayout is supported by the format. 250 virtual void verifySecLayout(SectionLayout SL) = 0; 251 252 // specify the order to write sections. 253 virtual std::error_code writeSections(const SampleProfileMap &ProfileMap) = 0; 254 255 // Dispatch section writer for each section. \p LayoutIdx is the sequence 256 // number indicating where the section is located in SectionHdrLayout. 257 virtual std::error_code writeOneSection(SecType Type, uint32_t LayoutIdx, 258 const SampleProfileMap &ProfileMap); 259 260 // Helper function to write name table. 261 virtual std::error_code writeNameTable() override; 262 virtual std::error_code 263 writeContextIdx(const SampleContext &Context) override; 264 std::error_code writeCSNameIdx(const SampleContext &Context); 265 std::error_code writeCSNameTableSection(); 266 267 std::error_code writeFuncMetadata(const SampleProfileMap &Profiles); 268 std::error_code writeFuncMetadata(const FunctionSamples &Profile); 269 270 // Functions to write various kinds of sections. 271 std::error_code writeNameTableSection(const SampleProfileMap &ProfileMap); 272 std::error_code writeFuncOffsetTable(); 273 std::error_code writeProfileSymbolListSection(); 274 275 SectionLayout SecLayout = DefaultLayout; 276 // Specifiy the order of sections in section header table. Note 277 // the order of sections in SecHdrTable may be different that the 278 // order in SectionHdrLayout. sample Reader will follow the order 279 // in SectionHdrLayout to read each section. 280 SmallVector<SecHdrTableEntry, 8> SectionHdrLayout = 281 ExtBinaryHdrLayoutTable[DefaultLayout]; 282 283 // Save the start of SecLBRProfile so we can compute the offset to the 284 // start of SecLBRProfile for each Function's Profile and will keep it 285 // in FuncOffsetTable. 286 uint64_t SecLBRProfileStart = 0; 287 288 private: 289 void allocSecHdrTable(); 290 std::error_code writeSecHdrTable(); 291 virtual std::error_code 292 writeHeader(const SampleProfileMap &ProfileMap) override; 293 std::error_code compressAndOutput(); 294 295 // We will swap the raw_ostream held by LocalBufStream and that 296 // held by OutputStream if we try to add a section which needs 297 // compression. After the swap, all the data written to output 298 // will be temporarily buffered into the underlying raw_string_ostream 299 // originally held by LocalBufStream. After the data writing for the 300 // section is completed, compress the data in the local buffer, 301 // swap the raw_ostream back and write the compressed data to the 302 // real output. 303 std::unique_ptr<raw_ostream> LocalBufStream; 304 // The location where the output stream starts. 305 uint64_t FileStart; 306 // The location in the output stream where the SecHdrTable should be 307 // written to. 308 uint64_t SecHdrTableOffset; 309 // The table contains SecHdrTableEntry entries in order of how they are 310 // populated in the writer. It may be different from the order in 311 // SectionHdrLayout which specifies the sequence in which sections will 312 // be read. 313 std::vector<SecHdrTableEntry> SecHdrTable; 314 315 // FuncOffsetTable maps function context to its profile offset in 316 // SecLBRProfile section. It is used to load function profile on demand. 317 MapVector<SampleContext, uint64_t> FuncOffsetTable; 318 // Whether to use MD5 to represent string. 319 bool UseMD5 = false; 320 321 /// CSNameTable maps function context to its offset in SecCSNameTable section. 322 /// The offset will be used everywhere where the context is referenced. 323 MapVector<SampleContext, uint32_t> CSNameTable; 324 325 ProfileSymbolList *ProfSymList = nullptr; 326 }; 327 328 class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase { 329 public: 330 SampleProfileWriterExtBinary(std::unique_ptr<raw_ostream> &OS) 331 : SampleProfileWriterExtBinaryBase(OS) {} 332 333 private: 334 std::error_code writeDefaultLayout(const SampleProfileMap &ProfileMap); 335 std::error_code writeCtxSplitLayout(const SampleProfileMap &ProfileMap); 336 337 virtual std::error_code 338 writeSections(const SampleProfileMap &ProfileMap) override; 339 340 virtual std::error_code writeCustomSection(SecType Type) override { 341 return sampleprof_error::success; 342 }; 343 344 virtual void verifySecLayout(SectionLayout SL) override { 345 assert((SL == DefaultLayout || SL == CtxSplitLayout) && 346 "Unsupported layout"); 347 } 348 }; 349 350 // CompactBinary is a compact format of binary profile which both reduces 351 // the profile size and the load time needed when compiling. It has two 352 // major difference with Binary format. 353 // 1. It represents all the strings in name table using md5 hash. 354 // 2. It saves a function offset table which maps function name index to 355 // the offset of its function profile to the start of the binary profile, 356 // so by using the function offset table, for those function profiles which 357 // will not be needed when compiling a module, the profile reader does't 358 // have to read them and it saves compile time if the profile size is huge. 359 // The layout of the compact format is shown as follows: 360 // 361 // Part1: Profile header, the same as binary format, containing magic 362 // number, version, summary, name table... 363 // Part2: Function Offset Table Offset, which saves the position of 364 // Part4. 365 // Part3: Function profile collection 366 // function1 profile start 367 // .... 368 // function2 profile start 369 // .... 370 // function3 profile start 371 // .... 372 // ...... 373 // Part4: Function Offset Table 374 // function1 name index --> function1 profile start 375 // function2 name index --> function2 profile start 376 // function3 name index --> function3 profile start 377 // 378 // We need Part2 because profile reader can use it to find out and read 379 // function offset table without reading Part3 first. 380 class SampleProfileWriterCompactBinary : public SampleProfileWriterBinary { 381 using SampleProfileWriterBinary::SampleProfileWriterBinary; 382 383 public: 384 virtual std::error_code writeSample(const FunctionSamples &S) override; 385 virtual std::error_code write(const SampleProfileMap &ProfileMap) override; 386 387 protected: 388 /// The table mapping from function name to the offset of its FunctionSample 389 /// towards profile start. 390 MapVector<StringRef, uint64_t> FuncOffsetTable; 391 /// The offset of the slot to be filled with the offset of FuncOffsetTable 392 /// towards profile start. 393 uint64_t TableOffset; 394 virtual std::error_code writeNameTable() override; 395 virtual std::error_code 396 writeHeader(const SampleProfileMap &ProfileMap) override; 397 std::error_code writeFuncOffsetTable(); 398 }; 399 400 } // end namespace sampleprof 401 } // end namespace llvm 402 403 #endif // LLVM_PROFILEDATA_SAMPLEPROFWRITER_H 404