1 //===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class that writes LLVM sample profiles. It
10 // supports two file formats: text and binary. The textual representation
11 // is useful for debugging and testing purposes. The binary representation
12 // is more compact, resulting in smaller file sizes. However, they can
13 // both be used interchangeably.
14 //
15 // See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16 // supported formats.
17 //
18 //===----------------------------------------------------------------------===//
19
20 #include "llvm/ProfileData/SampleProfWriter.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/ADT/StringSet.h"
23 #include "llvm/ProfileData/ProfileCommon.h"
24 #include "llvm/ProfileData/SampleProf.h"
25 #include "llvm/Support/Compression.h"
26 #include "llvm/Support/Endian.h"
27 #include "llvm/Support/EndianStream.h"
28 #include "llvm/Support/ErrorOr.h"
29 #include "llvm/Support/FileSystem.h"
30 #include "llvm/Support/LEB128.h"
31 #include "llvm/Support/MD5.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include <algorithm>
34 #include <cstdint>
35 #include <memory>
36 #include <set>
37 #include <system_error>
38 #include <utility>
39 #include <vector>
40
41 using namespace llvm;
42 using namespace sampleprof;
43
writeFuncProfiles(const StringMap<FunctionSamples> & ProfileMap)44 std::error_code SampleProfileWriter::writeFuncProfiles(
45 const StringMap<FunctionSamples> &ProfileMap) {
46 // Sort the ProfileMap by total samples.
47 typedef std::pair<StringRef, const FunctionSamples *> NameFunctionSamples;
48 std::vector<NameFunctionSamples> V;
49 for (const auto &I : ProfileMap) {
50 assert(I.getKey() == I.second.getNameWithContext() &&
51 "Inconsistent profile map");
52 V.push_back(std::make_pair(I.second.getNameWithContext(), &I.second));
53 }
54 llvm::stable_sort(
55 V, [](const NameFunctionSamples &A, const NameFunctionSamples &B) {
56 if (A.second->getTotalSamples() == B.second->getTotalSamples())
57 return A.first > B.first;
58 return A.second->getTotalSamples() > B.second->getTotalSamples();
59 });
60
61 for (const auto &I : V) {
62 if (std::error_code EC = writeSample(*I.second))
63 return EC;
64 }
65 return sampleprof_error::success;
66 }
67
68 std::error_code
write(const StringMap<FunctionSamples> & ProfileMap)69 SampleProfileWriter::write(const StringMap<FunctionSamples> &ProfileMap) {
70 if (std::error_code EC = writeHeader(ProfileMap))
71 return EC;
72
73 if (std::error_code EC = writeFuncProfiles(ProfileMap))
74 return EC;
75
76 return sampleprof_error::success;
77 }
78
79 /// Return the current position and prepare to use it as the start
80 /// position of a section given the section type \p Type and its position
81 /// \p LayoutIdx in SectionHdrLayout.
82 uint64_t
markSectionStart(SecType Type,uint32_t LayoutIdx)83 SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,
84 uint32_t LayoutIdx) {
85 uint64_t SectionStart = OutputStream->tell();
86 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
87 const auto &Entry = SectionHdrLayout[LayoutIdx];
88 assert(Entry.Type == Type && "Unexpected section type");
89 // Use LocalBuf as a temporary output for writting data.
90 if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress))
91 LocalBufStream.swap(OutputStream);
92 return SectionStart;
93 }
94
compressAndOutput()95 std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
96 if (!llvm::zlib::isAvailable())
97 return sampleprof_error::zlib_unavailable;
98 std::string &UncompressedStrings =
99 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
100 if (UncompressedStrings.size() == 0)
101 return sampleprof_error::success;
102 auto &OS = *OutputStream;
103 SmallString<128> CompressedStrings;
104 llvm::Error E = zlib::compress(UncompressedStrings, CompressedStrings,
105 zlib::BestSizeCompression);
106 if (E)
107 return sampleprof_error::compress_failed;
108 encodeULEB128(UncompressedStrings.size(), OS);
109 encodeULEB128(CompressedStrings.size(), OS);
110 OS << CompressedStrings.str();
111 UncompressedStrings.clear();
112 return sampleprof_error::success;
113 }
114
115 /// Add a new section into section header table given the section type
116 /// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
117 /// location \p SectionStart where the section should be written to.
addNewSection(SecType Type,uint32_t LayoutIdx,uint64_t SectionStart)118 std::error_code SampleProfileWriterExtBinaryBase::addNewSection(
119 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
120 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
121 const auto &Entry = SectionHdrLayout[LayoutIdx];
122 assert(Entry.Type == Type && "Unexpected section type");
123 if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress)) {
124 LocalBufStream.swap(OutputStream);
125 if (std::error_code EC = compressAndOutput())
126 return EC;
127 }
128 SecHdrTable.push_back({Type, Entry.Flags, SectionStart - FileStart,
129 OutputStream->tell() - SectionStart, LayoutIdx});
130 return sampleprof_error::success;
131 }
132
write(const StringMap<FunctionSamples> & ProfileMap)133 std::error_code SampleProfileWriterExtBinaryBase::write(
134 const StringMap<FunctionSamples> &ProfileMap) {
135 if (std::error_code EC = writeHeader(ProfileMap))
136 return EC;
137
138 std::string LocalBuf;
139 LocalBufStream = std::make_unique<raw_string_ostream>(LocalBuf);
140 if (std::error_code EC = writeSections(ProfileMap))
141 return EC;
142
143 if (std::error_code EC = writeSecHdrTable())
144 return EC;
145
146 return sampleprof_error::success;
147 }
148
149 std::error_code
writeSample(const FunctionSamples & S)150 SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
151 uint64_t Offset = OutputStream->tell();
152 StringRef Name = S.getNameWithContext();
153 FuncOffsetTable[Name] = Offset - SecLBRProfileStart;
154 encodeULEB128(S.getHeadSamples(), *OutputStream);
155 return writeBody(S);
156 }
157
writeFuncOffsetTable()158 std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
159 auto &OS = *OutputStream;
160
161 // Write out the table size.
162 encodeULEB128(FuncOffsetTable.size(), OS);
163
164 // Write out FuncOffsetTable.
165 for (auto Entry : FuncOffsetTable) {
166 if (std::error_code EC =
167 writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS))
168 return EC;
169 encodeULEB128(Entry.second, OS);
170 }
171 FuncOffsetTable.clear();
172 return sampleprof_error::success;
173 }
174
writeFuncMetadata(const StringMap<FunctionSamples> & Profiles)175 std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
176 const StringMap<FunctionSamples> &Profiles) {
177 if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS)
178 return sampleprof_error::success;
179 auto &OS = *OutputStream;
180 for (const auto &Entry : Profiles) {
181 if (std::error_code EC = writeNameIdx(Entry.second.getNameWithContext(),
182 FunctionSamples::ProfileIsCS))
183 return EC;
184 if (FunctionSamples::ProfileIsProbeBased)
185 encodeULEB128(Entry.second.getFunctionHash(), OS);
186 if (FunctionSamples::ProfileIsCS)
187 encodeULEB128(Entry.second.getContext().getAllAttributes(), OS);
188 }
189 return sampleprof_error::success;
190 }
191
writeNameTable()192 std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
193 if (!UseMD5)
194 return SampleProfileWriterBinary::writeNameTable();
195
196 auto &OS = *OutputStream;
197 std::set<StringRef> V;
198 stablizeNameTable(V);
199
200 // Write out the MD5 name table. We wrote unencoded MD5 so reader can
201 // retrieve the name using the name index without having to read the
202 // whole name table.
203 encodeULEB128(NameTable.size(), OS);
204 support::endian::Writer Writer(OS, support::little);
205 for (auto N : V)
206 Writer.write(MD5Hash(N));
207 return sampleprof_error::success;
208 }
209
writeNameTableSection(const StringMap<FunctionSamples> & ProfileMap)210 std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
211 const StringMap<FunctionSamples> &ProfileMap) {
212 for (const auto &I : ProfileMap) {
213 assert(I.first() == I.second.getNameWithContext() &&
214 "Inconsistent profile map");
215 addName(I.second.getNameWithContext(), FunctionSamples::ProfileIsCS);
216 addNames(I.second);
217 }
218
219 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
220 // so compiler won't strip the suffix during profile matching after
221 // seeing the flag in the profile.
222 for (const auto &I : NameTable) {
223 if (I.first.find(FunctionSamples::UniqSuffix) != StringRef::npos) {
224 addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix);
225 break;
226 }
227 }
228
229 if (auto EC = writeNameTable())
230 return EC;
231 return sampleprof_error::success;
232 }
233
234 std::error_code
writeProfileSymbolListSection()235 SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {
236 if (ProfSymList && ProfSymList->size() > 0)
237 if (std::error_code EC = ProfSymList->write(*OutputStream))
238 return EC;
239
240 return sampleprof_error::success;
241 }
242
writeOneSection(SecType Type,uint32_t LayoutIdx,const StringMap<FunctionSamples> & ProfileMap)243 std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
244 SecType Type, uint32_t LayoutIdx,
245 const StringMap<FunctionSamples> &ProfileMap) {
246 // The setting of SecFlagCompress should happen before markSectionStart.
247 if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())
248 setToCompressSection(SecProfileSymbolList);
249 if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
250 addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased);
251 if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
252 addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFullContext);
253 if (Type == SecFuncMetadata && FunctionSamples::ProfileIsCS)
254 addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagHasAttribute);
255 if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
256 addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFSDiscriminator);
257
258 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
259 switch (Type) {
260 case SecProfSummary:
261 computeSummary(ProfileMap);
262 if (auto EC = writeSummary())
263 return EC;
264 break;
265 case SecNameTable:
266 if (auto EC = writeNameTableSection(ProfileMap))
267 return EC;
268 break;
269 case SecLBRProfile:
270 SecLBRProfileStart = OutputStream->tell();
271 if (std::error_code EC = writeFuncProfiles(ProfileMap))
272 return EC;
273 break;
274 case SecFuncOffsetTable:
275 if (auto EC = writeFuncOffsetTable())
276 return EC;
277 break;
278 case SecFuncMetadata:
279 if (std::error_code EC = writeFuncMetadata(ProfileMap))
280 return EC;
281 break;
282 case SecProfileSymbolList:
283 if (auto EC = writeProfileSymbolListSection())
284 return EC;
285 break;
286 default:
287 if (auto EC = writeCustomSection(Type))
288 return EC;
289 break;
290 }
291 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
292 return EC;
293 return sampleprof_error::success;
294 }
295
writeDefaultLayout(const StringMap<FunctionSamples> & ProfileMap)296 std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
297 const StringMap<FunctionSamples> &ProfileMap) {
298 // The const indices passed to writeOneSection below are specifying the
299 // positions of the sections in SectionHdrLayout. Look at
300 // initSectionHdrLayout to find out where each section is located in
301 // SectionHdrLayout.
302 if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))
303 return EC;
304 if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))
305 return EC;
306 if (auto EC = writeOneSection(SecLBRProfile, 3, ProfileMap))
307 return EC;
308 if (auto EC = writeOneSection(SecProfileSymbolList, 4, ProfileMap))
309 return EC;
310 if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ProfileMap))
311 return EC;
312 if (auto EC = writeOneSection(SecFuncMetadata, 5, ProfileMap))
313 return EC;
314 return sampleprof_error::success;
315 }
316
317 static void
splitProfileMapToTwo(const StringMap<FunctionSamples> & ProfileMap,StringMap<FunctionSamples> & ContextProfileMap,StringMap<FunctionSamples> & NoContextProfileMap)318 splitProfileMapToTwo(const StringMap<FunctionSamples> &ProfileMap,
319 StringMap<FunctionSamples> &ContextProfileMap,
320 StringMap<FunctionSamples> &NoContextProfileMap) {
321 for (const auto &I : ProfileMap) {
322 if (I.second.getCallsiteSamples().size())
323 ContextProfileMap.insert({I.first(), I.second});
324 else
325 NoContextProfileMap.insert({I.first(), I.second});
326 }
327 }
328
writeCtxSplitLayout(const StringMap<FunctionSamples> & ProfileMap)329 std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
330 const StringMap<FunctionSamples> &ProfileMap) {
331 StringMap<FunctionSamples> ContextProfileMap, NoContextProfileMap;
332 splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap);
333
334 if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))
335 return EC;
336 if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))
337 return EC;
338 if (auto EC = writeOneSection(SecLBRProfile, 3, ContextProfileMap))
339 return EC;
340 if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ContextProfileMap))
341 return EC;
342 // Mark the section to have no context. Note section flag needs to be set
343 // before writing the section.
344 addSectionFlag(5, SecCommonFlags::SecFlagFlat);
345 if (auto EC = writeOneSection(SecLBRProfile, 5, NoContextProfileMap))
346 return EC;
347 // Mark the section to have no context. Note section flag needs to be set
348 // before writing the section.
349 addSectionFlag(4, SecCommonFlags::SecFlagFlat);
350 if (auto EC = writeOneSection(SecFuncOffsetTable, 4, NoContextProfileMap))
351 return EC;
352 if (auto EC = writeOneSection(SecProfileSymbolList, 6, ProfileMap))
353 return EC;
354 if (auto EC = writeOneSection(SecFuncMetadata, 7, ProfileMap))
355 return EC;
356
357 return sampleprof_error::success;
358 }
359
writeSections(const StringMap<FunctionSamples> & ProfileMap)360 std::error_code SampleProfileWriterExtBinary::writeSections(
361 const StringMap<FunctionSamples> &ProfileMap) {
362 std::error_code EC;
363 if (SecLayout == DefaultLayout)
364 EC = writeDefaultLayout(ProfileMap);
365 else if (SecLayout == CtxSplitLayout)
366 EC = writeCtxSplitLayout(ProfileMap);
367 else
368 llvm_unreachable("Unsupported layout");
369 return EC;
370 }
371
write(const StringMap<FunctionSamples> & ProfileMap)372 std::error_code SampleProfileWriterCompactBinary::write(
373 const StringMap<FunctionSamples> &ProfileMap) {
374 if (std::error_code EC = SampleProfileWriter::write(ProfileMap))
375 return EC;
376 if (std::error_code EC = writeFuncOffsetTable())
377 return EC;
378 return sampleprof_error::success;
379 }
380
381 /// Write samples to a text file.
382 ///
383 /// Note: it may be tempting to implement this in terms of
384 /// FunctionSamples::print(). Please don't. The dump functionality is intended
385 /// for debugging and has no specified form.
386 ///
387 /// The format used here is more structured and deliberate because
388 /// it needs to be parsed by the SampleProfileReaderText class.
writeSample(const FunctionSamples & S)389 std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
390 auto &OS = *OutputStream;
391 if (FunctionSamples::ProfileIsCS)
392 OS << "[" << S.getNameWithContext() << "]:" << S.getTotalSamples();
393 else
394 OS << S.getName() << ":" << S.getTotalSamples();
395
396 if (Indent == 0)
397 OS << ":" << S.getHeadSamples();
398 OS << "\n";
399
400 SampleSorter<LineLocation, SampleRecord> SortedSamples(S.getBodySamples());
401 for (const auto &I : SortedSamples.get()) {
402 LineLocation Loc = I->first;
403 const SampleRecord &Sample = I->second;
404 OS.indent(Indent + 1);
405 if (Loc.Discriminator == 0)
406 OS << Loc.LineOffset << ": ";
407 else
408 OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
409
410 OS << Sample.getSamples();
411
412 for (const auto &J : Sample.getSortedCallTargets())
413 OS << " " << J.first << ":" << J.second;
414 OS << "\n";
415 }
416
417 SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
418 S.getCallsiteSamples());
419 Indent += 1;
420 for (const auto &I : SortedCallsiteSamples.get())
421 for (const auto &FS : I->second) {
422 LineLocation Loc = I->first;
423 const FunctionSamples &CalleeSamples = FS.second;
424 OS.indent(Indent);
425 if (Loc.Discriminator == 0)
426 OS << Loc.LineOffset << ": ";
427 else
428 OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
429 if (std::error_code EC = writeSample(CalleeSamples))
430 return EC;
431 }
432 Indent -= 1;
433
434 if (Indent == 0) {
435 if (FunctionSamples::ProfileIsProbeBased) {
436 OS.indent(Indent + 1);
437 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
438 }
439 if (FunctionSamples::ProfileIsCS) {
440 OS.indent(Indent + 1);
441 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
442 }
443 }
444
445 return sampleprof_error::success;
446 }
447
writeNameIdx(StringRef FName,bool IsContextName)448 std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName,
449 bool IsContextName) {
450 std::string BracketedName;
451 if (IsContextName) {
452 BracketedName = "[" + FName.str() + "]";
453 FName = StringRef(BracketedName);
454 }
455
456 const auto &Ret = NameTable.find(FName);
457 if (Ret == NameTable.end())
458 return sampleprof_error::truncated_name_table;
459 encodeULEB128(Ret->second, *OutputStream);
460 return sampleprof_error::success;
461 }
462
addName(StringRef FName,bool IsContextName)463 void SampleProfileWriterBinary::addName(StringRef FName, bool IsContextName) {
464 if (IsContextName) {
465 auto It = BracketedContextStr.insert("[" + FName.str() + "]");
466 FName = StringRef(*It.first);
467 }
468 NameTable.insert(std::make_pair(FName, 0));
469 }
470
addNames(const FunctionSamples & S)471 void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
472 // Add all the names in indirect call targets.
473 for (const auto &I : S.getBodySamples()) {
474 const SampleRecord &Sample = I.second;
475 for (const auto &J : Sample.getCallTargets())
476 addName(J.first());
477 }
478
479 // Recursively add all the names for inlined callsites.
480 for (const auto &J : S.getCallsiteSamples())
481 for (const auto &FS : J.second) {
482 const FunctionSamples &CalleeSamples = FS.second;
483 addName(CalleeSamples.getName());
484 addNames(CalleeSamples);
485 }
486 }
487
stablizeNameTable(std::set<StringRef> & V)488 void SampleProfileWriterBinary::stablizeNameTable(std::set<StringRef> &V) {
489 // Sort the names to make NameTable deterministic.
490 for (const auto &I : NameTable)
491 V.insert(I.first);
492 int i = 0;
493 for (const StringRef &N : V)
494 NameTable[N] = i++;
495 }
496
writeNameTable()497 std::error_code SampleProfileWriterBinary::writeNameTable() {
498 auto &OS = *OutputStream;
499 std::set<StringRef> V;
500 stablizeNameTable(V);
501
502 // Write out the name table.
503 encodeULEB128(NameTable.size(), OS);
504 for (auto N : V) {
505 OS << N;
506 encodeULEB128(0, OS);
507 }
508 return sampleprof_error::success;
509 }
510
writeFuncOffsetTable()511 std::error_code SampleProfileWriterCompactBinary::writeFuncOffsetTable() {
512 auto &OS = *OutputStream;
513
514 // Fill the slot remembered by TableOffset with the offset of FuncOffsetTable.
515 auto &OFS = static_cast<raw_fd_ostream &>(OS);
516 uint64_t FuncOffsetTableStart = OS.tell();
517 if (OFS.seek(TableOffset) == (uint64_t)-1)
518 return sampleprof_error::ostream_seek_unsupported;
519 support::endian::Writer Writer(*OutputStream, support::little);
520 Writer.write(FuncOffsetTableStart);
521 if (OFS.seek(FuncOffsetTableStart) == (uint64_t)-1)
522 return sampleprof_error::ostream_seek_unsupported;
523
524 // Write out the table size.
525 encodeULEB128(FuncOffsetTable.size(), OS);
526
527 // Write out FuncOffsetTable.
528 for (auto Entry : FuncOffsetTable) {
529 if (std::error_code EC =
530 writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS))
531 return EC;
532 encodeULEB128(Entry.second, OS);
533 }
534 return sampleprof_error::success;
535 }
536
writeNameTable()537 std::error_code SampleProfileWriterCompactBinary::writeNameTable() {
538 auto &OS = *OutputStream;
539 std::set<StringRef> V;
540 stablizeNameTable(V);
541
542 // Write out the name table.
543 encodeULEB128(NameTable.size(), OS);
544 for (auto N : V) {
545 encodeULEB128(MD5Hash(N), OS);
546 }
547 return sampleprof_error::success;
548 }
549
550 std::error_code
writeMagicIdent(SampleProfileFormat Format)551 SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
552 auto &OS = *OutputStream;
553 // Write file magic identifier.
554 encodeULEB128(SPMagic(Format), OS);
555 encodeULEB128(SPVersion(), OS);
556 return sampleprof_error::success;
557 }
558
writeHeader(const StringMap<FunctionSamples> & ProfileMap)559 std::error_code SampleProfileWriterBinary::writeHeader(
560 const StringMap<FunctionSamples> &ProfileMap) {
561 writeMagicIdent(Format);
562
563 computeSummary(ProfileMap);
564 if (auto EC = writeSummary())
565 return EC;
566
567 // Generate the name table for all the functions referenced in the profile.
568 for (const auto &I : ProfileMap) {
569 assert(I.first() == I.second.getNameWithContext() &&
570 "Inconsistent profile map");
571 addName(I.first(), FunctionSamples::ProfileIsCS);
572 addNames(I.second);
573 }
574
575 writeNameTable();
576 return sampleprof_error::success;
577 }
578
setToCompressAllSections()579 void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
580 for (auto &Entry : SectionHdrLayout)
581 addSecFlag(Entry, SecCommonFlags::SecFlagCompress);
582 }
583
setToCompressSection(SecType Type)584 void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
585 addSectionFlag(Type, SecCommonFlags::SecFlagCompress);
586 }
587
allocSecHdrTable()588 void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
589 support::endian::Writer Writer(*OutputStream, support::little);
590
591 Writer.write(static_cast<uint64_t>(SectionHdrLayout.size()));
592 SecHdrTableOffset = OutputStream->tell();
593 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
594 Writer.write(static_cast<uint64_t>(-1));
595 Writer.write(static_cast<uint64_t>(-1));
596 Writer.write(static_cast<uint64_t>(-1));
597 Writer.write(static_cast<uint64_t>(-1));
598 }
599 }
600
writeSecHdrTable()601 std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
602 auto &OFS = static_cast<raw_fd_ostream &>(*OutputStream);
603 uint64_t Saved = OutputStream->tell();
604
605 // Set OutputStream to the location saved in SecHdrTableOffset.
606 if (OFS.seek(SecHdrTableOffset) == (uint64_t)-1)
607 return sampleprof_error::ostream_seek_unsupported;
608 support::endian::Writer Writer(*OutputStream, support::little);
609
610 assert(SecHdrTable.size() == SectionHdrLayout.size() &&
611 "SecHdrTable entries doesn't match SectionHdrLayout");
612 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
613 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
614 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
615 }
616
617 // Write the section header table in the order specified in
618 // SectionHdrLayout. SectionHdrLayout specifies the sections
619 // order in which profile reader expect to read, so the section
620 // header table should be written in the order in SectionHdrLayout.
621 // Note that the section order in SecHdrTable may be different
622 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
623 // needs to be computed after SecLBRProfile (the order in SecHdrTable),
624 // but it needs to be read before SecLBRProfile (the order in
625 // SectionHdrLayout). So we use IndexMap above to switch the order.
626 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
627 LayoutIdx++) {
628 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
629 "Incorrect LayoutIdx in SecHdrTable");
630 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
631 Writer.write(static_cast<uint64_t>(Entry.Type));
632 Writer.write(static_cast<uint64_t>(Entry.Flags));
633 Writer.write(static_cast<uint64_t>(Entry.Offset));
634 Writer.write(static_cast<uint64_t>(Entry.Size));
635 }
636
637 // Reset OutputStream.
638 if (OFS.seek(Saved) == (uint64_t)-1)
639 return sampleprof_error::ostream_seek_unsupported;
640
641 return sampleprof_error::success;
642 }
643
writeHeader(const StringMap<FunctionSamples> & ProfileMap)644 std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
645 const StringMap<FunctionSamples> &ProfileMap) {
646 auto &OS = *OutputStream;
647 FileStart = OS.tell();
648 writeMagicIdent(Format);
649
650 allocSecHdrTable();
651 return sampleprof_error::success;
652 }
653
writeHeader(const StringMap<FunctionSamples> & ProfileMap)654 std::error_code SampleProfileWriterCompactBinary::writeHeader(
655 const StringMap<FunctionSamples> &ProfileMap) {
656 support::endian::Writer Writer(*OutputStream, support::little);
657 if (auto EC = SampleProfileWriterBinary::writeHeader(ProfileMap))
658 return EC;
659
660 // Reserve a slot for the offset of function offset table. The slot will
661 // be populated with the offset of FuncOffsetTable later.
662 TableOffset = OutputStream->tell();
663 Writer.write(static_cast<uint64_t>(-2));
664 return sampleprof_error::success;
665 }
666
writeSummary()667 std::error_code SampleProfileWriterBinary::writeSummary() {
668 auto &OS = *OutputStream;
669 encodeULEB128(Summary->getTotalCount(), OS);
670 encodeULEB128(Summary->getMaxCount(), OS);
671 encodeULEB128(Summary->getMaxFunctionCount(), OS);
672 encodeULEB128(Summary->getNumCounts(), OS);
673 encodeULEB128(Summary->getNumFunctions(), OS);
674 std::vector<ProfileSummaryEntry> &Entries = Summary->getDetailedSummary();
675 encodeULEB128(Entries.size(), OS);
676 for (auto Entry : Entries) {
677 encodeULEB128(Entry.Cutoff, OS);
678 encodeULEB128(Entry.MinCount, OS);
679 encodeULEB128(Entry.NumCounts, OS);
680 }
681 return sampleprof_error::success;
682 }
writeBody(const FunctionSamples & S)683 std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
684 auto &OS = *OutputStream;
685
686 if (std::error_code EC =
687 writeNameIdx(S.getNameWithContext(), FunctionSamples::ProfileIsCS))
688 return EC;
689
690 encodeULEB128(S.getTotalSamples(), OS);
691
692 // Emit all the body samples.
693 encodeULEB128(S.getBodySamples().size(), OS);
694 for (const auto &I : S.getBodySamples()) {
695 LineLocation Loc = I.first;
696 const SampleRecord &Sample = I.second;
697 encodeULEB128(Loc.LineOffset, OS);
698 encodeULEB128(Loc.Discriminator, OS);
699 encodeULEB128(Sample.getSamples(), OS);
700 encodeULEB128(Sample.getCallTargets().size(), OS);
701 for (const auto &J : Sample.getSortedCallTargets()) {
702 StringRef Callee = J.first;
703 uint64_t CalleeSamples = J.second;
704 if (std::error_code EC = writeNameIdx(Callee))
705 return EC;
706 encodeULEB128(CalleeSamples, OS);
707 }
708 }
709
710 // Recursively emit all the callsite samples.
711 uint64_t NumCallsites = 0;
712 for (const auto &J : S.getCallsiteSamples())
713 NumCallsites += J.second.size();
714 encodeULEB128(NumCallsites, OS);
715 for (const auto &J : S.getCallsiteSamples())
716 for (const auto &FS : J.second) {
717 LineLocation Loc = J.first;
718 const FunctionSamples &CalleeSamples = FS.second;
719 encodeULEB128(Loc.LineOffset, OS);
720 encodeULEB128(Loc.Discriminator, OS);
721 if (std::error_code EC = writeBody(CalleeSamples))
722 return EC;
723 }
724
725 return sampleprof_error::success;
726 }
727
728 /// Write samples of a top-level function to a binary file.
729 ///
730 /// \returns true if the samples were written successfully, false otherwise.
731 std::error_code
writeSample(const FunctionSamples & S)732 SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
733 encodeULEB128(S.getHeadSamples(), *OutputStream);
734 return writeBody(S);
735 }
736
737 std::error_code
writeSample(const FunctionSamples & S)738 SampleProfileWriterCompactBinary::writeSample(const FunctionSamples &S) {
739 uint64_t Offset = OutputStream->tell();
740 StringRef Name = S.getName();
741 FuncOffsetTable[Name] = Offset;
742 encodeULEB128(S.getHeadSamples(), *OutputStream);
743 return writeBody(S);
744 }
745
746 /// Create a sample profile file writer based on the specified format.
747 ///
748 /// \param Filename The file to create.
749 ///
750 /// \param Format Encoding format for the profile file.
751 ///
752 /// \returns an error code indicating the status of the created writer.
753 ErrorOr<std::unique_ptr<SampleProfileWriter>>
create(StringRef Filename,SampleProfileFormat Format)754 SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
755 std::error_code EC;
756 std::unique_ptr<raw_ostream> OS;
757 if (Format == SPF_Binary || Format == SPF_Ext_Binary ||
758 Format == SPF_Compact_Binary)
759 OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
760 else
761 OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
762 if (EC)
763 return EC;
764
765 return create(OS, Format);
766 }
767
768 /// Create a sample profile stream writer based on the specified format.
769 ///
770 /// \param OS The output stream to store the profile data to.
771 ///
772 /// \param Format Encoding format for the profile file.
773 ///
774 /// \returns an error code indicating the status of the created writer.
775 ErrorOr<std::unique_ptr<SampleProfileWriter>>
create(std::unique_ptr<raw_ostream> & OS,SampleProfileFormat Format)776 SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
777 SampleProfileFormat Format) {
778 std::error_code EC;
779 std::unique_ptr<SampleProfileWriter> Writer;
780
781 // Currently only Text and Extended Binary format are supported for CSSPGO.
782 if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
783 (Format == SPF_Binary || Format == SPF_Compact_Binary))
784 return sampleprof_error::unsupported_writing_format;
785
786 if (Format == SPF_Binary)
787 Writer.reset(new SampleProfileWriterRawBinary(OS));
788 else if (Format == SPF_Ext_Binary)
789 Writer.reset(new SampleProfileWriterExtBinary(OS));
790 else if (Format == SPF_Compact_Binary)
791 Writer.reset(new SampleProfileWriterCompactBinary(OS));
792 else if (Format == SPF_Text)
793 Writer.reset(new SampleProfileWriterText(OS));
794 else if (Format == SPF_GCC)
795 EC = sampleprof_error::unsupported_writing_format;
796 else
797 EC = sampleprof_error::unrecognized_format;
798
799 if (EC)
800 return EC;
801
802 Writer->Format = Format;
803 return std::move(Writer);
804 }
805
computeSummary(const StringMap<FunctionSamples> & ProfileMap)806 void SampleProfileWriter::computeSummary(
807 const StringMap<FunctionSamples> &ProfileMap) {
808 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
809 Summary = Builder.computeSummaryForProfiles(ProfileMap);
810 }
811