1 //===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class that writes LLVM sample profiles. It
10 // supports two file formats: text and binary. The textual representation
11 // is useful for debugging and testing purposes. The binary representation
12 // is more compact, resulting in smaller file sizes. However, they can
13 // both be used interchangeably.
14 //
15 // See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16 // supported formats.
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "llvm/ProfileData/SampleProfWriter.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/ADT/StringSet.h"
23 #include "llvm/ProfileData/ProfileCommon.h"
24 #include "llvm/ProfileData/SampleProf.h"
25 #include "llvm/Support/Compression.h"
26 #include "llvm/Support/Endian.h"
27 #include "llvm/Support/EndianStream.h"
28 #include "llvm/Support/ErrorOr.h"
29 #include "llvm/Support/FileSystem.h"
30 #include "llvm/Support/LEB128.h"
31 #include "llvm/Support/MD5.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include <algorithm>
34 #include <cstdint>
35 #include <memory>
36 #include <set>
37 #include <system_error>
38 #include <utility>
39 #include <vector>
40 
41 using namespace llvm;
42 using namespace sampleprof;
43 
writeFuncProfiles(const StringMap<FunctionSamples> & ProfileMap)44 std::error_code SampleProfileWriter::writeFuncProfiles(
45     const StringMap<FunctionSamples> &ProfileMap) {
46   // Sort the ProfileMap by total samples.
47   typedef std::pair<StringRef, const FunctionSamples *> NameFunctionSamples;
48   std::vector<NameFunctionSamples> V;
49   for (const auto &I : ProfileMap) {
50     assert(I.getKey() == I.second.getNameWithContext() &&
51            "Inconsistent profile map");
52     V.push_back(std::make_pair(I.second.getNameWithContext(), &I.second));
53   }
54   llvm::stable_sort(
55       V, [](const NameFunctionSamples &A, const NameFunctionSamples &B) {
56         if (A.second->getTotalSamples() == B.second->getTotalSamples())
57           return A.first > B.first;
58         return A.second->getTotalSamples() > B.second->getTotalSamples();
59       });
60 
61   for (const auto &I : V) {
62     if (std::error_code EC = writeSample(*I.second))
63       return EC;
64   }
65   return sampleprof_error::success;
66 }
67 
68 std::error_code
write(const StringMap<FunctionSamples> & ProfileMap)69 SampleProfileWriter::write(const StringMap<FunctionSamples> &ProfileMap) {
70   if (std::error_code EC = writeHeader(ProfileMap))
71     return EC;
72 
73   if (std::error_code EC = writeFuncProfiles(ProfileMap))
74     return EC;
75 
76   return sampleprof_error::success;
77 }
78 
79 /// Return the current position and prepare to use it as the start
80 /// position of a section given the section type \p Type and its position
81 /// \p LayoutIdx in SectionHdrLayout.
82 uint64_t
markSectionStart(SecType Type,uint32_t LayoutIdx)83 SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,
84                                                    uint32_t LayoutIdx) {
85   uint64_t SectionStart = OutputStream->tell();
86   assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
87   const auto &Entry = SectionHdrLayout[LayoutIdx];
88   assert(Entry.Type == Type && "Unexpected section type");
89   // Use LocalBuf as a temporary output for writting data.
90   if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress))
91     LocalBufStream.swap(OutputStream);
92   return SectionStart;
93 }
94 
compressAndOutput()95 std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
96   if (!llvm::zlib::isAvailable())
97     return sampleprof_error::zlib_unavailable;
98   std::string &UncompressedStrings =
99       static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
100   if (UncompressedStrings.size() == 0)
101     return sampleprof_error::success;
102   auto &OS = *OutputStream;
103   SmallString<128> CompressedStrings;
104   llvm::Error E = zlib::compress(UncompressedStrings, CompressedStrings,
105                                  zlib::BestSizeCompression);
106   if (E)
107     return sampleprof_error::compress_failed;
108   encodeULEB128(UncompressedStrings.size(), OS);
109   encodeULEB128(CompressedStrings.size(), OS);
110   OS << CompressedStrings.str();
111   UncompressedStrings.clear();
112   return sampleprof_error::success;
113 }
114 
115 /// Add a new section into section header table given the section type
116 /// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
117 /// location \p SectionStart where the section should be written to.
addNewSection(SecType Type,uint32_t LayoutIdx,uint64_t SectionStart)118 std::error_code SampleProfileWriterExtBinaryBase::addNewSection(
119     SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
120   assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
121   const auto &Entry = SectionHdrLayout[LayoutIdx];
122   assert(Entry.Type == Type && "Unexpected section type");
123   if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress)) {
124     LocalBufStream.swap(OutputStream);
125     if (std::error_code EC = compressAndOutput())
126       return EC;
127   }
128   SecHdrTable.push_back({Type, Entry.Flags, SectionStart - FileStart,
129                          OutputStream->tell() - SectionStart, LayoutIdx});
130   return sampleprof_error::success;
131 }
132 
write(const StringMap<FunctionSamples> & ProfileMap)133 std::error_code SampleProfileWriterExtBinaryBase::write(
134     const StringMap<FunctionSamples> &ProfileMap) {
135   if (std::error_code EC = writeHeader(ProfileMap))
136     return EC;
137 
138   std::string LocalBuf;
139   LocalBufStream = std::make_unique<raw_string_ostream>(LocalBuf);
140   if (std::error_code EC = writeSections(ProfileMap))
141     return EC;
142 
143   if (std::error_code EC = writeSecHdrTable())
144     return EC;
145 
146   return sampleprof_error::success;
147 }
148 
149 std::error_code
writeSample(const FunctionSamples & S)150 SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
151   uint64_t Offset = OutputStream->tell();
152   StringRef Name = S.getNameWithContext();
153   FuncOffsetTable[Name] = Offset - SecLBRProfileStart;
154   encodeULEB128(S.getHeadSamples(), *OutputStream);
155   return writeBody(S);
156 }
157 
writeFuncOffsetTable()158 std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
159   auto &OS = *OutputStream;
160 
161   // Write out the table size.
162   encodeULEB128(FuncOffsetTable.size(), OS);
163 
164   // Write out FuncOffsetTable.
165   for (auto Entry : FuncOffsetTable) {
166     if (std::error_code EC =
167             writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS))
168       return EC;
169     encodeULEB128(Entry.second, OS);
170   }
171   FuncOffsetTable.clear();
172   return sampleprof_error::success;
173 }
174 
writeFuncMetadata(const StringMap<FunctionSamples> & Profiles)175 std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
176     const StringMap<FunctionSamples> &Profiles) {
177   if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS)
178     return sampleprof_error::success;
179   auto &OS = *OutputStream;
180   for (const auto &Entry : Profiles) {
181     if (std::error_code EC = writeNameIdx(Entry.second.getNameWithContext(),
182                                           FunctionSamples::ProfileIsCS))
183       return EC;
184     if (FunctionSamples::ProfileIsProbeBased)
185       encodeULEB128(Entry.second.getFunctionHash(), OS);
186     if (FunctionSamples::ProfileIsCS)
187       encodeULEB128(Entry.second.getContext().getAllAttributes(), OS);
188   }
189   return sampleprof_error::success;
190 }
191 
writeNameTable()192 std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
193   if (!UseMD5)
194     return SampleProfileWriterBinary::writeNameTable();
195 
196   auto &OS = *OutputStream;
197   std::set<StringRef> V;
198   stablizeNameTable(V);
199 
200   // Write out the MD5 name table. We wrote unencoded MD5 so reader can
201   // retrieve the name using the name index without having to read the
202   // whole name table.
203   encodeULEB128(NameTable.size(), OS);
204   support::endian::Writer Writer(OS, support::little);
205   for (auto N : V)
206     Writer.write(MD5Hash(N));
207   return sampleprof_error::success;
208 }
209 
writeNameTableSection(const StringMap<FunctionSamples> & ProfileMap)210 std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
211     const StringMap<FunctionSamples> &ProfileMap) {
212   for (const auto &I : ProfileMap) {
213     assert(I.first() == I.second.getNameWithContext() &&
214            "Inconsistent profile map");
215     addName(I.second.getNameWithContext(), FunctionSamples::ProfileIsCS);
216     addNames(I.second);
217   }
218 
219   // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
220   // so compiler won't strip the suffix during profile matching after
221   // seeing the flag in the profile.
222   for (const auto &I : NameTable) {
223     if (I.first.find(FunctionSamples::UniqSuffix) != StringRef::npos) {
224       addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix);
225       break;
226     }
227   }
228 
229   if (auto EC = writeNameTable())
230     return EC;
231   return sampleprof_error::success;
232 }
233 
234 std::error_code
writeProfileSymbolListSection()235 SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {
236   if (ProfSymList && ProfSymList->size() > 0)
237     if (std::error_code EC = ProfSymList->write(*OutputStream))
238       return EC;
239 
240   return sampleprof_error::success;
241 }
242 
writeOneSection(SecType Type,uint32_t LayoutIdx,const StringMap<FunctionSamples> & ProfileMap)243 std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
244     SecType Type, uint32_t LayoutIdx,
245     const StringMap<FunctionSamples> &ProfileMap) {
246   // The setting of SecFlagCompress should happen before markSectionStart.
247   if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())
248     setToCompressSection(SecProfileSymbolList);
249   if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
250     addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased);
251   if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
252     addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFullContext);
253   if (Type == SecFuncMetadata && FunctionSamples::ProfileIsCS)
254     addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagHasAttribute);
255 
256   uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
257   switch (Type) {
258   case SecProfSummary:
259     computeSummary(ProfileMap);
260     if (auto EC = writeSummary())
261       return EC;
262     break;
263   case SecNameTable:
264     if (auto EC = writeNameTableSection(ProfileMap))
265       return EC;
266     break;
267   case SecLBRProfile:
268     SecLBRProfileStart = OutputStream->tell();
269     if (std::error_code EC = writeFuncProfiles(ProfileMap))
270       return EC;
271     break;
272   case SecFuncOffsetTable:
273     if (auto EC = writeFuncOffsetTable())
274       return EC;
275     break;
276   case SecFuncMetadata:
277     if (std::error_code EC = writeFuncMetadata(ProfileMap))
278       return EC;
279     break;
280   case SecProfileSymbolList:
281     if (auto EC = writeProfileSymbolListSection())
282       return EC;
283     break;
284   default:
285     if (auto EC = writeCustomSection(Type))
286       return EC;
287     break;
288   }
289   if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
290     return EC;
291   return sampleprof_error::success;
292 }
293 
writeDefaultLayout(const StringMap<FunctionSamples> & ProfileMap)294 std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
295     const StringMap<FunctionSamples> &ProfileMap) {
296   // The const indices passed to writeOneSection below are specifying the
297   // positions of the sections in SectionHdrLayout. Look at
298   // initSectionHdrLayout to find out where each section is located in
299   // SectionHdrLayout.
300   if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))
301     return EC;
302   if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))
303     return EC;
304   if (auto EC = writeOneSection(SecLBRProfile, 3, ProfileMap))
305     return EC;
306   if (auto EC = writeOneSection(SecProfileSymbolList, 4, ProfileMap))
307     return EC;
308   if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ProfileMap))
309     return EC;
310   if (auto EC = writeOneSection(SecFuncMetadata, 5, ProfileMap))
311     return EC;
312   return sampleprof_error::success;
313 }
314 
315 static void
splitProfileMapToTwo(const StringMap<FunctionSamples> & ProfileMap,StringMap<FunctionSamples> & ContextProfileMap,StringMap<FunctionSamples> & NoContextProfileMap)316 splitProfileMapToTwo(const StringMap<FunctionSamples> &ProfileMap,
317                      StringMap<FunctionSamples> &ContextProfileMap,
318                      StringMap<FunctionSamples> &NoContextProfileMap) {
319   for (const auto &I : ProfileMap) {
320     if (I.second.getCallsiteSamples().size())
321       ContextProfileMap.insert({I.first(), I.second});
322     else
323       NoContextProfileMap.insert({I.first(), I.second});
324   }
325 }
326 
writeCtxSplitLayout(const StringMap<FunctionSamples> & ProfileMap)327 std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
328     const StringMap<FunctionSamples> &ProfileMap) {
329   StringMap<FunctionSamples> ContextProfileMap, NoContextProfileMap;
330   splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap);
331 
332   if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))
333     return EC;
334   if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))
335     return EC;
336   if (auto EC = writeOneSection(SecLBRProfile, 3, ContextProfileMap))
337     return EC;
338   if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ContextProfileMap))
339     return EC;
340   // Mark the section to have no context. Note section flag needs to be set
341   // before writing the section.
342   addSectionFlag(5, SecCommonFlags::SecFlagFlat);
343   if (auto EC = writeOneSection(SecLBRProfile, 5, NoContextProfileMap))
344     return EC;
345   // Mark the section to have no context. Note section flag needs to be set
346   // before writing the section.
347   addSectionFlag(4, SecCommonFlags::SecFlagFlat);
348   if (auto EC = writeOneSection(SecFuncOffsetTable, 4, NoContextProfileMap))
349     return EC;
350   if (auto EC = writeOneSection(SecProfileSymbolList, 6, ProfileMap))
351     return EC;
352   if (auto EC = writeOneSection(SecFuncMetadata, 7, ProfileMap))
353     return EC;
354 
355   return sampleprof_error::success;
356 }
357 
writeSections(const StringMap<FunctionSamples> & ProfileMap)358 std::error_code SampleProfileWriterExtBinary::writeSections(
359     const StringMap<FunctionSamples> &ProfileMap) {
360   std::error_code EC;
361   if (SecLayout == DefaultLayout)
362     EC = writeDefaultLayout(ProfileMap);
363   else if (SecLayout == CtxSplitLayout)
364     EC = writeCtxSplitLayout(ProfileMap);
365   else
366     llvm_unreachable("Unsupported layout");
367   return EC;
368 }
369 
write(const StringMap<FunctionSamples> & ProfileMap)370 std::error_code SampleProfileWriterCompactBinary::write(
371     const StringMap<FunctionSamples> &ProfileMap) {
372   if (std::error_code EC = SampleProfileWriter::write(ProfileMap))
373     return EC;
374   if (std::error_code EC = writeFuncOffsetTable())
375     return EC;
376   return sampleprof_error::success;
377 }
378 
379 /// Write samples to a text file.
380 ///
381 /// Note: it may be tempting to implement this in terms of
382 /// FunctionSamples::print().  Please don't.  The dump functionality is intended
383 /// for debugging and has no specified form.
384 ///
385 /// The format used here is more structured and deliberate because
386 /// it needs to be parsed by the SampleProfileReaderText class.
writeSample(const FunctionSamples & S)387 std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
388   auto &OS = *OutputStream;
389   if (FunctionSamples::ProfileIsCS)
390     OS << "[" << S.getNameWithContext() << "]:" << S.getTotalSamples();
391   else
392     OS << S.getName() << ":" << S.getTotalSamples();
393 
394   if (Indent == 0)
395     OS << ":" << S.getHeadSamples();
396   OS << "\n";
397 
398   SampleSorter<LineLocation, SampleRecord> SortedSamples(S.getBodySamples());
399   for (const auto &I : SortedSamples.get()) {
400     LineLocation Loc = I->first;
401     const SampleRecord &Sample = I->second;
402     OS.indent(Indent + 1);
403     if (Loc.Discriminator == 0)
404       OS << Loc.LineOffset << ": ";
405     else
406       OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
407 
408     OS << Sample.getSamples();
409 
410     for (const auto &J : Sample.getSortedCallTargets())
411       OS << " " << J.first << ":" << J.second;
412     OS << "\n";
413   }
414 
415   SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
416       S.getCallsiteSamples());
417   Indent += 1;
418   for (const auto &I : SortedCallsiteSamples.get())
419     for (const auto &FS : I->second) {
420       LineLocation Loc = I->first;
421       const FunctionSamples &CalleeSamples = FS.second;
422       OS.indent(Indent);
423       if (Loc.Discriminator == 0)
424         OS << Loc.LineOffset << ": ";
425       else
426         OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
427       if (std::error_code EC = writeSample(CalleeSamples))
428         return EC;
429     }
430   Indent -= 1;
431 
432   if (Indent == 0) {
433     if (FunctionSamples::ProfileIsProbeBased) {
434       OS.indent(Indent + 1);
435       OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
436     }
437     if (FunctionSamples::ProfileIsCS) {
438       OS.indent(Indent + 1);
439       OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
440     }
441   }
442 
443   return sampleprof_error::success;
444 }
445 
writeNameIdx(StringRef FName,bool IsContextName)446 std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName,
447                                                         bool IsContextName) {
448   std::string BracketedName;
449   if (IsContextName) {
450     BracketedName = "[" + FName.str() + "]";
451     FName = StringRef(BracketedName);
452   }
453 
454   const auto &Ret = NameTable.find(FName);
455   if (Ret == NameTable.end())
456     return sampleprof_error::truncated_name_table;
457   encodeULEB128(Ret->second, *OutputStream);
458   return sampleprof_error::success;
459 }
460 
addName(StringRef FName,bool IsContextName)461 void SampleProfileWriterBinary::addName(StringRef FName, bool IsContextName) {
462   if (IsContextName) {
463     auto It = BracketedContextStr.insert("[" + FName.str() + "]");
464     FName = StringRef(*It.first);
465   }
466   NameTable.insert(std::make_pair(FName, 0));
467 }
468 
addNames(const FunctionSamples & S)469 void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
470   // Add all the names in indirect call targets.
471   for (const auto &I : S.getBodySamples()) {
472     const SampleRecord &Sample = I.second;
473     for (const auto &J : Sample.getCallTargets())
474       addName(J.first());
475   }
476 
477   // Recursively add all the names for inlined callsites.
478   for (const auto &J : S.getCallsiteSamples())
479     for (const auto &FS : J.second) {
480       const FunctionSamples &CalleeSamples = FS.second;
481       addName(CalleeSamples.getName());
482       addNames(CalleeSamples);
483     }
484 }
485 
stablizeNameTable(std::set<StringRef> & V)486 void SampleProfileWriterBinary::stablizeNameTable(std::set<StringRef> &V) {
487   // Sort the names to make NameTable deterministic.
488   for (const auto &I : NameTable)
489     V.insert(I.first);
490   int i = 0;
491   for (const StringRef &N : V)
492     NameTable[N] = i++;
493 }
494 
writeNameTable()495 std::error_code SampleProfileWriterBinary::writeNameTable() {
496   auto &OS = *OutputStream;
497   std::set<StringRef> V;
498   stablizeNameTable(V);
499 
500   // Write out the name table.
501   encodeULEB128(NameTable.size(), OS);
502   for (auto N : V) {
503     OS << N;
504     encodeULEB128(0, OS);
505   }
506   return sampleprof_error::success;
507 }
508 
writeFuncOffsetTable()509 std::error_code SampleProfileWriterCompactBinary::writeFuncOffsetTable() {
510   auto &OS = *OutputStream;
511 
512   // Fill the slot remembered by TableOffset with the offset of FuncOffsetTable.
513   auto &OFS = static_cast<raw_fd_ostream &>(OS);
514   uint64_t FuncOffsetTableStart = OS.tell();
515   if (OFS.seek(TableOffset) == (uint64_t)-1)
516     return sampleprof_error::ostream_seek_unsupported;
517   support::endian::Writer Writer(*OutputStream, support::little);
518   Writer.write(FuncOffsetTableStart);
519   if (OFS.seek(FuncOffsetTableStart) == (uint64_t)-1)
520     return sampleprof_error::ostream_seek_unsupported;
521 
522   // Write out the table size.
523   encodeULEB128(FuncOffsetTable.size(), OS);
524 
525   // Write out FuncOffsetTable.
526   for (auto Entry : FuncOffsetTable) {
527     if (std::error_code EC =
528             writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS))
529       return EC;
530     encodeULEB128(Entry.second, OS);
531   }
532   return sampleprof_error::success;
533 }
534 
writeNameTable()535 std::error_code SampleProfileWriterCompactBinary::writeNameTable() {
536   auto &OS = *OutputStream;
537   std::set<StringRef> V;
538   stablizeNameTable(V);
539 
540   // Write out the name table.
541   encodeULEB128(NameTable.size(), OS);
542   for (auto N : V) {
543     encodeULEB128(MD5Hash(N), OS);
544   }
545   return sampleprof_error::success;
546 }
547 
548 std::error_code
writeMagicIdent(SampleProfileFormat Format)549 SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
550   auto &OS = *OutputStream;
551   // Write file magic identifier.
552   encodeULEB128(SPMagic(Format), OS);
553   encodeULEB128(SPVersion(), OS);
554   return sampleprof_error::success;
555 }
556 
writeHeader(const StringMap<FunctionSamples> & ProfileMap)557 std::error_code SampleProfileWriterBinary::writeHeader(
558     const StringMap<FunctionSamples> &ProfileMap) {
559   writeMagicIdent(Format);
560 
561   computeSummary(ProfileMap);
562   if (auto EC = writeSummary())
563     return EC;
564 
565   // Generate the name table for all the functions referenced in the profile.
566   for (const auto &I : ProfileMap) {
567     assert(I.first() == I.second.getNameWithContext() &&
568            "Inconsistent profile map");
569     addName(I.first(), FunctionSamples::ProfileIsCS);
570     addNames(I.second);
571   }
572 
573   writeNameTable();
574   return sampleprof_error::success;
575 }
576 
setToCompressAllSections()577 void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
578   for (auto &Entry : SectionHdrLayout)
579     addSecFlag(Entry, SecCommonFlags::SecFlagCompress);
580 }
581 
setToCompressSection(SecType Type)582 void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
583   addSectionFlag(Type, SecCommonFlags::SecFlagCompress);
584 }
585 
allocSecHdrTable()586 void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
587   support::endian::Writer Writer(*OutputStream, support::little);
588 
589   Writer.write(static_cast<uint64_t>(SectionHdrLayout.size()));
590   SecHdrTableOffset = OutputStream->tell();
591   for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
592     Writer.write(static_cast<uint64_t>(-1));
593     Writer.write(static_cast<uint64_t>(-1));
594     Writer.write(static_cast<uint64_t>(-1));
595     Writer.write(static_cast<uint64_t>(-1));
596   }
597 }
598 
writeSecHdrTable()599 std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
600   auto &OFS = static_cast<raw_fd_ostream &>(*OutputStream);
601   uint64_t Saved = OutputStream->tell();
602 
603   // Set OutputStream to the location saved in SecHdrTableOffset.
604   if (OFS.seek(SecHdrTableOffset) == (uint64_t)-1)
605     return sampleprof_error::ostream_seek_unsupported;
606   support::endian::Writer Writer(*OutputStream, support::little);
607 
608   assert(SecHdrTable.size() == SectionHdrLayout.size() &&
609          "SecHdrTable entries doesn't match SectionHdrLayout");
610   SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
611   for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
612     IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
613   }
614 
615   // Write the section header table in the order specified in
616   // SectionHdrLayout. SectionHdrLayout specifies the sections
617   // order in which profile reader expect to read, so the section
618   // header table should be written in the order in SectionHdrLayout.
619   // Note that the section order in SecHdrTable may be different
620   // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
621   // needs to be computed after SecLBRProfile (the order in SecHdrTable),
622   // but it needs to be read before SecLBRProfile (the order in
623   // SectionHdrLayout). So we use IndexMap above to switch the order.
624   for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
625        LayoutIdx++) {
626     assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
627            "Incorrect LayoutIdx in SecHdrTable");
628     auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
629     Writer.write(static_cast<uint64_t>(Entry.Type));
630     Writer.write(static_cast<uint64_t>(Entry.Flags));
631     Writer.write(static_cast<uint64_t>(Entry.Offset));
632     Writer.write(static_cast<uint64_t>(Entry.Size));
633   }
634 
635   // Reset OutputStream.
636   if (OFS.seek(Saved) == (uint64_t)-1)
637     return sampleprof_error::ostream_seek_unsupported;
638 
639   return sampleprof_error::success;
640 }
641 
writeHeader(const StringMap<FunctionSamples> & ProfileMap)642 std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
643     const StringMap<FunctionSamples> &ProfileMap) {
644   auto &OS = *OutputStream;
645   FileStart = OS.tell();
646   writeMagicIdent(Format);
647 
648   allocSecHdrTable();
649   return sampleprof_error::success;
650 }
651 
writeHeader(const StringMap<FunctionSamples> & ProfileMap)652 std::error_code SampleProfileWriterCompactBinary::writeHeader(
653     const StringMap<FunctionSamples> &ProfileMap) {
654   support::endian::Writer Writer(*OutputStream, support::little);
655   if (auto EC = SampleProfileWriterBinary::writeHeader(ProfileMap))
656     return EC;
657 
658   // Reserve a slot for the offset of function offset table. The slot will
659   // be populated with the offset of FuncOffsetTable later.
660   TableOffset = OutputStream->tell();
661   Writer.write(static_cast<uint64_t>(-2));
662   return sampleprof_error::success;
663 }
664 
writeSummary()665 std::error_code SampleProfileWriterBinary::writeSummary() {
666   auto &OS = *OutputStream;
667   encodeULEB128(Summary->getTotalCount(), OS);
668   encodeULEB128(Summary->getMaxCount(), OS);
669   encodeULEB128(Summary->getMaxFunctionCount(), OS);
670   encodeULEB128(Summary->getNumCounts(), OS);
671   encodeULEB128(Summary->getNumFunctions(), OS);
672   std::vector<ProfileSummaryEntry> &Entries = Summary->getDetailedSummary();
673   encodeULEB128(Entries.size(), OS);
674   for (auto Entry : Entries) {
675     encodeULEB128(Entry.Cutoff, OS);
676     encodeULEB128(Entry.MinCount, OS);
677     encodeULEB128(Entry.NumCounts, OS);
678   }
679   return sampleprof_error::success;
680 }
writeBody(const FunctionSamples & S)681 std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
682   auto &OS = *OutputStream;
683 
684   if (std::error_code EC =
685           writeNameIdx(S.getNameWithContext(), FunctionSamples::ProfileIsCS))
686     return EC;
687 
688   encodeULEB128(S.getTotalSamples(), OS);
689 
690   // Emit all the body samples.
691   encodeULEB128(S.getBodySamples().size(), OS);
692   for (const auto &I : S.getBodySamples()) {
693     LineLocation Loc = I.first;
694     const SampleRecord &Sample = I.second;
695     encodeULEB128(Loc.LineOffset, OS);
696     encodeULEB128(Loc.Discriminator, OS);
697     encodeULEB128(Sample.getSamples(), OS);
698     encodeULEB128(Sample.getCallTargets().size(), OS);
699     for (const auto &J : Sample.getSortedCallTargets()) {
700       StringRef Callee = J.first;
701       uint64_t CalleeSamples = J.second;
702       if (std::error_code EC = writeNameIdx(Callee))
703         return EC;
704       encodeULEB128(CalleeSamples, OS);
705     }
706   }
707 
708   // Recursively emit all the callsite samples.
709   uint64_t NumCallsites = 0;
710   for (const auto &J : S.getCallsiteSamples())
711     NumCallsites += J.second.size();
712   encodeULEB128(NumCallsites, OS);
713   for (const auto &J : S.getCallsiteSamples())
714     for (const auto &FS : J.second) {
715       LineLocation Loc = J.first;
716       const FunctionSamples &CalleeSamples = FS.second;
717       encodeULEB128(Loc.LineOffset, OS);
718       encodeULEB128(Loc.Discriminator, OS);
719       if (std::error_code EC = writeBody(CalleeSamples))
720         return EC;
721     }
722 
723   return sampleprof_error::success;
724 }
725 
726 /// Write samples of a top-level function to a binary file.
727 ///
728 /// \returns true if the samples were written successfully, false otherwise.
729 std::error_code
writeSample(const FunctionSamples & S)730 SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
731   encodeULEB128(S.getHeadSamples(), *OutputStream);
732   return writeBody(S);
733 }
734 
735 std::error_code
writeSample(const FunctionSamples & S)736 SampleProfileWriterCompactBinary::writeSample(const FunctionSamples &S) {
737   uint64_t Offset = OutputStream->tell();
738   StringRef Name = S.getName();
739   FuncOffsetTable[Name] = Offset;
740   encodeULEB128(S.getHeadSamples(), *OutputStream);
741   return writeBody(S);
742 }
743 
744 /// Create a sample profile file writer based on the specified format.
745 ///
746 /// \param Filename The file to create.
747 ///
748 /// \param Format Encoding format for the profile file.
749 ///
750 /// \returns an error code indicating the status of the created writer.
751 ErrorOr<std::unique_ptr<SampleProfileWriter>>
create(StringRef Filename,SampleProfileFormat Format)752 SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
753   std::error_code EC;
754   std::unique_ptr<raw_ostream> OS;
755   if (Format == SPF_Binary || Format == SPF_Ext_Binary ||
756       Format == SPF_Compact_Binary)
757     OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
758   else
759     OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
760   if (EC)
761     return EC;
762 
763   return create(OS, Format);
764 }
765 
766 /// Create a sample profile stream writer based on the specified format.
767 ///
768 /// \param OS The output stream to store the profile data to.
769 ///
770 /// \param Format Encoding format for the profile file.
771 ///
772 /// \returns an error code indicating the status of the created writer.
773 ErrorOr<std::unique_ptr<SampleProfileWriter>>
create(std::unique_ptr<raw_ostream> & OS,SampleProfileFormat Format)774 SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
775                             SampleProfileFormat Format) {
776   std::error_code EC;
777   std::unique_ptr<SampleProfileWriter> Writer;
778 
779   // Currently only Text and Extended Binary format are supported for CSSPGO.
780   if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
781       (Format == SPF_Binary || Format == SPF_Compact_Binary))
782     return sampleprof_error::unsupported_writing_format;
783 
784   if (Format == SPF_Binary)
785     Writer.reset(new SampleProfileWriterRawBinary(OS));
786   else if (Format == SPF_Ext_Binary)
787     Writer.reset(new SampleProfileWriterExtBinary(OS));
788   else if (Format == SPF_Compact_Binary)
789     Writer.reset(new SampleProfileWriterCompactBinary(OS));
790   else if (Format == SPF_Text)
791     Writer.reset(new SampleProfileWriterText(OS));
792   else if (Format == SPF_GCC)
793     EC = sampleprof_error::unsupported_writing_format;
794   else
795     EC = sampleprof_error::unrecognized_format;
796 
797   if (EC)
798     return EC;
799 
800   Writer->Format = Format;
801   return std::move(Writer);
802 }
803 
computeSummary(const StringMap<FunctionSamples> & ProfileMap)804 void SampleProfileWriter::computeSummary(
805     const StringMap<FunctionSamples> &ProfileMap) {
806   SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
807   Summary = Builder.computeSummaryForProfiles(ProfileMap);
808 }
809