10b57cec5SDimitry Andric //===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // This file contains support for writing profiling data for clang's
100b57cec5SDimitry Andric // instrumentation based PGO and coverage.
110b57cec5SDimitry Andric //
120b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
130b57cec5SDimitry Andric
140b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfWriter.h"
150b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h"
1606c3fb27SDimitry Andric #include "llvm/ADT/SetVector.h"
170b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
180b57cec5SDimitry Andric #include "llvm/IR/ProfileSummary.h"
190b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProf.h"
2081ad6265SDimitry Andric #include "llvm/ProfileData/MemProf.h"
210b57cec5SDimitry Andric #include "llvm/ProfileData/ProfileCommon.h"
220b57cec5SDimitry Andric #include "llvm/Support/Endian.h"
230b57cec5SDimitry Andric #include "llvm/Support/EndianStream.h"
240b57cec5SDimitry Andric #include "llvm/Support/Error.h"
250b57cec5SDimitry Andric #include "llvm/Support/MemoryBuffer.h"
260b57cec5SDimitry Andric #include "llvm/Support/OnDiskHashTable.h"
270b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h"
280b57cec5SDimitry Andric #include <cstdint>
290b57cec5SDimitry Andric #include <memory>
300b57cec5SDimitry Andric #include <string>
310b57cec5SDimitry Andric #include <tuple>
320b57cec5SDimitry Andric #include <utility>
330b57cec5SDimitry Andric #include <vector>
340b57cec5SDimitry Andric
350b57cec5SDimitry Andric using namespace llvm;
360b57cec5SDimitry Andric
370b57cec5SDimitry Andric // A struct to define how the data stream should be patched. For Indexed
380b57cec5SDimitry Andric // profiling, only uint64_t data type is needed.
390b57cec5SDimitry Andric struct PatchItem {
400b57cec5SDimitry Andric uint64_t Pos; // Where to patch.
410b57cec5SDimitry Andric uint64_t *D; // Pointer to an array of source data.
420b57cec5SDimitry Andric int N; // Number of elements in \c D array.
430b57cec5SDimitry Andric };
440b57cec5SDimitry Andric
450b57cec5SDimitry Andric namespace llvm {
460b57cec5SDimitry Andric
470b57cec5SDimitry Andric // A wrapper class to abstract writer stream with support of bytes
480b57cec5SDimitry Andric // back patching.
490b57cec5SDimitry Andric class ProfOStream {
500b57cec5SDimitry Andric public:
ProfOStream(raw_fd_ostream & FD)510b57cec5SDimitry Andric ProfOStream(raw_fd_ostream &FD)
525f757f3fSDimitry Andric : IsFDOStream(true), OS(FD), LE(FD, llvm::endianness::little) {}
ProfOStream(raw_string_ostream & STR)530b57cec5SDimitry Andric ProfOStream(raw_string_ostream &STR)
545f757f3fSDimitry Andric : IsFDOStream(false), OS(STR), LE(STR, llvm::endianness::little) {}
550b57cec5SDimitry Andric
tell()560b57cec5SDimitry Andric uint64_t tell() { return OS.tell(); }
write(uint64_t V)570b57cec5SDimitry Andric void write(uint64_t V) { LE.write<uint64_t>(V); }
writeByte(uint8_t V)58bdd1243dSDimitry Andric void writeByte(uint8_t V) { LE.write<uint8_t>(V); }
590b57cec5SDimitry Andric
600b57cec5SDimitry Andric // \c patch can only be called when all data is written and flushed.
610b57cec5SDimitry Andric // For raw_string_ostream, the patch is done on the target string
620b57cec5SDimitry Andric // directly and it won't be reflected in the stream's internal buffer.
patch(PatchItem * P,int NItems)630b57cec5SDimitry Andric void patch(PatchItem *P, int NItems) {
640b57cec5SDimitry Andric using namespace support;
650b57cec5SDimitry Andric
660b57cec5SDimitry Andric if (IsFDOStream) {
670b57cec5SDimitry Andric raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
6881ad6265SDimitry Andric const uint64_t LastPos = FDOStream.tell();
690b57cec5SDimitry Andric for (int K = 0; K < NItems; K++) {
700b57cec5SDimitry Andric FDOStream.seek(P[K].Pos);
710b57cec5SDimitry Andric for (int I = 0; I < P[K].N; I++)
720b57cec5SDimitry Andric write(P[K].D[I]);
730b57cec5SDimitry Andric }
7481ad6265SDimitry Andric // Reset the stream to the last position after patching so that users
7581ad6265SDimitry Andric // don't accidentally overwrite data. This makes it consistent with
7681ad6265SDimitry Andric // the string stream below which replaces the data directly.
7781ad6265SDimitry Andric FDOStream.seek(LastPos);
780b57cec5SDimitry Andric } else {
790b57cec5SDimitry Andric raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
800b57cec5SDimitry Andric std::string &Data = SOStream.str(); // with flush
810b57cec5SDimitry Andric for (int K = 0; K < NItems; K++) {
820b57cec5SDimitry Andric for (int I = 0; I < P[K].N; I++) {
835f757f3fSDimitry Andric uint64_t Bytes =
845f757f3fSDimitry Andric endian::byte_swap<uint64_t, llvm::endianness::little>(P[K].D[I]);
850b57cec5SDimitry Andric Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t),
860b57cec5SDimitry Andric (const char *)&Bytes, sizeof(uint64_t));
870b57cec5SDimitry Andric }
880b57cec5SDimitry Andric }
890b57cec5SDimitry Andric }
900b57cec5SDimitry Andric }
910b57cec5SDimitry Andric
920b57cec5SDimitry Andric // If \c OS is an instance of \c raw_fd_ostream, this field will be
930b57cec5SDimitry Andric // true. Otherwise, \c OS will be an raw_string_ostream.
940b57cec5SDimitry Andric bool IsFDOStream;
950b57cec5SDimitry Andric raw_ostream &OS;
960b57cec5SDimitry Andric support::endian::Writer LE;
970b57cec5SDimitry Andric };
980b57cec5SDimitry Andric
990b57cec5SDimitry Andric class InstrProfRecordWriterTrait {
1000b57cec5SDimitry Andric public:
1010b57cec5SDimitry Andric using key_type = StringRef;
1020b57cec5SDimitry Andric using key_type_ref = StringRef;
1030b57cec5SDimitry Andric
1040b57cec5SDimitry Andric using data_type = const InstrProfWriter::ProfilingData *const;
1050b57cec5SDimitry Andric using data_type_ref = const InstrProfWriter::ProfilingData *const;
1060b57cec5SDimitry Andric
1070b57cec5SDimitry Andric using hash_value_type = uint64_t;
1080b57cec5SDimitry Andric using offset_type = uint64_t;
1090b57cec5SDimitry Andric
1105f757f3fSDimitry Andric llvm::endianness ValueProfDataEndianness = llvm::endianness::little;
1110b57cec5SDimitry Andric InstrProfSummaryBuilder *SummaryBuilder;
1120b57cec5SDimitry Andric InstrProfSummaryBuilder *CSSummaryBuilder;
1130b57cec5SDimitry Andric
1140b57cec5SDimitry Andric InstrProfRecordWriterTrait() = default;
1150b57cec5SDimitry Andric
ComputeHash(key_type_ref K)1160b57cec5SDimitry Andric static hash_value_type ComputeHash(key_type_ref K) {
1170b57cec5SDimitry Andric return IndexedInstrProf::ComputeHash(K);
1180b57cec5SDimitry Andric }
1190b57cec5SDimitry Andric
1200b57cec5SDimitry Andric static std::pair<offset_type, offset_type>
EmitKeyDataLength(raw_ostream & Out,key_type_ref K,data_type_ref V)1210b57cec5SDimitry Andric EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
1220b57cec5SDimitry Andric using namespace support;
1230b57cec5SDimitry Andric
1245f757f3fSDimitry Andric endian::Writer LE(Out, llvm::endianness::little);
1250b57cec5SDimitry Andric
1260b57cec5SDimitry Andric offset_type N = K.size();
1270b57cec5SDimitry Andric LE.write<offset_type>(N);
1280b57cec5SDimitry Andric
1290b57cec5SDimitry Andric offset_type M = 0;
1300b57cec5SDimitry Andric for (const auto &ProfileData : *V) {
1310b57cec5SDimitry Andric const InstrProfRecord &ProfRecord = ProfileData.second;
1320b57cec5SDimitry Andric M += sizeof(uint64_t); // The function hash
1330b57cec5SDimitry Andric M += sizeof(uint64_t); // The size of the Counts vector
1340b57cec5SDimitry Andric M += ProfRecord.Counts.size() * sizeof(uint64_t);
1355f757f3fSDimitry Andric M += sizeof(uint64_t); // The size of the Bitmap vector
1365f757f3fSDimitry Andric M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t);
1370b57cec5SDimitry Andric
1380b57cec5SDimitry Andric // Value data
1390b57cec5SDimitry Andric M += ValueProfData::getSize(ProfileData.second);
1400b57cec5SDimitry Andric }
1410b57cec5SDimitry Andric LE.write<offset_type>(M);
1420b57cec5SDimitry Andric
1430b57cec5SDimitry Andric return std::make_pair(N, M);
1440b57cec5SDimitry Andric }
1450b57cec5SDimitry Andric
EmitKey(raw_ostream & Out,key_type_ref K,offset_type N)1460b57cec5SDimitry Andric void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
1470b57cec5SDimitry Andric Out.write(K.data(), N);
1480b57cec5SDimitry Andric }
1490b57cec5SDimitry Andric
EmitData(raw_ostream & Out,key_type_ref,data_type_ref V,offset_type)1500b57cec5SDimitry Andric void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
1510b57cec5SDimitry Andric using namespace support;
1520b57cec5SDimitry Andric
1535f757f3fSDimitry Andric endian::Writer LE(Out, llvm::endianness::little);
1540b57cec5SDimitry Andric for (const auto &ProfileData : *V) {
1550b57cec5SDimitry Andric const InstrProfRecord &ProfRecord = ProfileData.second;
1560b57cec5SDimitry Andric if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))
1570b57cec5SDimitry Andric CSSummaryBuilder->addRecord(ProfRecord);
1580b57cec5SDimitry Andric else
1590b57cec5SDimitry Andric SummaryBuilder->addRecord(ProfRecord);
1600b57cec5SDimitry Andric
1610b57cec5SDimitry Andric LE.write<uint64_t>(ProfileData.first); // Function hash
1620b57cec5SDimitry Andric LE.write<uint64_t>(ProfRecord.Counts.size());
1630b57cec5SDimitry Andric for (uint64_t I : ProfRecord.Counts)
1640b57cec5SDimitry Andric LE.write<uint64_t>(I);
1650b57cec5SDimitry Andric
1665f757f3fSDimitry Andric LE.write<uint64_t>(ProfRecord.BitmapBytes.size());
1675f757f3fSDimitry Andric for (uint64_t I : ProfRecord.BitmapBytes)
1685f757f3fSDimitry Andric LE.write<uint64_t>(I);
1695f757f3fSDimitry Andric
1700b57cec5SDimitry Andric // Write value data
1710b57cec5SDimitry Andric std::unique_ptr<ValueProfData> VDataPtr =
1720b57cec5SDimitry Andric ValueProfData::serializeFrom(ProfileData.second);
1730b57cec5SDimitry Andric uint32_t S = VDataPtr->getSize();
1740b57cec5SDimitry Andric VDataPtr->swapBytesFromHost(ValueProfDataEndianness);
1750b57cec5SDimitry Andric Out.write((const char *)VDataPtr.get(), S);
1760b57cec5SDimitry Andric }
1770b57cec5SDimitry Andric }
1780b57cec5SDimitry Andric };
1790b57cec5SDimitry Andric
1800b57cec5SDimitry Andric } // end namespace llvm
1810b57cec5SDimitry Andric
InstrProfWriter(bool Sparse,uint64_t TemporalProfTraceReservoirSize,uint64_t MaxTemporalProfTraceLength)18206c3fb27SDimitry Andric InstrProfWriter::InstrProfWriter(bool Sparse,
18306c3fb27SDimitry Andric uint64_t TemporalProfTraceReservoirSize,
18406c3fb27SDimitry Andric uint64_t MaxTemporalProfTraceLength)
18506c3fb27SDimitry Andric : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
18606c3fb27SDimitry Andric TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
18706c3fb27SDimitry Andric InfoObj(new InstrProfRecordWriterTrait()) {}
1880b57cec5SDimitry Andric
~InstrProfWriter()1890b57cec5SDimitry Andric InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
1900b57cec5SDimitry Andric
1910b57cec5SDimitry Andric // Internal interface for testing purpose only.
setValueProfDataEndianness(llvm::endianness Endianness)1925f757f3fSDimitry Andric void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) {
1930b57cec5SDimitry Andric InfoObj->ValueProfDataEndianness = Endianness;
1940b57cec5SDimitry Andric }
1950b57cec5SDimitry Andric
setOutputSparse(bool Sparse)1960b57cec5SDimitry Andric void InstrProfWriter::setOutputSparse(bool Sparse) {
1970b57cec5SDimitry Andric this->Sparse = Sparse;
1980b57cec5SDimitry Andric }
1990b57cec5SDimitry Andric
addRecord(NamedInstrProfRecord && I,uint64_t Weight,function_ref<void (Error)> Warn)2000b57cec5SDimitry Andric void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
2010b57cec5SDimitry Andric function_ref<void(Error)> Warn) {
2020b57cec5SDimitry Andric auto Name = I.Name;
2030b57cec5SDimitry Andric auto Hash = I.Hash;
2040b57cec5SDimitry Andric addRecord(Name, Hash, std::move(I), Weight, Warn);
2050b57cec5SDimitry Andric }
2060b57cec5SDimitry Andric
overlapRecord(NamedInstrProfRecord && Other,OverlapStats & Overlap,OverlapStats & FuncLevelOverlap,const OverlapFuncFilters & FuncFilter)2070b57cec5SDimitry Andric void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,
2080b57cec5SDimitry Andric OverlapStats &Overlap,
2090b57cec5SDimitry Andric OverlapStats &FuncLevelOverlap,
2100b57cec5SDimitry Andric const OverlapFuncFilters &FuncFilter) {
2110b57cec5SDimitry Andric auto Name = Other.Name;
2120b57cec5SDimitry Andric auto Hash = Other.Hash;
2138bcb0991SDimitry Andric Other.accumulateCounts(FuncLevelOverlap.Test);
21406c3fb27SDimitry Andric if (!FunctionData.contains(Name)) {
2150b57cec5SDimitry Andric Overlap.addOneUnique(FuncLevelOverlap.Test);
2160b57cec5SDimitry Andric return;
2170b57cec5SDimitry Andric }
2180b57cec5SDimitry Andric if (FuncLevelOverlap.Test.CountSum < 1.0f) {
2190b57cec5SDimitry Andric Overlap.Overlap.NumEntries += 1;
2200b57cec5SDimitry Andric return;
2210b57cec5SDimitry Andric }
2220b57cec5SDimitry Andric auto &ProfileDataMap = FunctionData[Name];
2230b57cec5SDimitry Andric bool NewFunc;
2240b57cec5SDimitry Andric ProfilingData::iterator Where;
2250b57cec5SDimitry Andric std::tie(Where, NewFunc) =
2260b57cec5SDimitry Andric ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
2270b57cec5SDimitry Andric if (NewFunc) {
2280b57cec5SDimitry Andric Overlap.addOneMismatch(FuncLevelOverlap.Test);
2290b57cec5SDimitry Andric return;
2300b57cec5SDimitry Andric }
2310b57cec5SDimitry Andric InstrProfRecord &Dest = Where->second;
2320b57cec5SDimitry Andric
2330b57cec5SDimitry Andric uint64_t ValueCutoff = FuncFilter.ValueCutoff;
234349cc55cSDimitry Andric if (!FuncFilter.NameFilter.empty() && Name.contains(FuncFilter.NameFilter))
2350b57cec5SDimitry Andric ValueCutoff = 0;
2360b57cec5SDimitry Andric
2370b57cec5SDimitry Andric Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
2380b57cec5SDimitry Andric }
2390b57cec5SDimitry Andric
addRecord(StringRef Name,uint64_t Hash,InstrProfRecord && I,uint64_t Weight,function_ref<void (Error)> Warn)2400b57cec5SDimitry Andric void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
2410b57cec5SDimitry Andric InstrProfRecord &&I, uint64_t Weight,
2420b57cec5SDimitry Andric function_ref<void(Error)> Warn) {
2430b57cec5SDimitry Andric auto &ProfileDataMap = FunctionData[Name];
2440b57cec5SDimitry Andric
2450b57cec5SDimitry Andric bool NewFunc;
2460b57cec5SDimitry Andric ProfilingData::iterator Where;
2470b57cec5SDimitry Andric std::tie(Where, NewFunc) =
2480b57cec5SDimitry Andric ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
2490b57cec5SDimitry Andric InstrProfRecord &Dest = Where->second;
2500b57cec5SDimitry Andric
2510b57cec5SDimitry Andric auto MapWarn = [&](instrprof_error E) {
2520b57cec5SDimitry Andric Warn(make_error<InstrProfError>(E));
2530b57cec5SDimitry Andric };
2540b57cec5SDimitry Andric
2550b57cec5SDimitry Andric if (NewFunc) {
2560b57cec5SDimitry Andric // We've never seen a function with this name and hash, add it.
2570b57cec5SDimitry Andric Dest = std::move(I);
2580b57cec5SDimitry Andric if (Weight > 1)
259e8d8bef9SDimitry Andric Dest.scale(Weight, 1, MapWarn);
2600b57cec5SDimitry Andric } else {
2610b57cec5SDimitry Andric // We're updating a function we've seen before.
2620b57cec5SDimitry Andric Dest.merge(I, Weight, MapWarn);
2630b57cec5SDimitry Andric }
2640b57cec5SDimitry Andric
2650b57cec5SDimitry Andric Dest.sortValueData();
2660b57cec5SDimitry Andric }
2670b57cec5SDimitry Andric
addMemProfRecord(const Function::GUID Id,const memprof::IndexedMemProfRecord & Record)26881ad6265SDimitry Andric void InstrProfWriter::addMemProfRecord(
26981ad6265SDimitry Andric const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {
27081ad6265SDimitry Andric auto Result = MemProfRecordData.insert({Id, Record});
27181ad6265SDimitry Andric // If we inserted a new record then we are done.
27281ad6265SDimitry Andric if (Result.second) {
27381ad6265SDimitry Andric return;
27481ad6265SDimitry Andric }
27581ad6265SDimitry Andric memprof::IndexedMemProfRecord &Existing = Result.first->second;
27681ad6265SDimitry Andric Existing.merge(Record);
27781ad6265SDimitry Andric }
27881ad6265SDimitry Andric
addMemProfFrame(const memprof::FrameId Id,const memprof::Frame & Frame,function_ref<void (Error)> Warn)27981ad6265SDimitry Andric bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,
28081ad6265SDimitry Andric const memprof::Frame &Frame,
28181ad6265SDimitry Andric function_ref<void(Error)> Warn) {
28281ad6265SDimitry Andric auto Result = MemProfFrameData.insert({Id, Frame});
28381ad6265SDimitry Andric // If a mapping already exists for the current frame id and it does not
28481ad6265SDimitry Andric // match the new mapping provided then reset the existing contents and bail
28581ad6265SDimitry Andric // out. We don't support the merging of memprof data whose Frame -> Id
28681ad6265SDimitry Andric // mapping across profiles is inconsistent.
28781ad6265SDimitry Andric if (!Result.second && Result.first->second != Frame) {
28881ad6265SDimitry Andric Warn(make_error<InstrProfError>(instrprof_error::malformed,
28981ad6265SDimitry Andric "frame to id mapping mismatch"));
29081ad6265SDimitry Andric return false;
29181ad6265SDimitry Andric }
29281ad6265SDimitry Andric return true;
29381ad6265SDimitry Andric }
29481ad6265SDimitry Andric
addBinaryIds(ArrayRef<llvm::object::BuildID> BIs)295bdd1243dSDimitry Andric void InstrProfWriter::addBinaryIds(ArrayRef<llvm::object::BuildID> BIs) {
296bdd1243dSDimitry Andric llvm::append_range(BinaryIds, BIs);
297bdd1243dSDimitry Andric }
298bdd1243dSDimitry Andric
addTemporalProfileTrace(TemporalProfTraceTy Trace)29906c3fb27SDimitry Andric void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) {
30006c3fb27SDimitry Andric if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
30106c3fb27SDimitry Andric Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);
30206c3fb27SDimitry Andric if (Trace.FunctionNameRefs.empty())
30306c3fb27SDimitry Andric return;
30406c3fb27SDimitry Andric
30506c3fb27SDimitry Andric if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
30606c3fb27SDimitry Andric // Simply append the trace if we have not yet hit our reservoir size limit.
30706c3fb27SDimitry Andric TemporalProfTraces.push_back(std::move(Trace));
30806c3fb27SDimitry Andric } else {
30906c3fb27SDimitry Andric // Otherwise, replace a random trace in the stream.
31006c3fb27SDimitry Andric std::uniform_int_distribution<uint64_t> Distribution(
31106c3fb27SDimitry Andric 0, TemporalProfTraceStreamSize);
31206c3fb27SDimitry Andric uint64_t RandomIndex = Distribution(RNG);
31306c3fb27SDimitry Andric if (RandomIndex < TemporalProfTraces.size())
31406c3fb27SDimitry Andric TemporalProfTraces[RandomIndex] = std::move(Trace);
31506c3fb27SDimitry Andric }
31606c3fb27SDimitry Andric ++TemporalProfTraceStreamSize;
31706c3fb27SDimitry Andric }
31806c3fb27SDimitry Andric
addTemporalProfileTraces(SmallVectorImpl<TemporalProfTraceTy> & SrcTraces,uint64_t SrcStreamSize)31906c3fb27SDimitry Andric void InstrProfWriter::addTemporalProfileTraces(
32006c3fb27SDimitry Andric SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
32106c3fb27SDimitry Andric // Assume that the source has the same reservoir size as the destination to
32206c3fb27SDimitry Andric // avoid needing to record it in the indexed profile format.
32306c3fb27SDimitry Andric bool IsDestSampled =
32406c3fb27SDimitry Andric (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize);
32506c3fb27SDimitry Andric bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize);
32606c3fb27SDimitry Andric if (!IsDestSampled && IsSrcSampled) {
32706c3fb27SDimitry Andric // If one of the traces are sampled, ensure that it belongs to Dest.
32806c3fb27SDimitry Andric std::swap(TemporalProfTraces, SrcTraces);
32906c3fb27SDimitry Andric std::swap(TemporalProfTraceStreamSize, SrcStreamSize);
33006c3fb27SDimitry Andric std::swap(IsDestSampled, IsSrcSampled);
33106c3fb27SDimitry Andric }
33206c3fb27SDimitry Andric if (!IsSrcSampled) {
33306c3fb27SDimitry Andric // If the source stream is not sampled, we add each source trace normally.
33406c3fb27SDimitry Andric for (auto &Trace : SrcTraces)
33506c3fb27SDimitry Andric addTemporalProfileTrace(std::move(Trace));
33606c3fb27SDimitry Andric return;
33706c3fb27SDimitry Andric }
33806c3fb27SDimitry Andric // Otherwise, we find the traces that would have been removed if we added
33906c3fb27SDimitry Andric // the whole source stream.
34006c3fb27SDimitry Andric SmallSetVector<uint64_t, 8> IndicesToReplace;
34106c3fb27SDimitry Andric for (uint64_t I = 0; I < SrcStreamSize; I++) {
34206c3fb27SDimitry Andric std::uniform_int_distribution<uint64_t> Distribution(
34306c3fb27SDimitry Andric 0, TemporalProfTraceStreamSize);
34406c3fb27SDimitry Andric uint64_t RandomIndex = Distribution(RNG);
34506c3fb27SDimitry Andric if (RandomIndex < TemporalProfTraces.size())
34606c3fb27SDimitry Andric IndicesToReplace.insert(RandomIndex);
34706c3fb27SDimitry Andric ++TemporalProfTraceStreamSize;
34806c3fb27SDimitry Andric }
34906c3fb27SDimitry Andric // Then we insert a random sample of the source traces.
35006c3fb27SDimitry Andric llvm::shuffle(SrcTraces.begin(), SrcTraces.end(), RNG);
35106c3fb27SDimitry Andric for (const auto &[Index, Trace] : llvm::zip(IndicesToReplace, SrcTraces))
35206c3fb27SDimitry Andric TemporalProfTraces[Index] = std::move(Trace);
35306c3fb27SDimitry Andric }
35406c3fb27SDimitry Andric
mergeRecordsFromWriter(InstrProfWriter && IPW,function_ref<void (Error)> Warn)3550b57cec5SDimitry Andric void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
3560b57cec5SDimitry Andric function_ref<void(Error)> Warn) {
3570b57cec5SDimitry Andric for (auto &I : IPW.FunctionData)
3580b57cec5SDimitry Andric for (auto &Func : I.getValue())
3590b57cec5SDimitry Andric addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn);
36081ad6265SDimitry Andric
361bdd1243dSDimitry Andric BinaryIds.reserve(BinaryIds.size() + IPW.BinaryIds.size());
362bdd1243dSDimitry Andric for (auto &I : IPW.BinaryIds)
363bdd1243dSDimitry Andric addBinaryIds(I);
364bdd1243dSDimitry Andric
36506c3fb27SDimitry Andric addTemporalProfileTraces(IPW.TemporalProfTraces,
36606c3fb27SDimitry Andric IPW.TemporalProfTraceStreamSize);
36706c3fb27SDimitry Andric
36881ad6265SDimitry Andric MemProfFrameData.reserve(IPW.MemProfFrameData.size());
36981ad6265SDimitry Andric for (auto &I : IPW.MemProfFrameData) {
37081ad6265SDimitry Andric // If we weren't able to add the frame mappings then it doesn't make sense
37181ad6265SDimitry Andric // to try to merge the records from this profile.
37281ad6265SDimitry Andric if (!addMemProfFrame(I.first, I.second, Warn))
37381ad6265SDimitry Andric return;
37481ad6265SDimitry Andric }
37581ad6265SDimitry Andric
37681ad6265SDimitry Andric MemProfRecordData.reserve(IPW.MemProfRecordData.size());
37781ad6265SDimitry Andric for (auto &I : IPW.MemProfRecordData) {
37881ad6265SDimitry Andric addMemProfRecord(I.first, I.second);
37981ad6265SDimitry Andric }
3800b57cec5SDimitry Andric }
3810b57cec5SDimitry Andric
shouldEncodeData(const ProfilingData & PD)3820b57cec5SDimitry Andric bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
3830b57cec5SDimitry Andric if (!Sparse)
3840b57cec5SDimitry Andric return true;
3850b57cec5SDimitry Andric for (const auto &Func : PD) {
3860b57cec5SDimitry Andric const InstrProfRecord &IPR = Func.second;
3870b57cec5SDimitry Andric if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; }))
3880b57cec5SDimitry Andric return true;
3895f757f3fSDimitry Andric if (llvm::any_of(IPR.BitmapBytes, [](uint8_t Byte) { return Byte > 0; }))
3905f757f3fSDimitry Andric return true;
3910b57cec5SDimitry Andric }
3920b57cec5SDimitry Andric return false;
3930b57cec5SDimitry Andric }
3940b57cec5SDimitry Andric
setSummary(IndexedInstrProf::Summary * TheSummary,ProfileSummary & PS)3950b57cec5SDimitry Andric static void setSummary(IndexedInstrProf::Summary *TheSummary,
3960b57cec5SDimitry Andric ProfileSummary &PS) {
3970b57cec5SDimitry Andric using namespace IndexedInstrProf;
3980b57cec5SDimitry Andric
399349cc55cSDimitry Andric const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
4000b57cec5SDimitry Andric TheSummary->NumSummaryFields = Summary::NumKinds;
4010b57cec5SDimitry Andric TheSummary->NumCutoffEntries = Res.size();
4020b57cec5SDimitry Andric TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());
4030b57cec5SDimitry Andric TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());
4040b57cec5SDimitry Andric TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());
4050b57cec5SDimitry Andric TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());
4060b57cec5SDimitry Andric TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());
4070b57cec5SDimitry Andric TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());
4080b57cec5SDimitry Andric for (unsigned I = 0; I < Res.size(); I++)
4090b57cec5SDimitry Andric TheSummary->setEntry(I, Res[I]);
4100b57cec5SDimitry Andric }
4110b57cec5SDimitry Andric
writeImpl(ProfOStream & OS)412fe6060f1SDimitry Andric Error InstrProfWriter::writeImpl(ProfOStream &OS) {
4130b57cec5SDimitry Andric using namespace IndexedInstrProf;
414bdd1243dSDimitry Andric using namespace support;
4150b57cec5SDimitry Andric
4160b57cec5SDimitry Andric OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
4170b57cec5SDimitry Andric
4180b57cec5SDimitry Andric InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
4190b57cec5SDimitry Andric InfoObj->SummaryBuilder = &ISB;
4200b57cec5SDimitry Andric InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs);
4210b57cec5SDimitry Andric InfoObj->CSSummaryBuilder = &CSISB;
4220b57cec5SDimitry Andric
4230b57cec5SDimitry Andric // Populate the hash table generator.
42406c3fb27SDimitry Andric SmallVector<std::pair<StringRef, const ProfilingData *>, 0> OrderedData;
4250b57cec5SDimitry Andric for (const auto &I : FunctionData)
4260b57cec5SDimitry Andric if (shouldEncodeData(I.getValue()))
42706c3fb27SDimitry Andric OrderedData.emplace_back((I.getKey()), &I.getValue());
42806c3fb27SDimitry Andric llvm::sort(OrderedData, less_first());
42906c3fb27SDimitry Andric for (const auto &I : OrderedData)
43006c3fb27SDimitry Andric Generator.insert(I.first, I.second);
43181ad6265SDimitry Andric
4320b57cec5SDimitry Andric // Write the header.
4330b57cec5SDimitry Andric IndexedInstrProf::Header Header;
4340b57cec5SDimitry Andric Header.Magic = IndexedInstrProf::Magic;
4350b57cec5SDimitry Andric Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion;
43681ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
4370b57cec5SDimitry Andric Header.Version |= VARIANT_MASK_IR_PROF;
43881ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
4390b57cec5SDimitry Andric Header.Version |= VARIANT_MASK_CSIR_PROF;
44081ad6265SDimitry Andric if (static_cast<bool>(ProfileKind &
44181ad6265SDimitry Andric InstrProfKind::FunctionEntryInstrumentation))
442e8d8bef9SDimitry Andric Header.Version |= VARIANT_MASK_INSTR_ENTRY;
4431fd87a68SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
4441fd87a68SDimitry Andric Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
4451fd87a68SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
4461fd87a68SDimitry Andric Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
44781ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
44881ad6265SDimitry Andric Header.Version |= VARIANT_MASK_MEMPROF;
44906c3fb27SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
45006c3fb27SDimitry Andric Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
451e8d8bef9SDimitry Andric
4520b57cec5SDimitry Andric Header.Unused = 0;
4530b57cec5SDimitry Andric Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
4540b57cec5SDimitry Andric Header.HashOffset = 0;
45581ad6265SDimitry Andric Header.MemProfOffset = 0;
456bdd1243dSDimitry Andric Header.BinaryIdOffset = 0;
45706c3fb27SDimitry Andric Header.TemporalProfTracesOffset = 0;
4580b57cec5SDimitry Andric int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);
4590b57cec5SDimitry Andric
46006c3fb27SDimitry Andric // Only write out all the fields except 'HashOffset', 'MemProfOffset',
46106c3fb27SDimitry Andric // 'BinaryIdOffset' and `TemporalProfTracesOffset`. We need to remember the
46206c3fb27SDimitry Andric // offset of these fields to allow back patching later.
46306c3fb27SDimitry Andric for (int I = 0; I < N - 4; I++)
4640b57cec5SDimitry Andric OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
4650b57cec5SDimitry Andric
4660b57cec5SDimitry Andric // Save the location of Header.HashOffset field in \c OS.
4670b57cec5SDimitry Andric uint64_t HashTableStartFieldOffset = OS.tell();
4680b57cec5SDimitry Andric // Reserve the space for HashOffset field.
4690b57cec5SDimitry Andric OS.write(0);
4700b57cec5SDimitry Andric
47181ad6265SDimitry Andric // Save the location of MemProf profile data. This is stored in two parts as
47281ad6265SDimitry Andric // the schema and as a separate on-disk chained hashtable.
47381ad6265SDimitry Andric uint64_t MemProfSectionOffset = OS.tell();
47481ad6265SDimitry Andric // Reserve space for the MemProf table field to be patched later if this
47581ad6265SDimitry Andric // profile contains memory profile information.
47681ad6265SDimitry Andric OS.write(0);
47781ad6265SDimitry Andric
478bdd1243dSDimitry Andric // Save the location of binary ids section.
479bdd1243dSDimitry Andric uint64_t BinaryIdSectionOffset = OS.tell();
480bdd1243dSDimitry Andric // Reserve space for the BinaryIdOffset field to be patched later if this
481bdd1243dSDimitry Andric // profile contains binary ids.
482bdd1243dSDimitry Andric OS.write(0);
483bdd1243dSDimitry Andric
48406c3fb27SDimitry Andric uint64_t TemporalProfTracesOffset = OS.tell();
48506c3fb27SDimitry Andric OS.write(0);
48606c3fb27SDimitry Andric
4870b57cec5SDimitry Andric // Reserve space to write profile summary data.
4880b57cec5SDimitry Andric uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
4890b57cec5SDimitry Andric uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
4900b57cec5SDimitry Andric // Remember the summary offset.
4910b57cec5SDimitry Andric uint64_t SummaryOffset = OS.tell();
4920b57cec5SDimitry Andric for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
4930b57cec5SDimitry Andric OS.write(0);
4940b57cec5SDimitry Andric uint64_t CSSummaryOffset = 0;
4950b57cec5SDimitry Andric uint64_t CSSummarySize = 0;
49681ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
4970b57cec5SDimitry Andric CSSummaryOffset = OS.tell();
4980b57cec5SDimitry Andric CSSummarySize = SummarySize / sizeof(uint64_t);
4990b57cec5SDimitry Andric for (unsigned I = 0; I < CSSummarySize; I++)
5000b57cec5SDimitry Andric OS.write(0);
5010b57cec5SDimitry Andric }
5020b57cec5SDimitry Andric
5030b57cec5SDimitry Andric // Write the hash table.
5040b57cec5SDimitry Andric uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
5050b57cec5SDimitry Andric
50681ad6265SDimitry Andric // Write the MemProf profile data if we have it. This includes a simple schema
50781ad6265SDimitry Andric // with the format described below followed by the hashtable:
50881ad6265SDimitry Andric // uint64_t RecordTableOffset = RecordTableGenerator.Emit
50981ad6265SDimitry Andric // uint64_t FramePayloadOffset = Stream offset before emitting the frame table
51081ad6265SDimitry Andric // uint64_t FrameTableOffset = FrameTableGenerator.Emit
51181ad6265SDimitry Andric // uint64_t Num schema entries
51281ad6265SDimitry Andric // uint64_t Schema entry 0
51381ad6265SDimitry Andric // uint64_t Schema entry 1
51481ad6265SDimitry Andric // ....
51581ad6265SDimitry Andric // uint64_t Schema entry N - 1
51681ad6265SDimitry Andric // OnDiskChainedHashTable MemProfRecordData
51781ad6265SDimitry Andric // OnDiskChainedHashTable MemProfFrameData
51881ad6265SDimitry Andric uint64_t MemProfSectionStart = 0;
51981ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
52081ad6265SDimitry Andric MemProfSectionStart = OS.tell();
52181ad6265SDimitry Andric OS.write(0ULL); // Reserve space for the memprof record table offset.
52281ad6265SDimitry Andric OS.write(0ULL); // Reserve space for the memprof frame payload offset.
52381ad6265SDimitry Andric OS.write(0ULL); // Reserve space for the memprof frame table offset.
52481ad6265SDimitry Andric
52581ad6265SDimitry Andric auto Schema = memprof::PortableMemInfoBlock::getSchema();
52681ad6265SDimitry Andric OS.write(static_cast<uint64_t>(Schema.size()));
52781ad6265SDimitry Andric for (const auto Id : Schema) {
52881ad6265SDimitry Andric OS.write(static_cast<uint64_t>(Id));
52981ad6265SDimitry Andric }
53081ad6265SDimitry Andric
53181ad6265SDimitry Andric auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
53281ad6265SDimitry Andric RecordWriter->Schema = &Schema;
53381ad6265SDimitry Andric OnDiskChainedHashTableGenerator<memprof::RecordWriterTrait>
53481ad6265SDimitry Andric RecordTableGenerator;
53581ad6265SDimitry Andric for (auto &I : MemProfRecordData) {
53681ad6265SDimitry Andric // Insert the key (func hash) and value (memprof record).
53781ad6265SDimitry Andric RecordTableGenerator.insert(I.first, I.second);
53881ad6265SDimitry Andric }
5395f757f3fSDimitry Andric // Release the memory of this MapVector as it is no longer needed.
5405f757f3fSDimitry Andric MemProfRecordData.clear();
54181ad6265SDimitry Andric
5425f757f3fSDimitry Andric // The call to Emit invokes RecordWriterTrait::EmitData which destructs
5435f757f3fSDimitry Andric // the memprof record copies owned by the RecordTableGenerator. This works
5445f757f3fSDimitry Andric // because the RecordTableGenerator is not used after this point.
54581ad6265SDimitry Andric uint64_t RecordTableOffset =
54681ad6265SDimitry Andric RecordTableGenerator.Emit(OS.OS, *RecordWriter);
54781ad6265SDimitry Andric
54881ad6265SDimitry Andric uint64_t FramePayloadOffset = OS.tell();
54981ad6265SDimitry Andric
55081ad6265SDimitry Andric auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
55181ad6265SDimitry Andric OnDiskChainedHashTableGenerator<memprof::FrameWriterTrait>
55281ad6265SDimitry Andric FrameTableGenerator;
55381ad6265SDimitry Andric for (auto &I : MemProfFrameData) {
55481ad6265SDimitry Andric // Insert the key (frame id) and value (frame contents).
55581ad6265SDimitry Andric FrameTableGenerator.insert(I.first, I.second);
55681ad6265SDimitry Andric }
5575f757f3fSDimitry Andric // Release the memory of this MapVector as it is no longer needed.
5585f757f3fSDimitry Andric MemProfFrameData.clear();
55981ad6265SDimitry Andric
56081ad6265SDimitry Andric uint64_t FrameTableOffset = FrameTableGenerator.Emit(OS.OS, *FrameWriter);
56181ad6265SDimitry Andric
56281ad6265SDimitry Andric PatchItem PatchItems[] = {
56381ad6265SDimitry Andric {MemProfSectionStart, &RecordTableOffset, 1},
56481ad6265SDimitry Andric {MemProfSectionStart + sizeof(uint64_t), &FramePayloadOffset, 1},
56581ad6265SDimitry Andric {MemProfSectionStart + 2 * sizeof(uint64_t), &FrameTableOffset, 1},
56681ad6265SDimitry Andric };
56781ad6265SDimitry Andric OS.patch(PatchItems, 3);
56881ad6265SDimitry Andric }
56981ad6265SDimitry Andric
570bdd1243dSDimitry Andric // BinaryIdSection has two parts:
571bdd1243dSDimitry Andric // 1. uint64_t BinaryIdsSectionSize
572bdd1243dSDimitry Andric // 2. list of binary ids that consist of:
573bdd1243dSDimitry Andric // a. uint64_t BinaryIdLength
574bdd1243dSDimitry Andric // b. uint8_t BinaryIdData
575bdd1243dSDimitry Andric // c. uint8_t Padding (if necessary)
576bdd1243dSDimitry Andric uint64_t BinaryIdSectionStart = OS.tell();
577bdd1243dSDimitry Andric // Calculate size of binary section.
578bdd1243dSDimitry Andric uint64_t BinaryIdsSectionSize = 0;
579bdd1243dSDimitry Andric
580bdd1243dSDimitry Andric // Remove duplicate binary ids.
581bdd1243dSDimitry Andric llvm::sort(BinaryIds);
582bdd1243dSDimitry Andric BinaryIds.erase(std::unique(BinaryIds.begin(), BinaryIds.end()),
583bdd1243dSDimitry Andric BinaryIds.end());
584bdd1243dSDimitry Andric
585bdd1243dSDimitry Andric for (auto BI : BinaryIds) {
586bdd1243dSDimitry Andric // Increment by binary id length data type size.
587bdd1243dSDimitry Andric BinaryIdsSectionSize += sizeof(uint64_t);
588bdd1243dSDimitry Andric // Increment by binary id data length, aligned to 8 bytes.
589bdd1243dSDimitry Andric BinaryIdsSectionSize += alignToPowerOf2(BI.size(), sizeof(uint64_t));
590bdd1243dSDimitry Andric }
591bdd1243dSDimitry Andric // Write binary ids section size.
592bdd1243dSDimitry Andric OS.write(BinaryIdsSectionSize);
593bdd1243dSDimitry Andric
594bdd1243dSDimitry Andric for (auto BI : BinaryIds) {
595bdd1243dSDimitry Andric uint64_t BILen = BI.size();
596bdd1243dSDimitry Andric // Write binary id length.
597bdd1243dSDimitry Andric OS.write(BILen);
598bdd1243dSDimitry Andric // Write binary id data.
599bdd1243dSDimitry Andric for (unsigned K = 0; K < BILen; K++)
600bdd1243dSDimitry Andric OS.writeByte(BI[K]);
601bdd1243dSDimitry Andric // Write padding if necessary.
602bdd1243dSDimitry Andric uint64_t PaddingSize = alignToPowerOf2(BILen, sizeof(uint64_t)) - BILen;
603bdd1243dSDimitry Andric for (unsigned K = 0; K < PaddingSize; K++)
604bdd1243dSDimitry Andric OS.writeByte(0);
605bdd1243dSDimitry Andric }
606bdd1243dSDimitry Andric
60706c3fb27SDimitry Andric uint64_t TemporalProfTracesSectionStart = 0;
60806c3fb27SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
60906c3fb27SDimitry Andric TemporalProfTracesSectionStart = OS.tell();
61006c3fb27SDimitry Andric OS.write(TemporalProfTraces.size());
61106c3fb27SDimitry Andric OS.write(TemporalProfTraceStreamSize);
61206c3fb27SDimitry Andric for (auto &Trace : TemporalProfTraces) {
61306c3fb27SDimitry Andric OS.write(Trace.Weight);
61406c3fb27SDimitry Andric OS.write(Trace.FunctionNameRefs.size());
61506c3fb27SDimitry Andric for (auto &NameRef : Trace.FunctionNameRefs)
61606c3fb27SDimitry Andric OS.write(NameRef);
61706c3fb27SDimitry Andric }
61806c3fb27SDimitry Andric }
61906c3fb27SDimitry Andric
6200b57cec5SDimitry Andric // Allocate space for data to be serialized out.
6210b57cec5SDimitry Andric std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
6220b57cec5SDimitry Andric IndexedInstrProf::allocSummary(SummarySize);
6230b57cec5SDimitry Andric // Compute the Summary and copy the data to the data
6240b57cec5SDimitry Andric // structure to be serialized out (to disk or buffer).
6250b57cec5SDimitry Andric std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
6260b57cec5SDimitry Andric setSummary(TheSummary.get(), *PS);
6270b57cec5SDimitry Andric InfoObj->SummaryBuilder = nullptr;
6280b57cec5SDimitry Andric
6290b57cec5SDimitry Andric // For Context Sensitive summary.
6300b57cec5SDimitry Andric std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
63181ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
6320b57cec5SDimitry Andric TheCSSummary = IndexedInstrProf::allocSummary(SummarySize);
6330b57cec5SDimitry Andric std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
6340b57cec5SDimitry Andric setSummary(TheCSSummary.get(), *CSPS);
6350b57cec5SDimitry Andric }
6360b57cec5SDimitry Andric InfoObj->CSSummaryBuilder = nullptr;
6370b57cec5SDimitry Andric
6380b57cec5SDimitry Andric // Now do the final patch:
6390b57cec5SDimitry Andric PatchItem PatchItems[] = {
6400b57cec5SDimitry Andric // Patch the Header.HashOffset field.
6410b57cec5SDimitry Andric {HashTableStartFieldOffset, &HashTableStart, 1},
642bdd1243dSDimitry Andric // Patch the Header.MemProfOffset (=0 for profiles without MemProf
643bdd1243dSDimitry Andric // data).
64481ad6265SDimitry Andric {MemProfSectionOffset, &MemProfSectionStart, 1},
645bdd1243dSDimitry Andric // Patch the Header.BinaryIdSectionOffset.
646bdd1243dSDimitry Andric {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
64706c3fb27SDimitry Andric // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
64806c3fb27SDimitry Andric // traces).
64906c3fb27SDimitry Andric {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
6500b57cec5SDimitry Andric // Patch the summary data.
6510b57cec5SDimitry Andric {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
6520b57cec5SDimitry Andric (int)(SummarySize / sizeof(uint64_t))},
6530b57cec5SDimitry Andric {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()),
6540b57cec5SDimitry Andric (int)CSSummarySize}};
6550b57cec5SDimitry Andric
656bdd1243dSDimitry Andric OS.patch(PatchItems, std::size(PatchItems));
657fe6060f1SDimitry Andric
658fe6060f1SDimitry Andric for (const auto &I : FunctionData)
659fe6060f1SDimitry Andric for (const auto &F : I.getValue())
660fe6060f1SDimitry Andric if (Error E = validateRecord(F.second))
661fe6060f1SDimitry Andric return E;
662fe6060f1SDimitry Andric
663fe6060f1SDimitry Andric return Error::success();
6640b57cec5SDimitry Andric }
6650b57cec5SDimitry Andric
write(raw_fd_ostream & OS)666fe6060f1SDimitry Andric Error InstrProfWriter::write(raw_fd_ostream &OS) {
6670b57cec5SDimitry Andric // Write the hash table.
6680b57cec5SDimitry Andric ProfOStream POS(OS);
669fe6060f1SDimitry Andric return writeImpl(POS);
6700b57cec5SDimitry Andric }
6710b57cec5SDimitry Andric
write(raw_string_ostream & OS)67206c3fb27SDimitry Andric Error InstrProfWriter::write(raw_string_ostream &OS) {
67306c3fb27SDimitry Andric ProfOStream POS(OS);
67406c3fb27SDimitry Andric return writeImpl(POS);
67506c3fb27SDimitry Andric }
67606c3fb27SDimitry Andric
writeBuffer()6770b57cec5SDimitry Andric std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
6780b57cec5SDimitry Andric std::string Data;
6790b57cec5SDimitry Andric raw_string_ostream OS(Data);
6800b57cec5SDimitry Andric // Write the hash table.
68106c3fb27SDimitry Andric if (Error E = write(OS))
682fe6060f1SDimitry Andric return nullptr;
6830b57cec5SDimitry Andric // Return this in an aligned memory buffer.
6840b57cec5SDimitry Andric return MemoryBuffer::getMemBufferCopy(Data);
6850b57cec5SDimitry Andric }
6860b57cec5SDimitry Andric
6870b57cec5SDimitry Andric static const char *ValueProfKindStr[] = {
6880b57cec5SDimitry Andric #define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
6890b57cec5SDimitry Andric #include "llvm/ProfileData/InstrProfData.inc"
6900b57cec5SDimitry Andric };
6910b57cec5SDimitry Andric
validateRecord(const InstrProfRecord & Func)692fe6060f1SDimitry Andric Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
693fe6060f1SDimitry Andric for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
694fe6060f1SDimitry Andric uint32_t NS = Func.getNumValueSites(VK);
695fe6060f1SDimitry Andric if (!NS)
696fe6060f1SDimitry Andric continue;
697fe6060f1SDimitry Andric for (uint32_t S = 0; S < NS; S++) {
698fe6060f1SDimitry Andric uint32_t ND = Func.getNumValueDataForSite(VK, S);
699fe6060f1SDimitry Andric std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
700bdd1243dSDimitry Andric DenseSet<uint64_t> SeenValues;
701fe6060f1SDimitry Andric for (uint32_t I = 0; I < ND; I++)
702bdd1243dSDimitry Andric if ((VK != IPVK_IndirectCallTarget) && !SeenValues.insert(VD[I].Value).second)
703fe6060f1SDimitry Andric return make_error<InstrProfError>(instrprof_error::invalid_prof);
704fe6060f1SDimitry Andric }
705fe6060f1SDimitry Andric }
706fe6060f1SDimitry Andric
707fe6060f1SDimitry Andric return Error::success();
708fe6060f1SDimitry Andric }
709fe6060f1SDimitry Andric
writeRecordInText(StringRef Name,uint64_t Hash,const InstrProfRecord & Func,InstrProfSymtab & Symtab,raw_fd_ostream & OS)7100b57cec5SDimitry Andric void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
7110b57cec5SDimitry Andric const InstrProfRecord &Func,
7120b57cec5SDimitry Andric InstrProfSymtab &Symtab,
7130b57cec5SDimitry Andric raw_fd_ostream &OS) {
7140b57cec5SDimitry Andric OS << Name << "\n";
7150b57cec5SDimitry Andric OS << "# Func Hash:\n" << Hash << "\n";
7160b57cec5SDimitry Andric OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
7170b57cec5SDimitry Andric OS << "# Counter Values:\n";
7180b57cec5SDimitry Andric for (uint64_t Count : Func.Counts)
7190b57cec5SDimitry Andric OS << Count << "\n";
7200b57cec5SDimitry Andric
7215f757f3fSDimitry Andric if (Func.BitmapBytes.size() > 0) {
7225f757f3fSDimitry Andric OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n";
7235f757f3fSDimitry Andric OS << "# Bitmap Byte Values:\n";
7245f757f3fSDimitry Andric for (uint8_t Byte : Func.BitmapBytes) {
7255f757f3fSDimitry Andric OS << "0x";
7265f757f3fSDimitry Andric OS.write_hex(Byte);
7275f757f3fSDimitry Andric OS << "\n";
7285f757f3fSDimitry Andric }
7295f757f3fSDimitry Andric OS << "\n";
7305f757f3fSDimitry Andric }
7315f757f3fSDimitry Andric
7320b57cec5SDimitry Andric uint32_t NumValueKinds = Func.getNumValueKinds();
7330b57cec5SDimitry Andric if (!NumValueKinds) {
7340b57cec5SDimitry Andric OS << "\n";
7350b57cec5SDimitry Andric return;
7360b57cec5SDimitry Andric }
7370b57cec5SDimitry Andric
7380b57cec5SDimitry Andric OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
7390b57cec5SDimitry Andric for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
7400b57cec5SDimitry Andric uint32_t NS = Func.getNumValueSites(VK);
7410b57cec5SDimitry Andric if (!NS)
7420b57cec5SDimitry Andric continue;
7430b57cec5SDimitry Andric OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
7440b57cec5SDimitry Andric OS << "# NumValueSites:\n" << NS << "\n";
7450b57cec5SDimitry Andric for (uint32_t S = 0; S < NS; S++) {
7460b57cec5SDimitry Andric uint32_t ND = Func.getNumValueDataForSite(VK, S);
7470b57cec5SDimitry Andric OS << ND << "\n";
7480b57cec5SDimitry Andric std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
7490b57cec5SDimitry Andric for (uint32_t I = 0; I < ND; I++) {
7500b57cec5SDimitry Andric if (VK == IPVK_IndirectCallTarget)
7515f757f3fSDimitry Andric OS << Symtab.getFuncOrVarNameIfDefined(VD[I].Value) << ":"
7520b57cec5SDimitry Andric << VD[I].Count << "\n";
7530b57cec5SDimitry Andric else
7540b57cec5SDimitry Andric OS << VD[I].Value << ":" << VD[I].Count << "\n";
7550b57cec5SDimitry Andric }
7560b57cec5SDimitry Andric }
7570b57cec5SDimitry Andric }
7580b57cec5SDimitry Andric
7590b57cec5SDimitry Andric OS << "\n";
7600b57cec5SDimitry Andric }
7610b57cec5SDimitry Andric
writeText(raw_fd_ostream & OS)7620b57cec5SDimitry Andric Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
7631fd87a68SDimitry Andric // Check CS first since it implies an IR level profile.
76481ad6265SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
7650b57cec5SDimitry Andric OS << "# CSIR level Instrumentation Flag\n:csir\n";
76681ad6265SDimitry Andric else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
7671fd87a68SDimitry Andric OS << "# IR level Instrumentation Flag\n:ir\n";
7681fd87a68SDimitry Andric
76981ad6265SDimitry Andric if (static_cast<bool>(ProfileKind &
77081ad6265SDimitry Andric InstrProfKind::FunctionEntryInstrumentation))
771e8d8bef9SDimitry Andric OS << "# Always instrument the function entry block\n:entry_first\n";
7725f757f3fSDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
7735f757f3fSDimitry Andric OS << "# Instrument block coverage\n:single_byte_coverage\n";
7740b57cec5SDimitry Andric InstrProfSymtab Symtab;
7750b57cec5SDimitry Andric
7760b57cec5SDimitry Andric using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;
7770b57cec5SDimitry Andric using RecordType = std::pair<StringRef, FuncPair>;
7780b57cec5SDimitry Andric SmallVector<RecordType, 4> OrderedFuncData;
7790b57cec5SDimitry Andric
7800b57cec5SDimitry Andric for (const auto &I : FunctionData) {
7810b57cec5SDimitry Andric if (shouldEncodeData(I.getValue())) {
7820b57cec5SDimitry Andric if (Error E = Symtab.addFuncName(I.getKey()))
7830b57cec5SDimitry Andric return E;
7840b57cec5SDimitry Andric for (const auto &Func : I.getValue())
7850b57cec5SDimitry Andric OrderedFuncData.push_back(std::make_pair(I.getKey(), Func));
7860b57cec5SDimitry Andric }
7870b57cec5SDimitry Andric }
7880b57cec5SDimitry Andric
78906c3fb27SDimitry Andric if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
79006c3fb27SDimitry Andric writeTextTemporalProfTraceData(OS, Symtab);
79106c3fb27SDimitry Andric
7920b57cec5SDimitry Andric llvm::sort(OrderedFuncData, [](const RecordType &A, const RecordType &B) {
7930b57cec5SDimitry Andric return std::tie(A.first, A.second.first) <
7940b57cec5SDimitry Andric std::tie(B.first, B.second.first);
7950b57cec5SDimitry Andric });
7960b57cec5SDimitry Andric
7970b57cec5SDimitry Andric for (const auto &record : OrderedFuncData) {
7980b57cec5SDimitry Andric const StringRef &Name = record.first;
7990b57cec5SDimitry Andric const FuncPair &Func = record.second;
8000b57cec5SDimitry Andric writeRecordInText(Name, Func.first, Func.second, Symtab, OS);
8010b57cec5SDimitry Andric }
8020b57cec5SDimitry Andric
803fe6060f1SDimitry Andric for (const auto &record : OrderedFuncData) {
804fe6060f1SDimitry Andric const FuncPair &Func = record.second;
805fe6060f1SDimitry Andric if (Error E = validateRecord(Func.second))
806fe6060f1SDimitry Andric return E;
807fe6060f1SDimitry Andric }
808fe6060f1SDimitry Andric
8090b57cec5SDimitry Andric return Error::success();
8100b57cec5SDimitry Andric }
81106c3fb27SDimitry Andric
writeTextTemporalProfTraceData(raw_fd_ostream & OS,InstrProfSymtab & Symtab)81206c3fb27SDimitry Andric void InstrProfWriter::writeTextTemporalProfTraceData(raw_fd_ostream &OS,
81306c3fb27SDimitry Andric InstrProfSymtab &Symtab) {
81406c3fb27SDimitry Andric OS << ":temporal_prof_traces\n";
81506c3fb27SDimitry Andric OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";
81606c3fb27SDimitry Andric OS << "# Temporal Profile Trace Stream Size:\n"
81706c3fb27SDimitry Andric << TemporalProfTraceStreamSize << "\n";
81806c3fb27SDimitry Andric for (auto &Trace : TemporalProfTraces) {
81906c3fb27SDimitry Andric OS << "# Weight:\n" << Trace.Weight << "\n";
82006c3fb27SDimitry Andric for (auto &NameRef : Trace.FunctionNameRefs)
8215f757f3fSDimitry Andric OS << Symtab.getFuncOrVarName(NameRef) << ",";
82206c3fb27SDimitry Andric OS << "\n";
82306c3fb27SDimitry Andric }
82406c3fb27SDimitry Andric OS << "\n";
82506c3fb27SDimitry Andric }
826