1 //===-- InstrProfCorrelator.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ProfileData/InstrProfCorrelator.h"
10 #include "llvm/Object/MachO.h"
11 #include "llvm/Support/Debug.h"
12 #include "llvm/Support/FileSystem.h"
13 #include "llvm/Support/Path.h"
14 
15 #define DEBUG_TYPE "correlator"
16 
17 using namespace llvm;
18 
19 /// Get the __llvm_prf_cnts section.
20 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) {
21   for (auto &Section : Obj.sections())
22     if (auto SectionName = Section.getName())
23       if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME)
24         return Section;
25   return make_error<InstrProfError>(
26       instrprof_error::unable_to_correlate_profile,
27       "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")");
28 }
29 
30 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
31 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
32 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
33 
34 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
35 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
36                                   const object::ObjectFile &Obj) {
37   auto CountersSection = getCountersSection(Obj);
38   if (auto Err = CountersSection.takeError())
39     return std::move(Err);
40   auto C = std::make_unique<Context>();
41   C->Buffer = std::move(Buffer);
42   C->CountersSectionStart = CountersSection->getAddress();
43   C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
44   C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
45   return Expected<std::unique_ptr<Context>>(std::move(C));
46 }
47 
48 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
49 InstrProfCorrelator::get(StringRef DebugInfoFilename) {
50   auto DsymObjectsOrErr =
51       object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename);
52   if (auto Err = DsymObjectsOrErr.takeError())
53     return std::move(Err);
54   if (!DsymObjectsOrErr->empty()) {
55     // TODO: Enable profile correlation when there are multiple objects in a
56     // dSYM bundle.
57     if (DsymObjectsOrErr->size() > 1)
58       return make_error<InstrProfError>(
59           instrprof_error::unable_to_correlate_profile,
60           "using multiple objects is not yet supported");
61     DebugInfoFilename = *DsymObjectsOrErr->begin();
62   }
63   auto BufferOrErr =
64       errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename));
65   if (auto Err = BufferOrErr.takeError())
66     return std::move(Err);
67 
68   return get(std::move(*BufferOrErr));
69 }
70 
71 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
72 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) {
73   auto BinOrErr = object::createBinary(*Buffer);
74   if (auto Err = BinOrErr.takeError())
75     return std::move(Err);
76 
77   if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
78     auto CtxOrErr = Context::get(std::move(Buffer), *Obj);
79     if (auto Err = CtxOrErr.takeError())
80       return std::move(Err);
81     auto T = Obj->makeTriple();
82     if (T.isArch64Bit())
83       return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj);
84     if (T.isArch32Bit())
85       return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj);
86   }
87   return make_error<InstrProfError>(
88       instrprof_error::unable_to_correlate_profile, "not an object file");
89 }
90 
91 Optional<size_t> InstrProfCorrelator::getDataSize() const {
92   if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) {
93     return C->getDataSize();
94   } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) {
95     return C->getDataSize();
96   }
97   return {};
98 }
99 
100 namespace llvm {
101 
102 template <>
103 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
104     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
105     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
106                               std::move(Ctx)) {}
107 template <>
108 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
109     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
110     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
111                               std::move(Ctx)) {}
112 template <>
113 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
114   return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
115 }
116 template <>
117 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
118   return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
119 }
120 
121 } // end namespace llvm
122 
123 template <class IntPtrT>
124 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
125 InstrProfCorrelatorImpl<IntPtrT>::get(
126     std::unique_ptr<InstrProfCorrelator::Context> Ctx,
127     const object::ObjectFile &Obj) {
128   if (Obj.isELF() || Obj.isMachO()) {
129     auto DICtx = DWARFContext::create(Obj);
130     return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx),
131                                                                std::move(Ctx));
132   }
133   return make_error<InstrProfError>(
134       instrprof_error::unable_to_correlate_profile,
135       "unsupported debug info format (only DWARF is supported)");
136 }
137 
138 template <class IntPtrT>
139 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() {
140   assert(Data.empty() && Names.empty() && NamesVec.empty());
141   correlateProfileDataImpl();
142   if (Data.empty() || NamesVec.empty())
143     return make_error<InstrProfError>(
144         instrprof_error::unable_to_correlate_profile,
145         "could not find any profile metadata in debug info");
146   auto Result =
147       collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names);
148   CounterOffsets.clear();
149   NamesVec.clear();
150   return Result;
151 }
152 
153 template <class IntPtrT>
154 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
155                                                 uint64_t CFGHash,
156                                                 IntPtrT CounterOffset,
157                                                 IntPtrT FunctionPtr,
158                                                 uint32_t NumCounters) {
159   // Check if a probe was already added for this counter offset.
160   if (!CounterOffsets.insert(CounterOffset).second)
161     return;
162   Data.push_back({
163       maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)),
164       maybeSwap<uint64_t>(CFGHash),
165       // In this mode, CounterPtr actually stores the section relative address
166       // of the counter.
167       maybeSwap<IntPtrT>(CounterOffset),
168       maybeSwap<IntPtrT>(FunctionPtr),
169       // TODO: Value profiling is not yet supported.
170       /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
171       maybeSwap<uint32_t>(NumCounters),
172       /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
173   });
174   NamesVec.push_back(FunctionName.str());
175 }
176 
177 template <class IntPtrT>
178 llvm::Optional<uint64_t>
179 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
180   auto Locations = Die.getLocations(dwarf::DW_AT_location);
181   if (!Locations) {
182     consumeError(Locations.takeError());
183     return {};
184   }
185   auto &DU = *Die.getDwarfUnit();
186   auto AddressSize = DU.getAddressByteSize();
187   for (auto &Location : *Locations) {
188     DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
189     DWARFExpression Expr(Data, AddressSize);
190     for (auto &Op : Expr) {
191       if (Op.getCode() == dwarf::DW_OP_addr) {
192         return Op.getRawOperand(0);
193       } else if (Op.getCode() == dwarf::DW_OP_addrx) {
194         uint64_t Index = Op.getRawOperand(0);
195         if (auto SA = DU.getAddrOffsetSectionItem(Index))
196           return SA->Address;
197       }
198     }
199   }
200   return {};
201 }
202 
203 template <class IntPtrT>
204 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
205   const auto &ParentDie = Die.getParent();
206   if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
207     return false;
208   if (Die.getTag() != dwarf::DW_TAG_variable)
209     return false;
210   if (!ParentDie.isSubprogramDIE())
211     return false;
212   if (!Die.hasChildren())
213     return false;
214   if (const char *Name = Die.getName(DINameKind::ShortName))
215     return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
216   return false;
217 }
218 
219 template <class IntPtrT>
220 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl() {
221   auto maybeAddProbe = [&](DWARFDie Die) {
222     if (!isDIEOfProbe(Die))
223       return;
224     Optional<const char *> FunctionName;
225     Optional<uint64_t> CFGHash;
226     Optional<uint64_t> CounterPtr = getLocation(Die);
227     auto FunctionPtr =
228         dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc));
229     Optional<uint64_t> NumCounters;
230     for (const DWARFDie &Child : Die.children()) {
231       if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
232         continue;
233       auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
234       auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
235       if (!AnnotationFormName || !AnnotationFormValue)
236         continue;
237       auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
238       if (auto Err = AnnotationNameOrErr.takeError()) {
239         consumeError(std::move(Err));
240         continue;
241       }
242       StringRef AnnotationName = *AnnotationNameOrErr;
243       if (AnnotationName.compare(
244               InstrProfCorrelator::FunctionNameAttributeName) == 0) {
245         if (auto EC =
246                 AnnotationFormValue->getAsCString().moveInto(FunctionName))
247           consumeError(std::move(EC));
248       } else if (AnnotationName.compare(
249                      InstrProfCorrelator::CFGHashAttributeName) == 0) {
250         CFGHash = AnnotationFormValue->getAsUnsignedConstant();
251       } else if (AnnotationName.compare(
252                      InstrProfCorrelator::NumCountersAttributeName) == 0) {
253         NumCounters = AnnotationFormValue->getAsUnsignedConstant();
254       }
255     }
256     if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
257       LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: "
258                         << FunctionName << "\n\tCFGHash: " << CFGHash
259                         << "\n\tCounterPtr: " << CounterPtr
260                         << "\n\tNumCounters: " << NumCounters);
261       LLVM_DEBUG(Die.dump(dbgs()));
262       return;
263     }
264     uint64_t CountersStart = this->Ctx->CountersSectionStart;
265     uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
266     if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
267       LLVM_DEBUG(
268           dbgs() << "CounterPtr out of range for probe\n\tFunction Name: "
269                  << FunctionName << "\n\tExpected: [0x"
270                  << Twine::utohexstr(CountersStart) << ", 0x"
271                  << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x"
272                  << Twine::utohexstr(*CounterPtr));
273       LLVM_DEBUG(Die.dump(dbgs()));
274       return;
275     }
276     if (!FunctionPtr) {
277       LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName
278                         << "\n");
279       LLVM_DEBUG(Die.dump(dbgs()));
280     }
281     this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart,
282                    FunctionPtr.getValueOr(0), *NumCounters);
283   };
284   for (auto &CU : DICtx->normal_units())
285     for (const auto &Entry : CU->dies())
286       maybeAddProbe(DWARFDie(CU.get(), &Entry));
287   for (auto &CU : DICtx->dwo_units())
288     for (const auto &Entry : CU->dies())
289       maybeAddProbe(DWARFDie(CU.get(), &Entry));
290 }
291