1 //===-- InstrProfCorrelator.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ProfileData/InstrProfCorrelator.h" 10 #include "llvm/DebugInfo/DIContext.h" 11 #include "llvm/DebugInfo/DWARF/DWARFContext.h" 12 #include "llvm/DebugInfo/DWARF/DWARFDie.h" 13 #include "llvm/DebugInfo/DWARF/DWARFExpression.h" 14 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h" 15 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h" 16 #include "llvm/DebugInfo/DWARF/DWARFUnit.h" 17 #include "llvm/Object/MachO.h" 18 #include "llvm/Support/Debug.h" 19 20 #define DEBUG_TYPE "correlator" 21 22 using namespace llvm; 23 24 /// Get the __llvm_prf_cnts section. 25 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) { 26 for (auto &Section : Obj.sections()) 27 if (auto SectionName = Section.getName()) 28 if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME) 29 return Section; 30 return make_error<InstrProfError>( 31 instrprof_error::unable_to_correlate_profile, 32 "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")"); 33 } 34 35 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name"; 36 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash"; 37 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; 38 39 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>> 40 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer, 41 const object::ObjectFile &Obj) { 42 auto CountersSection = getCountersSection(Obj); 43 if (auto Err = CountersSection.takeError()) 44 return std::move(Err); 45 auto C = std::make_unique<Context>(); 46 C->Buffer = std::move(Buffer); 47 C->CountersSectionStart = CountersSection->getAddress(); 48 C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize(); 49 C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; 50 return Expected<std::unique_ptr<Context>>(std::move(C)); 51 } 52 53 llvm::Expected<std::unique_ptr<InstrProfCorrelator>> 54 InstrProfCorrelator::get(StringRef DebugInfoFilename) { 55 auto DsymObjectsOrErr = 56 object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename); 57 if (auto Err = DsymObjectsOrErr.takeError()) 58 return std::move(Err); 59 if (!DsymObjectsOrErr->empty()) { 60 // TODO: Enable profile correlation when there are multiple objects in a 61 // dSYM bundle. 62 if (DsymObjectsOrErr->size() > 1) 63 return make_error<InstrProfError>( 64 instrprof_error::unable_to_correlate_profile, 65 "using multiple objects is not yet supported"); 66 DebugInfoFilename = *DsymObjectsOrErr->begin(); 67 } 68 auto BufferOrErr = 69 errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename)); 70 if (auto Err = BufferOrErr.takeError()) 71 return std::move(Err); 72 73 return get(std::move(*BufferOrErr)); 74 } 75 76 llvm::Expected<std::unique_ptr<InstrProfCorrelator>> 77 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) { 78 auto BinOrErr = object::createBinary(*Buffer); 79 if (auto Err = BinOrErr.takeError()) 80 return std::move(Err); 81 82 if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) { 83 auto CtxOrErr = Context::get(std::move(Buffer), *Obj); 84 if (auto Err = CtxOrErr.takeError()) 85 return std::move(Err); 86 auto T = Obj->makeTriple(); 87 if (T.isArch64Bit()) 88 return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj); 89 if (T.isArch32Bit()) 90 return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj); 91 } 92 return make_error<InstrProfError>( 93 instrprof_error::unable_to_correlate_profile, "not an object file"); 94 } 95 96 Optional<size_t> InstrProfCorrelator::getDataSize() const { 97 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) { 98 return C->getDataSize(); 99 } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) { 100 return C->getDataSize(); 101 } 102 return {}; 103 } 104 105 namespace llvm { 106 107 template <> 108 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl( 109 std::unique_ptr<InstrProfCorrelator::Context> Ctx) 110 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit, 111 std::move(Ctx)) {} 112 template <> 113 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl( 114 std::unique_ptr<InstrProfCorrelator::Context> Ctx) 115 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit, 116 std::move(Ctx)) {} 117 template <> 118 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) { 119 return C->getKind() == InstrProfCorrelatorKind::CK_32Bit; 120 } 121 template <> 122 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) { 123 return C->getKind() == InstrProfCorrelatorKind::CK_64Bit; 124 } 125 126 } // end namespace llvm 127 128 template <class IntPtrT> 129 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>> 130 InstrProfCorrelatorImpl<IntPtrT>::get( 131 std::unique_ptr<InstrProfCorrelator::Context> Ctx, 132 const object::ObjectFile &Obj) { 133 if (Obj.isELF() || Obj.isMachO()) { 134 auto DICtx = DWARFContext::create(Obj); 135 return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx), 136 std::move(Ctx)); 137 } 138 return make_error<InstrProfError>( 139 instrprof_error::unable_to_correlate_profile, 140 "unsupported debug info format (only DWARF is supported)"); 141 } 142 143 template <class IntPtrT> 144 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() { 145 assert(Data.empty() && Names.empty() && NamesVec.empty()); 146 correlateProfileDataImpl(); 147 if (Data.empty() || NamesVec.empty()) 148 return make_error<InstrProfError>( 149 instrprof_error::unable_to_correlate_profile, 150 "could not find any profile metadata in debug info"); 151 auto Result = 152 collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names); 153 CounterOffsets.clear(); 154 NamesVec.clear(); 155 return Result; 156 } 157 158 template <class IntPtrT> 159 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName, 160 uint64_t CFGHash, 161 IntPtrT CounterOffset, 162 IntPtrT FunctionPtr, 163 uint32_t NumCounters) { 164 // Check if a probe was already added for this counter offset. 165 if (!CounterOffsets.insert(CounterOffset).second) 166 return; 167 Data.push_back({ 168 maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)), 169 maybeSwap<uint64_t>(CFGHash), 170 // In this mode, CounterPtr actually stores the section relative address 171 // of the counter. 172 maybeSwap<IntPtrT>(CounterOffset), 173 maybeSwap<IntPtrT>(FunctionPtr), 174 // TODO: Value profiling is not yet supported. 175 /*ValuesPtr=*/maybeSwap<IntPtrT>(0), 176 maybeSwap<uint32_t>(NumCounters), 177 /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)}, 178 }); 179 NamesVec.push_back(FunctionName.str()); 180 } 181 182 template <class IntPtrT> 183 llvm::Optional<uint64_t> 184 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const { 185 auto Locations = Die.getLocations(dwarf::DW_AT_location); 186 if (!Locations) { 187 consumeError(Locations.takeError()); 188 return {}; 189 } 190 auto &DU = *Die.getDwarfUnit(); 191 auto AddressSize = DU.getAddressByteSize(); 192 for (auto &Location : *Locations) { 193 DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize); 194 DWARFExpression Expr(Data, AddressSize); 195 for (auto &Op : Expr) { 196 if (Op.getCode() == dwarf::DW_OP_addr) { 197 return Op.getRawOperand(0); 198 } else if (Op.getCode() == dwarf::DW_OP_addrx) { 199 uint64_t Index = Op.getRawOperand(0); 200 if (auto SA = DU.getAddrOffsetSectionItem(Index)) 201 return SA->Address; 202 } 203 } 204 } 205 return {}; 206 } 207 208 template <class IntPtrT> 209 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) { 210 const auto &ParentDie = Die.getParent(); 211 if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL()) 212 return false; 213 if (Die.getTag() != dwarf::DW_TAG_variable) 214 return false; 215 if (!ParentDie.isSubprogramDIE()) 216 return false; 217 if (!Die.hasChildren()) 218 return false; 219 if (const char *Name = Die.getName(DINameKind::ShortName)) 220 return StringRef(Name).startswith(getInstrProfCountersVarPrefix()); 221 return false; 222 } 223 224 template <class IntPtrT> 225 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl() { 226 auto maybeAddProbe = [&](DWARFDie Die) { 227 if (!isDIEOfProbe(Die)) 228 return; 229 Optional<const char *> FunctionName; 230 Optional<uint64_t> CFGHash; 231 Optional<uint64_t> CounterPtr = getLocation(Die); 232 auto FunctionPtr = 233 dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc)); 234 Optional<uint64_t> NumCounters; 235 for (const DWARFDie &Child : Die.children()) { 236 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation) 237 continue; 238 auto AnnotationFormName = Child.find(dwarf::DW_AT_name); 239 auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value); 240 if (!AnnotationFormName || !AnnotationFormValue) 241 continue; 242 auto AnnotationNameOrErr = AnnotationFormName->getAsCString(); 243 if (auto Err = AnnotationNameOrErr.takeError()) { 244 consumeError(std::move(Err)); 245 continue; 246 } 247 StringRef AnnotationName = *AnnotationNameOrErr; 248 if (AnnotationName.compare( 249 InstrProfCorrelator::FunctionNameAttributeName) == 0) { 250 if (auto EC = 251 AnnotationFormValue->getAsCString().moveInto(FunctionName)) 252 consumeError(std::move(EC)); 253 } else if (AnnotationName.compare( 254 InstrProfCorrelator::CFGHashAttributeName) == 0) { 255 CFGHash = AnnotationFormValue->getAsUnsignedConstant(); 256 } else if (AnnotationName.compare( 257 InstrProfCorrelator::NumCountersAttributeName) == 0) { 258 NumCounters = AnnotationFormValue->getAsUnsignedConstant(); 259 } 260 } 261 if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) { 262 LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: " 263 << FunctionName << "\n\tCFGHash: " << CFGHash 264 << "\n\tCounterPtr: " << CounterPtr 265 << "\n\tNumCounters: " << NumCounters); 266 LLVM_DEBUG(Die.dump(dbgs())); 267 return; 268 } 269 uint64_t CountersStart = this->Ctx->CountersSectionStart; 270 uint64_t CountersEnd = this->Ctx->CountersSectionEnd; 271 if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) { 272 LLVM_DEBUG( 273 dbgs() << "CounterPtr out of range for probe\n\tFunction Name: " 274 << FunctionName << "\n\tExpected: [0x" 275 << Twine::utohexstr(CountersStart) << ", 0x" 276 << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x" 277 << Twine::utohexstr(*CounterPtr)); 278 LLVM_DEBUG(Die.dump(dbgs())); 279 return; 280 } 281 if (!FunctionPtr) { 282 LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName 283 << "\n"); 284 LLVM_DEBUG(Die.dump(dbgs())); 285 } 286 this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart, 287 FunctionPtr.value_or(0), *NumCounters); 288 }; 289 for (auto &CU : DICtx->normal_units()) 290 for (const auto &Entry : CU->dies()) 291 maybeAddProbe(DWARFDie(CU.get(), &Entry)); 292 for (auto &CU : DICtx->dwo_units()) 293 for (const auto &Entry : CU->dies()) 294 maybeAddProbe(DWARFDie(CU.get(), &Entry)); 295 } 296