1 //===-- InstrProfCorrelator.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ProfileData/InstrProfCorrelator.h" 10 #include "llvm/Object/MachO.h" 11 #include "llvm/Support/Debug.h" 12 #include "llvm/Support/FileSystem.h" 13 #include "llvm/Support/Path.h" 14 15 #define DEBUG_TYPE "correlator" 16 17 using namespace llvm; 18 19 /// Get the __llvm_prf_cnts section. 20 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) { 21 for (auto &Section : Obj.sections()) 22 if (auto SectionName = Section.getName()) 23 if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME) 24 return Section; 25 return make_error<InstrProfError>( 26 instrprof_error::unable_to_correlate_profile, 27 "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")"); 28 } 29 30 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name"; 31 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash"; 32 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; 33 34 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>> 35 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer, 36 const object::ObjectFile &Obj) { 37 auto CountersSection = getCountersSection(Obj); 38 if (auto Err = CountersSection.takeError()) 39 return std::move(Err); 40 auto C = std::make_unique<Context>(); 41 C->Buffer = std::move(Buffer); 42 C->CountersSectionStart = CountersSection->getAddress(); 43 C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize(); 44 C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; 45 return Expected<std::unique_ptr<Context>>(std::move(C)); 46 } 47 48 llvm::Expected<std::unique_ptr<InstrProfCorrelator>> 49 InstrProfCorrelator::get(StringRef DebugInfoFilename) { 50 auto DsymObjectsOrErr = 51 object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename); 52 if (auto Err = DsymObjectsOrErr.takeError()) 53 return std::move(Err); 54 if (!DsymObjectsOrErr->empty()) { 55 // TODO: Enable profile correlation when there are multiple objects in a 56 // dSYM bundle. 57 if (DsymObjectsOrErr->size() > 1) 58 return make_error<InstrProfError>( 59 instrprof_error::unable_to_correlate_profile, 60 "using multiple objects is not yet supported"); 61 DebugInfoFilename = *DsymObjectsOrErr->begin(); 62 } 63 auto BufferOrErr = 64 errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename)); 65 if (auto Err = BufferOrErr.takeError()) 66 return std::move(Err); 67 68 return get(std::move(*BufferOrErr)); 69 } 70 71 llvm::Expected<std::unique_ptr<InstrProfCorrelator>> 72 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) { 73 auto BinOrErr = object::createBinary(*Buffer); 74 if (auto Err = BinOrErr.takeError()) 75 return std::move(Err); 76 77 if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) { 78 auto CtxOrErr = Context::get(std::move(Buffer), *Obj); 79 if (auto Err = CtxOrErr.takeError()) 80 return std::move(Err); 81 auto T = Obj->makeTriple(); 82 if (T.isArch64Bit()) 83 return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj); 84 if (T.isArch32Bit()) 85 return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj); 86 } 87 return make_error<InstrProfError>( 88 instrprof_error::unable_to_correlate_profile, "not an object file"); 89 } 90 91 Optional<size_t> InstrProfCorrelator::getDataSize() const { 92 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) { 93 return C->getDataSize(); 94 } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) { 95 return C->getDataSize(); 96 } 97 return {}; 98 } 99 100 namespace llvm { 101 102 template <> 103 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl( 104 std::unique_ptr<InstrProfCorrelator::Context> Ctx) 105 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit, 106 std::move(Ctx)) {} 107 template <> 108 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl( 109 std::unique_ptr<InstrProfCorrelator::Context> Ctx) 110 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit, 111 std::move(Ctx)) {} 112 template <> 113 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) { 114 return C->getKind() == InstrProfCorrelatorKind::CK_32Bit; 115 } 116 template <> 117 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) { 118 return C->getKind() == InstrProfCorrelatorKind::CK_64Bit; 119 } 120 121 } // end namespace llvm 122 123 template <class IntPtrT> 124 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>> 125 InstrProfCorrelatorImpl<IntPtrT>::get( 126 std::unique_ptr<InstrProfCorrelator::Context> Ctx, 127 const object::ObjectFile &Obj) { 128 if (Obj.isELF() || Obj.isMachO()) { 129 auto DICtx = DWARFContext::create(Obj); 130 return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx), 131 std::move(Ctx)); 132 } 133 return make_error<InstrProfError>( 134 instrprof_error::unable_to_correlate_profile, 135 "unsupported debug info format (only DWARF is supported)"); 136 } 137 138 template <class IntPtrT> 139 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() { 140 assert(Data.empty() && Names.empty() && NamesVec.empty()); 141 correlateProfileDataImpl(); 142 if (Data.empty() || NamesVec.empty()) 143 return make_error<InstrProfError>( 144 instrprof_error::unable_to_correlate_profile, 145 "could not find any profile metadata in debug info"); 146 auto Result = 147 collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names); 148 CounterOffsets.clear(); 149 NamesVec.clear(); 150 return Result; 151 } 152 153 template <class IntPtrT> 154 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName, 155 uint64_t CFGHash, 156 IntPtrT CounterOffset, 157 IntPtrT FunctionPtr, 158 uint32_t NumCounters) { 159 // Check if a probe was already added for this counter offset. 160 if (!CounterOffsets.insert(CounterOffset).second) 161 return; 162 Data.push_back({ 163 maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)), 164 maybeSwap<uint64_t>(CFGHash), 165 // In this mode, CounterPtr actually stores the section relative address 166 // of the counter. 167 maybeSwap<IntPtrT>(CounterOffset), 168 maybeSwap<IntPtrT>(FunctionPtr), 169 // TODO: Value profiling is not yet supported. 170 /*ValuesPtr=*/maybeSwap<IntPtrT>(0), 171 maybeSwap<uint32_t>(NumCounters), 172 /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)}, 173 }); 174 NamesVec.push_back(FunctionName.str()); 175 } 176 177 template <class IntPtrT> 178 llvm::Optional<uint64_t> 179 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const { 180 auto Locations = Die.getLocations(dwarf::DW_AT_location); 181 if (!Locations) { 182 consumeError(Locations.takeError()); 183 return {}; 184 } 185 auto &DU = *Die.getDwarfUnit(); 186 auto AddressSize = DU.getAddressByteSize(); 187 for (auto &Location : *Locations) { 188 DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize); 189 DWARFExpression Expr(Data, AddressSize); 190 for (auto &Op : Expr) { 191 if (Op.getCode() == dwarf::DW_OP_addr) { 192 return Op.getRawOperand(0); 193 } else if (Op.getCode() == dwarf::DW_OP_addrx) { 194 uint64_t Index = Op.getRawOperand(0); 195 if (auto SA = DU.getAddrOffsetSectionItem(Index)) 196 return SA->Address; 197 } 198 } 199 } 200 return {}; 201 } 202 203 template <class IntPtrT> 204 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) { 205 const auto &ParentDie = Die.getParent(); 206 if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL()) 207 return false; 208 if (Die.getTag() != dwarf::DW_TAG_variable) 209 return false; 210 if (!ParentDie.isSubprogramDIE()) 211 return false; 212 if (!Die.hasChildren()) 213 return false; 214 if (const char *Name = Die.getName(DINameKind::ShortName)) 215 return StringRef(Name).startswith(getInstrProfCountersVarPrefix()); 216 return false; 217 } 218 219 template <class IntPtrT> 220 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl() { 221 auto maybeAddProbe = [&](DWARFDie Die) { 222 if (!isDIEOfProbe(Die)) 223 return; 224 Optional<const char *> FunctionName; 225 Optional<uint64_t> CFGHash; 226 Optional<uint64_t> CounterPtr = getLocation(Die); 227 auto FunctionPtr = 228 dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc)); 229 Optional<uint64_t> NumCounters; 230 for (const DWARFDie &Child : Die.children()) { 231 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation) 232 continue; 233 auto AnnotationFormName = Child.find(dwarf::DW_AT_name); 234 auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value); 235 if (!AnnotationFormName || !AnnotationFormValue) 236 continue; 237 auto AnnotationNameOrErr = AnnotationFormName->getAsCString(); 238 if (auto Err = AnnotationNameOrErr.takeError()) { 239 consumeError(std::move(Err)); 240 continue; 241 } 242 StringRef AnnotationName = *AnnotationNameOrErr; 243 if (AnnotationName.compare( 244 InstrProfCorrelator::FunctionNameAttributeName) == 0) { 245 if (auto EC = 246 AnnotationFormValue->getAsCString().moveInto(FunctionName)) 247 consumeError(std::move(EC)); 248 } else if (AnnotationName.compare( 249 InstrProfCorrelator::CFGHashAttributeName) == 0) { 250 CFGHash = AnnotationFormValue->getAsUnsignedConstant(); 251 } else if (AnnotationName.compare( 252 InstrProfCorrelator::NumCountersAttributeName) == 0) { 253 NumCounters = AnnotationFormValue->getAsUnsignedConstant(); 254 } 255 } 256 if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) { 257 LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: " 258 << FunctionName << "\n\tCFGHash: " << CFGHash 259 << "\n\tCounterPtr: " << CounterPtr 260 << "\n\tNumCounters: " << NumCounters); 261 LLVM_DEBUG(Die.dump(dbgs())); 262 return; 263 } 264 uint64_t CountersStart = this->Ctx->CountersSectionStart; 265 uint64_t CountersEnd = this->Ctx->CountersSectionEnd; 266 if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) { 267 LLVM_DEBUG( 268 dbgs() << "CounterPtr out of range for probe\n\tFunction Name: " 269 << FunctionName << "\n\tExpected: [0x" 270 << Twine::utohexstr(CountersStart) << ", 0x" 271 << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x" 272 << Twine::utohexstr(*CounterPtr)); 273 LLVM_DEBUG(Die.dump(dbgs())); 274 return; 275 } 276 if (!FunctionPtr) { 277 LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName 278 << "\n"); 279 LLVM_DEBUG(Die.dump(dbgs())); 280 } 281 this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart, 282 FunctionPtr.getValueOr(0), *NumCounters); 283 }; 284 for (auto &CU : DICtx->normal_units()) 285 for (const auto &Entry : CU->dies()) 286 maybeAddProbe(DWARFDie(CU.get(), &Entry)); 287 for (auto &CU : DICtx->dwo_units()) 288 for (const auto &Entry : CU->dies()) 289 maybeAddProbe(DWARFDie(CU.get(), &Entry)); 290 } 291