1 //===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements functions and classes used to support LTO.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/LTO/LTO.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/TargetLibraryInfo.h"
17 #include "llvm/Analysis/TargetTransformInfo.h"
18 #include "llvm/Bitcode/BitcodeReader.h"
19 #include "llvm/Bitcode/BitcodeWriter.h"
20 #include "llvm/CodeGen/Analysis.h"
21 #include "llvm/Config/llvm-config.h"
22 #include "llvm/IR/AutoUpgrade.h"
23 #include "llvm/IR/DiagnosticPrinter.h"
24 #include "llvm/IR/LegacyPassManager.h"
25 #include "llvm/IR/Mangler.h"
26 #include "llvm/IR/Metadata.h"
27 #include "llvm/LTO/LTOBackend.h"
28 #include "llvm/LTO/SummaryBasedOptimizations.h"
29 #include "llvm/Linker/IRMover.h"
30 #include "llvm/Object/IRObjectFile.h"
31 #include "llvm/Support/Error.h"
32 #include "llvm/Support/ManagedStatic.h"
33 #include "llvm/Support/MemoryBuffer.h"
34 #include "llvm/Support/Path.h"
35 #include "llvm/Support/SHA1.h"
36 #include "llvm/Support/SourceMgr.h"
37 #include "llvm/Support/TargetRegistry.h"
38 #include "llvm/Support/ThreadPool.h"
39 #include "llvm/Support/Threading.h"
40 #include "llvm/Support/VCSRevision.h"
41 #include "llvm/Support/raw_ostream.h"
42 #include "llvm/Target/TargetMachine.h"
43 #include "llvm/Target/TargetOptions.h"
44 #include "llvm/Transforms/IPO.h"
45 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
46 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
47 #include "llvm/Transforms/Utils/SplitModule.h"
48
49 #include <set>
50
51 using namespace llvm;
52 using namespace lto;
53 using namespace object;
54
55 #define DEBUG_TYPE "lto"
56
57 static cl::opt<bool>
58 DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden,
59 cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
60
61 /// Enable global value internalization in LTO.
62 cl::opt<bool> EnableLTOInternalization(
63 "enable-lto-internalization", cl::init(true), cl::Hidden,
64 cl::desc("Enable global value internalization in LTO"));
65
66 // Computes a unique hash for the Module considering the current list of
67 // export/import and other global analysis results.
68 // The hash is produced in \p Key.
computeLTOCacheKey(SmallString<40> & Key,const Config & Conf,const ModuleSummaryIndex & Index,StringRef ModuleID,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const std::map<GlobalValue::GUID,GlobalValue::LinkageTypes> & ResolvedODR,const GVSummaryMapTy & DefinedGlobals,const std::set<GlobalValue::GUID> & CfiFunctionDefs,const std::set<GlobalValue::GUID> & CfiFunctionDecls)69 void llvm::computeLTOCacheKey(
70 SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index,
71 StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
72 const FunctionImporter::ExportSetTy &ExportList,
73 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
74 const GVSummaryMapTy &DefinedGlobals,
75 const std::set<GlobalValue::GUID> &CfiFunctionDefs,
76 const std::set<GlobalValue::GUID> &CfiFunctionDecls) {
77 // Compute the unique hash for this entry.
78 // This is based on the current compiler version, the module itself, the
79 // export list, the hash for every single module in the import list, the
80 // list of ResolvedODR for the module, and the list of preserved symbols.
81 SHA1 Hasher;
82
83 // Start with the compiler revision
84 Hasher.update(LLVM_VERSION_STRING);
85 #ifdef LLVM_REVISION
86 Hasher.update(LLVM_REVISION);
87 #endif
88
89 // Include the parts of the LTO configuration that affect code generation.
90 auto AddString = [&](StringRef Str) {
91 Hasher.update(Str);
92 Hasher.update(ArrayRef<uint8_t>{0});
93 };
94 auto AddUnsigned = [&](unsigned I) {
95 uint8_t Data[4];
96 Data[0] = I;
97 Data[1] = I >> 8;
98 Data[2] = I >> 16;
99 Data[3] = I >> 24;
100 Hasher.update(ArrayRef<uint8_t>{Data, 4});
101 };
102 auto AddUint64 = [&](uint64_t I) {
103 uint8_t Data[8];
104 Data[0] = I;
105 Data[1] = I >> 8;
106 Data[2] = I >> 16;
107 Data[3] = I >> 24;
108 Data[4] = I >> 32;
109 Data[5] = I >> 40;
110 Data[6] = I >> 48;
111 Data[7] = I >> 56;
112 Hasher.update(ArrayRef<uint8_t>{Data, 8});
113 };
114 AddString(Conf.CPU);
115 // FIXME: Hash more of Options. For now all clients initialize Options from
116 // command-line flags (which is unsupported in production), but may set
117 // RelaxELFRelocations. The clang driver can also pass FunctionSections,
118 // DataSections and DebuggerTuning via command line flags.
119 AddUnsigned(Conf.Options.RelaxELFRelocations);
120 AddUnsigned(Conf.Options.FunctionSections);
121 AddUnsigned(Conf.Options.DataSections);
122 AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
123 for (auto &A : Conf.MAttrs)
124 AddString(A);
125 if (Conf.RelocModel)
126 AddUnsigned(*Conf.RelocModel);
127 else
128 AddUnsigned(-1);
129 if (Conf.CodeModel)
130 AddUnsigned(*Conf.CodeModel);
131 else
132 AddUnsigned(-1);
133 AddUnsigned(Conf.CGOptLevel);
134 AddUnsigned(Conf.CGFileType);
135 AddUnsigned(Conf.OptLevel);
136 AddUnsigned(Conf.UseNewPM);
137 AddUnsigned(Conf.Freestanding);
138 AddString(Conf.OptPipeline);
139 AddString(Conf.AAPipeline);
140 AddString(Conf.OverrideTriple);
141 AddString(Conf.DefaultTriple);
142 AddString(Conf.DwoDir);
143
144 // Include the hash for the current module
145 auto ModHash = Index.getModuleHash(ModuleID);
146 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
147 for (auto F : ExportList)
148 // The export list can impact the internalization, be conservative here
149 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&F, sizeof(F)));
150
151 // Include the hash for every module we import functions from. The set of
152 // imported symbols for each module may affect code generation and is
153 // sensitive to link order, so include that as well.
154 for (auto &Entry : ImportList) {
155 auto ModHash = Index.getModuleHash(Entry.first());
156 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
157
158 AddUint64(Entry.second.size());
159 for (auto &Fn : Entry.second)
160 AddUint64(Fn);
161 }
162
163 // Include the hash for the resolved ODR.
164 for (auto &Entry : ResolvedODR) {
165 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
166 sizeof(GlobalValue::GUID)));
167 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
168 sizeof(GlobalValue::LinkageTypes)));
169 }
170
171 // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
172 // defined in this module.
173 std::set<GlobalValue::GUID> UsedCfiDefs;
174 std::set<GlobalValue::GUID> UsedCfiDecls;
175
176 // Typeids used in this module.
177 std::set<GlobalValue::GUID> UsedTypeIds;
178
179 auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
180 if (CfiFunctionDefs.count(ValueGUID))
181 UsedCfiDefs.insert(ValueGUID);
182 if (CfiFunctionDecls.count(ValueGUID))
183 UsedCfiDecls.insert(ValueGUID);
184 };
185
186 auto AddUsedThings = [&](GlobalValueSummary *GS) {
187 if (!GS) return;
188 AddUnsigned(GS->isLive());
189 for (const ValueInfo &VI : GS->refs()) {
190 AddUnsigned(VI.isDSOLocal());
191 AddUsedCfiGlobal(VI.getGUID());
192 }
193 if (auto *GVS = dyn_cast<GlobalVarSummary>(GS))
194 AddUnsigned(GVS->isReadOnly());
195 if (auto *FS = dyn_cast<FunctionSummary>(GS)) {
196 for (auto &TT : FS->type_tests())
197 UsedTypeIds.insert(TT);
198 for (auto &TT : FS->type_test_assume_vcalls())
199 UsedTypeIds.insert(TT.GUID);
200 for (auto &TT : FS->type_checked_load_vcalls())
201 UsedTypeIds.insert(TT.GUID);
202 for (auto &TT : FS->type_test_assume_const_vcalls())
203 UsedTypeIds.insert(TT.VFunc.GUID);
204 for (auto &TT : FS->type_checked_load_const_vcalls())
205 UsedTypeIds.insert(TT.VFunc.GUID);
206 for (auto &ET : FS->calls()) {
207 AddUnsigned(ET.first.isDSOLocal());
208 AddUsedCfiGlobal(ET.first.getGUID());
209 }
210 }
211 };
212
213 // Include the hash for the linkage type to reflect internalization and weak
214 // resolution, and collect any used type identifier resolutions.
215 for (auto &GS : DefinedGlobals) {
216 GlobalValue::LinkageTypes Linkage = GS.second->linkage();
217 Hasher.update(
218 ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
219 AddUsedCfiGlobal(GS.first);
220 AddUsedThings(GS.second);
221 }
222
223 // Imported functions may introduce new uses of type identifier resolutions,
224 // so we need to collect their used resolutions as well.
225 for (auto &ImpM : ImportList)
226 for (auto &ImpF : ImpM.second) {
227 GlobalValueSummary *S = Index.findSummaryInModule(ImpF, ImpM.first());
228 AddUsedThings(S);
229 // If this is an alias, we also care about any types/etc. that the aliasee
230 // may reference.
231 if (auto *AS = dyn_cast_or_null<AliasSummary>(S))
232 AddUsedThings(AS->getBaseObject());
233 }
234
235 auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
236 AddString(TId);
237
238 AddUnsigned(S.TTRes.TheKind);
239 AddUnsigned(S.TTRes.SizeM1BitWidth);
240
241 AddUint64(S.TTRes.AlignLog2);
242 AddUint64(S.TTRes.SizeM1);
243 AddUint64(S.TTRes.BitMask);
244 AddUint64(S.TTRes.InlineBits);
245
246 AddUint64(S.WPDRes.size());
247 for (auto &WPD : S.WPDRes) {
248 AddUnsigned(WPD.first);
249 AddUnsigned(WPD.second.TheKind);
250 AddString(WPD.second.SingleImplName);
251
252 AddUint64(WPD.second.ResByArg.size());
253 for (auto &ByArg : WPD.second.ResByArg) {
254 AddUint64(ByArg.first.size());
255 for (uint64_t Arg : ByArg.first)
256 AddUint64(Arg);
257 AddUnsigned(ByArg.second.TheKind);
258 AddUint64(ByArg.second.Info);
259 AddUnsigned(ByArg.second.Byte);
260 AddUnsigned(ByArg.second.Bit);
261 }
262 }
263 };
264
265 // Include the hash for all type identifiers used by this module.
266 for (GlobalValue::GUID TId : UsedTypeIds) {
267 auto TidIter = Index.typeIds().equal_range(TId);
268 for (auto It = TidIter.first; It != TidIter.second; ++It)
269 AddTypeIdSummary(It->second.first, It->second.second);
270 }
271
272 AddUnsigned(UsedCfiDefs.size());
273 for (auto &V : UsedCfiDefs)
274 AddUint64(V);
275
276 AddUnsigned(UsedCfiDecls.size());
277 for (auto &V : UsedCfiDecls)
278 AddUint64(V);
279
280 if (!Conf.SampleProfile.empty()) {
281 auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
282 if (FileOrErr) {
283 Hasher.update(FileOrErr.get()->getBuffer());
284
285 if (!Conf.ProfileRemapping.empty()) {
286 FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);
287 if (FileOrErr)
288 Hasher.update(FileOrErr.get()->getBuffer());
289 }
290 }
291 }
292
293 Key = toHex(Hasher.result());
294 }
295
thinLTOResolvePrevailingGUID(GlobalValueSummaryList & GVSummaryList,GlobalValue::GUID GUID,DenseSet<GlobalValueSummary * > & GlobalInvolvedWithAlias,function_ref<bool (GlobalValue::GUID,const GlobalValueSummary *)> isPrevailing,function_ref<void (StringRef,GlobalValue::GUID,GlobalValue::LinkageTypes)> recordNewLinkage)296 static void thinLTOResolvePrevailingGUID(
297 GlobalValueSummaryList &GVSummaryList, GlobalValue::GUID GUID,
298 DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
299 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
300 isPrevailing,
301 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
302 recordNewLinkage) {
303 for (auto &S : GVSummaryList) {
304 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
305 // Ignore local and appending linkage values since the linker
306 // doesn't resolve them.
307 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
308 GlobalValue::isAppendingLinkage(S->linkage()))
309 continue;
310 // We need to emit only one of these. The prevailing module will keep it,
311 // but turned into a weak, while the others will drop it when possible.
312 // This is both a compile-time optimization and a correctness
313 // transformation. This is necessary for correctness when we have exported
314 // a reference - we need to convert the linkonce to weak to
315 // ensure a copy is kept to satisfy the exported reference.
316 // FIXME: We may want to split the compile time and correctness
317 // aspects into separate routines.
318 if (isPrevailing(GUID, S.get())) {
319 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))
320 S->setLinkage(GlobalValue::getWeakLinkage(
321 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
322 }
323 // Alias and aliasee can't be turned into available_externally.
324 else if (!isa<AliasSummary>(S.get()) &&
325 !GlobalInvolvedWithAlias.count(S.get()))
326 S->setLinkage(GlobalValue::AvailableExternallyLinkage);
327 if (S->linkage() != OriginalLinkage)
328 recordNewLinkage(S->modulePath(), GUID, S->linkage());
329 }
330 }
331
332 /// Resolve linkage for prevailing symbols in the \p Index.
333 //
334 // We'd like to drop these functions if they are no longer referenced in the
335 // current module. However there is a chance that another module is still
336 // referencing them because of the import. We make sure we always emit at least
337 // one copy.
thinLTOResolvePrevailingInIndex(ModuleSummaryIndex & Index,function_ref<bool (GlobalValue::GUID,const GlobalValueSummary *)> isPrevailing,function_ref<void (StringRef,GlobalValue::GUID,GlobalValue::LinkageTypes)> recordNewLinkage)338 void llvm::thinLTOResolvePrevailingInIndex(
339 ModuleSummaryIndex &Index,
340 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
341 isPrevailing,
342 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
343 recordNewLinkage) {
344 // We won't optimize the globals that are referenced by an alias for now
345 // Ideally we should turn the alias into a global and duplicate the definition
346 // when needed.
347 DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
348 for (auto &I : Index)
349 for (auto &S : I.second.SummaryList)
350 if (auto AS = dyn_cast<AliasSummary>(S.get()))
351 GlobalInvolvedWithAlias.insert(&AS->getAliasee());
352
353 for (auto &I : Index)
354 thinLTOResolvePrevailingGUID(I.second.SummaryList, I.first,
355 GlobalInvolvedWithAlias, isPrevailing,
356 recordNewLinkage);
357 }
358
thinLTOInternalizeAndPromoteGUID(GlobalValueSummaryList & GVSummaryList,GlobalValue::GUID GUID,function_ref<bool (StringRef,GlobalValue::GUID)> isExported)359 static void thinLTOInternalizeAndPromoteGUID(
360 GlobalValueSummaryList &GVSummaryList, GlobalValue::GUID GUID,
361 function_ref<bool(StringRef, GlobalValue::GUID)> isExported) {
362 for (auto &S : GVSummaryList) {
363 if (isExported(S->modulePath(), GUID)) {
364 if (GlobalValue::isLocalLinkage(S->linkage()))
365 S->setLinkage(GlobalValue::ExternalLinkage);
366 } else if (EnableLTOInternalization &&
367 // Ignore local and appending linkage values since the linker
368 // doesn't resolve them.
369 !GlobalValue::isLocalLinkage(S->linkage()) &&
370 S->linkage() != GlobalValue::AppendingLinkage &&
371 // We can't internalize available_externally globals because this
372 // can break function pointer equality.
373 S->linkage() != GlobalValue::AvailableExternallyLinkage)
374 S->setLinkage(GlobalValue::InternalLinkage);
375 }
376 }
377
378 // Update the linkages in the given \p Index to mark exported values
379 // as external and non-exported values as internal.
thinLTOInternalizeAndPromoteInIndex(ModuleSummaryIndex & Index,function_ref<bool (StringRef,GlobalValue::GUID)> isExported)380 void llvm::thinLTOInternalizeAndPromoteInIndex(
381 ModuleSummaryIndex &Index,
382 function_ref<bool(StringRef, GlobalValue::GUID)> isExported) {
383 for (auto &I : Index)
384 thinLTOInternalizeAndPromoteGUID(I.second.SummaryList, I.first, isExported);
385 }
386
387 // Requires a destructor for std::vector<InputModule>.
388 InputFile::~InputFile() = default;
389
create(MemoryBufferRef Object)390 Expected<std::unique_ptr<InputFile>> InputFile::create(MemoryBufferRef Object) {
391 std::unique_ptr<InputFile> File(new InputFile);
392
393 Expected<IRSymtabFile> FOrErr = readIRSymtab(Object);
394 if (!FOrErr)
395 return FOrErr.takeError();
396
397 File->TargetTriple = FOrErr->TheReader.getTargetTriple();
398 File->SourceFileName = FOrErr->TheReader.getSourceFileName();
399 File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();
400 File->ComdatTable = FOrErr->TheReader.getComdatTable();
401
402 for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {
403 size_t Begin = File->Symbols.size();
404 for (const irsymtab::Reader::SymbolRef &Sym :
405 FOrErr->TheReader.module_symbols(I))
406 // Skip symbols that are irrelevant to LTO. Note that this condition needs
407 // to match the one in Skip() in LTO::addRegularLTO().
408 if (Sym.isGlobal() && !Sym.isFormatSpecific())
409 File->Symbols.push_back(Sym);
410 File->ModuleSymIndices.push_back({Begin, File->Symbols.size()});
411 }
412
413 File->Mods = FOrErr->Mods;
414 File->Strtab = std::move(FOrErr->Strtab);
415 return std::move(File);
416 }
417
getName() const418 StringRef InputFile::getName() const {
419 return Mods[0].getModuleIdentifier();
420 }
421
RegularLTOState(unsigned ParallelCodeGenParallelismLevel,Config & Conf)422 LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
423 Config &Conf)
424 : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
425 Ctx(Conf), CombinedModule(llvm::make_unique<Module>("ld-temp.o", Ctx)),
426 Mover(llvm::make_unique<IRMover>(*CombinedModule)) {}
427
ThinLTOState(ThinBackend Backend)428 LTO::ThinLTOState::ThinLTOState(ThinBackend Backend)
429 : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) {
430 if (!Backend)
431 this->Backend =
432 createInProcessThinBackend(llvm::heavyweight_hardware_concurrency());
433 }
434
LTO(Config Conf,ThinBackend Backend,unsigned ParallelCodeGenParallelismLevel)435 LTO::LTO(Config Conf, ThinBackend Backend,
436 unsigned ParallelCodeGenParallelismLevel)
437 : Conf(std::move(Conf)),
438 RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
439 ThinLTO(std::move(Backend)) {}
440
441 // Requires a destructor for MapVector<BitcodeModule>.
442 LTO::~LTO() = default;
443
444 // Add the symbols in the given module to the GlobalResolutions map, and resolve
445 // their partitions.
addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,ArrayRef<SymbolResolution> Res,unsigned Partition,bool InSummary)446 void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
447 ArrayRef<SymbolResolution> Res,
448 unsigned Partition, bool InSummary) {
449 auto *ResI = Res.begin();
450 auto *ResE = Res.end();
451 (void)ResE;
452 for (const InputFile::Symbol &Sym : Syms) {
453 assert(ResI != ResE);
454 SymbolResolution Res = *ResI++;
455
456 StringRef Name = Sym.getName();
457 Triple TT(RegularLTO.CombinedModule->getTargetTriple());
458 // Strip the __imp_ prefix from COFF dllimport symbols (similar to the
459 // way they are handled by lld), otherwise we can end up with two
460 // global resolutions (one with and one for a copy of the symbol without).
461 if (TT.isOSBinFormatCOFF() && Name.startswith("__imp_"))
462 Name = Name.substr(strlen("__imp_"));
463 auto &GlobalRes = GlobalResolutions[Name];
464 GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
465 if (Res.Prevailing) {
466 assert(!GlobalRes.Prevailing &&
467 "Multiple prevailing defs are not allowed");
468 GlobalRes.Prevailing = true;
469 GlobalRes.IRName = Sym.getIRName();
470 } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {
471 // Sometimes it can be two copies of symbol in a module and prevailing
472 // symbol can have no IR name. That might happen if symbol is defined in
473 // module level inline asm block. In case we have multiple modules with
474 // the same symbol we want to use IR name of the prevailing symbol.
475 // Otherwise, if we haven't seen a prevailing symbol, set the name so that
476 // we can later use it to check if there is any prevailing copy in IR.
477 GlobalRes.IRName = Sym.getIRName();
478 }
479
480 // Set the partition to external if we know it is re-defined by the linker
481 // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
482 // regular object, is referenced from llvm.compiler_used, or was already
483 // recorded as being referenced from a different partition.
484 if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||
485 (GlobalRes.Partition != GlobalResolution::Unknown &&
486 GlobalRes.Partition != Partition)) {
487 GlobalRes.Partition = GlobalResolution::External;
488 } else
489 // First recorded reference, save the current partition.
490 GlobalRes.Partition = Partition;
491
492 // Flag as visible outside of summary if visible from a regular object or
493 // from a module that does not have a summary.
494 GlobalRes.VisibleOutsideSummary |=
495 (Res.VisibleToRegularObj || Sym.isUsed() || !InSummary);
496 }
497 }
498
writeToResolutionFile(raw_ostream & OS,InputFile * Input,ArrayRef<SymbolResolution> Res)499 static void writeToResolutionFile(raw_ostream &OS, InputFile *Input,
500 ArrayRef<SymbolResolution> Res) {
501 StringRef Path = Input->getName();
502 OS << Path << '\n';
503 auto ResI = Res.begin();
504 for (const InputFile::Symbol &Sym : Input->symbols()) {
505 assert(ResI != Res.end());
506 SymbolResolution Res = *ResI++;
507
508 OS << "-r=" << Path << ',' << Sym.getName() << ',';
509 if (Res.Prevailing)
510 OS << 'p';
511 if (Res.FinalDefinitionInLinkageUnit)
512 OS << 'l';
513 if (Res.VisibleToRegularObj)
514 OS << 'x';
515 if (Res.LinkerRedefined)
516 OS << 'r';
517 OS << '\n';
518 }
519 OS.flush();
520 assert(ResI == Res.end());
521 }
522
add(std::unique_ptr<InputFile> Input,ArrayRef<SymbolResolution> Res)523 Error LTO::add(std::unique_ptr<InputFile> Input,
524 ArrayRef<SymbolResolution> Res) {
525 assert(!CalledGetMaxTasks);
526
527 if (Conf.ResolutionFile)
528 writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res);
529
530 if (RegularLTO.CombinedModule->getTargetTriple().empty())
531 RegularLTO.CombinedModule->setTargetTriple(Input->getTargetTriple());
532
533 const SymbolResolution *ResI = Res.begin();
534 for (unsigned I = 0; I != Input->Mods.size(); ++I)
535 if (Error Err = addModule(*Input, I, ResI, Res.end()))
536 return Err;
537
538 assert(ResI == Res.end());
539 return Error::success();
540 }
541
addModule(InputFile & Input,unsigned ModI,const SymbolResolution * & ResI,const SymbolResolution * ResE)542 Error LTO::addModule(InputFile &Input, unsigned ModI,
543 const SymbolResolution *&ResI,
544 const SymbolResolution *ResE) {
545 Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();
546 if (!LTOInfo)
547 return LTOInfo.takeError();
548
549 if (EnableSplitLTOUnit.hasValue()) {
550 // If only some modules were split, flag this in the index so that
551 // we can skip or error on optimizations that need consistently split
552 // modules (whole program devirt and lower type tests).
553 if (EnableSplitLTOUnit.getValue() != LTOInfo->EnableSplitLTOUnit)
554 ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
555 } else
556 EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
557
558 BitcodeModule BM = Input.Mods[ModI];
559 auto ModSyms = Input.module_symbols(ModI);
560 addModuleToGlobalRes(ModSyms, {ResI, ResE},
561 LTOInfo->IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,
562 LTOInfo->HasSummary);
563
564 if (LTOInfo->IsThinLTO)
565 return addThinLTO(BM, ModSyms, ResI, ResE);
566
567 Expected<RegularLTOState::AddedModule> ModOrErr =
568 addRegularLTO(BM, ModSyms, ResI, ResE);
569 if (!ModOrErr)
570 return ModOrErr.takeError();
571
572 if (!LTOInfo->HasSummary)
573 return linkRegularLTO(std::move(*ModOrErr), /*LivenessFromIndex=*/false);
574
575 // Regular LTO module summaries are added to a dummy module that represents
576 // the combined regular LTO module.
577 if (Error Err = BM.readSummary(ThinLTO.CombinedIndex, "", -1ull))
578 return Err;
579 RegularLTO.ModsWithSummaries.push_back(std::move(*ModOrErr));
580 return Error::success();
581 }
582
583 // Checks whether the given global value is in a non-prevailing comdat
584 // (comdat containing values the linker indicated were not prevailing,
585 // which we then dropped to available_externally), and if so, removes
586 // it from the comdat. This is called for all global values to ensure the
587 // comdat is empty rather than leaving an incomplete comdat. It is needed for
588 // regular LTO modules, in case we are in a mixed-LTO mode (both regular
589 // and thin LTO modules) compilation. Since the regular LTO module will be
590 // linked first in the final native link, we want to make sure the linker
591 // doesn't select any of these incomplete comdats that would be left
592 // in the regular LTO module without this cleanup.
593 static void
handleNonPrevailingComdat(GlobalValue & GV,std::set<const Comdat * > & NonPrevailingComdats)594 handleNonPrevailingComdat(GlobalValue &GV,
595 std::set<const Comdat *> &NonPrevailingComdats) {
596 Comdat *C = GV.getComdat();
597 if (!C)
598 return;
599
600 if (!NonPrevailingComdats.count(C))
601 return;
602
603 // Additionally need to drop externally visible global values from the comdat
604 // to available_externally, so that there aren't multiply defined linker
605 // errors.
606 if (!GV.hasLocalLinkage())
607 GV.setLinkage(GlobalValue::AvailableExternallyLinkage);
608
609 if (auto GO = dyn_cast<GlobalObject>(&GV))
610 GO->setComdat(nullptr);
611 }
612
613 // Add a regular LTO object to the link.
614 // The resulting module needs to be linked into the combined LTO module with
615 // linkRegularLTO.
616 Expected<LTO::RegularLTOState::AddedModule>
addRegularLTO(BitcodeModule BM,ArrayRef<InputFile::Symbol> Syms,const SymbolResolution * & ResI,const SymbolResolution * ResE)617 LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
618 const SymbolResolution *&ResI,
619 const SymbolResolution *ResE) {
620 RegularLTOState::AddedModule Mod;
621 Expected<std::unique_ptr<Module>> MOrErr =
622 BM.getLazyModule(RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
623 /*IsImporting*/ false);
624 if (!MOrErr)
625 return MOrErr.takeError();
626 Module &M = **MOrErr;
627 Mod.M = std::move(*MOrErr);
628
629 if (Error Err = M.materializeMetadata())
630 return std::move(Err);
631 UpgradeDebugInfo(M);
632
633 ModuleSymbolTable SymTab;
634 SymTab.addModule(&M);
635
636 for (GlobalVariable &GV : M.globals())
637 if (GV.hasAppendingLinkage())
638 Mod.Keep.push_back(&GV);
639
640 DenseSet<GlobalObject *> AliasedGlobals;
641 for (auto &GA : M.aliases())
642 if (GlobalObject *GO = GA.getBaseObject())
643 AliasedGlobals.insert(GO);
644
645 // In this function we need IR GlobalValues matching the symbols in Syms
646 // (which is not backed by a module), so we need to enumerate them in the same
647 // order. The symbol enumeration order of a ModuleSymbolTable intentionally
648 // matches the order of an irsymtab, but when we read the irsymtab in
649 // InputFile::create we omit some symbols that are irrelevant to LTO. The
650 // Skip() function skips the same symbols from the module as InputFile does
651 // from the symbol table.
652 auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();
653 auto Skip = [&]() {
654 while (MsymI != MsymE) {
655 auto Flags = SymTab.getSymbolFlags(*MsymI);
656 if ((Flags & object::BasicSymbolRef::SF_Global) &&
657 !(Flags & object::BasicSymbolRef::SF_FormatSpecific))
658 return;
659 ++MsymI;
660 }
661 };
662 Skip();
663
664 std::set<const Comdat *> NonPrevailingComdats;
665 for (const InputFile::Symbol &Sym : Syms) {
666 assert(ResI != ResE);
667 SymbolResolution Res = *ResI++;
668
669 assert(MsymI != MsymE);
670 ModuleSymbolTable::Symbol Msym = *MsymI++;
671 Skip();
672
673 if (GlobalValue *GV = Msym.dyn_cast<GlobalValue *>()) {
674 if (Res.Prevailing) {
675 if (Sym.isUndefined())
676 continue;
677 Mod.Keep.push_back(GV);
678 // For symbols re-defined with linker -wrap and -defsym options,
679 // set the linkage to weak to inhibit IPO. The linkage will be
680 // restored by the linker.
681 if (Res.LinkerRedefined)
682 GV->setLinkage(GlobalValue::WeakAnyLinkage);
683
684 GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();
685 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))
686 GV->setLinkage(GlobalValue::getWeakLinkage(
687 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
688 } else if (isa<GlobalObject>(GV) &&
689 (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||
690 GV->hasAvailableExternallyLinkage()) &&
691 !AliasedGlobals.count(cast<GlobalObject>(GV))) {
692 // Any of the above three types of linkage indicates that the
693 // chosen prevailing symbol will have the same semantics as this copy of
694 // the symbol, so we may be able to link it with available_externally
695 // linkage. We will decide later whether to do that when we link this
696 // module (in linkRegularLTO), based on whether it is undefined.
697 Mod.Keep.push_back(GV);
698 GV->setLinkage(GlobalValue::AvailableExternallyLinkage);
699 if (GV->hasComdat())
700 NonPrevailingComdats.insert(GV->getComdat());
701 cast<GlobalObject>(GV)->setComdat(nullptr);
702 }
703
704 // Set the 'local' flag based on the linker resolution for this symbol.
705 if (Res.FinalDefinitionInLinkageUnit) {
706 GV->setDSOLocal(true);
707 if (GV->hasDLLImportStorageClass())
708 GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
709 DefaultStorageClass);
710 }
711 }
712 // Common resolution: collect the maximum size/alignment over all commons.
713 // We also record if we see an instance of a common as prevailing, so that
714 // if none is prevailing we can ignore it later.
715 if (Sym.isCommon()) {
716 // FIXME: We should figure out what to do about commons defined by asm.
717 // For now they aren't reported correctly by ModuleSymbolTable.
718 auto &CommonRes = RegularLTO.Commons[Sym.getIRName()];
719 CommonRes.Size = std::max(CommonRes.Size, Sym.getCommonSize());
720 CommonRes.Align = std::max(CommonRes.Align, Sym.getCommonAlignment());
721 CommonRes.Prevailing |= Res.Prevailing;
722 }
723
724 }
725 if (!M.getComdatSymbolTable().empty())
726 for (GlobalValue &GV : M.global_values())
727 handleNonPrevailingComdat(GV, NonPrevailingComdats);
728 assert(MsymI == MsymE);
729 return std::move(Mod);
730 }
731
linkRegularLTO(RegularLTOState::AddedModule Mod,bool LivenessFromIndex)732 Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
733 bool LivenessFromIndex) {
734 std::vector<GlobalValue *> Keep;
735 for (GlobalValue *GV : Mod.Keep) {
736 if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GV->getGUID()))
737 continue;
738
739 if (!GV->hasAvailableExternallyLinkage()) {
740 Keep.push_back(GV);
741 continue;
742 }
743
744 // Only link available_externally definitions if we don't already have a
745 // definition.
746 GlobalValue *CombinedGV =
747 RegularLTO.CombinedModule->getNamedValue(GV->getName());
748 if (CombinedGV && !CombinedGV->isDeclaration())
749 continue;
750
751 Keep.push_back(GV);
752 }
753
754 return RegularLTO.Mover->move(std::move(Mod.M), Keep,
755 [](GlobalValue &, IRMover::ValueAdder) {},
756 /* IsPerformingImport */ false);
757 }
758
759 // Add a ThinLTO module to the link.
addThinLTO(BitcodeModule BM,ArrayRef<InputFile::Symbol> Syms,const SymbolResolution * & ResI,const SymbolResolution * ResE)760 Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
761 const SymbolResolution *&ResI,
762 const SymbolResolution *ResE) {
763 if (Error Err =
764 BM.readSummary(ThinLTO.CombinedIndex, BM.getModuleIdentifier(),
765 ThinLTO.ModuleMap.size()))
766 return Err;
767
768 for (const InputFile::Symbol &Sym : Syms) {
769 assert(ResI != ResE);
770 SymbolResolution Res = *ResI++;
771
772 if (!Sym.getIRName().empty()) {
773 auto GUID = GlobalValue::getGUID(GlobalValue::getGlobalIdentifier(
774 Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
775 if (Res.Prevailing) {
776 ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier();
777
778 // For linker redefined symbols (via --wrap or --defsym) we want to
779 // switch the linkage to `weak` to prevent IPOs from happening.
780 // Find the summary in the module for this very GV and record the new
781 // linkage so that we can switch it when we import the GV.
782 if (Res.LinkerRedefined)
783 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
784 GUID, BM.getModuleIdentifier()))
785 S->setLinkage(GlobalValue::WeakAnyLinkage);
786 }
787
788 // If the linker resolved the symbol to a local definition then mark it
789 // as local in the summary for the module we are adding.
790 if (Res.FinalDefinitionInLinkageUnit) {
791 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
792 GUID, BM.getModuleIdentifier())) {
793 S->setDSOLocal(true);
794 }
795 }
796 }
797 }
798
799 if (!ThinLTO.ModuleMap.insert({BM.getModuleIdentifier(), BM}).second)
800 return make_error<StringError>(
801 "Expected at most one ThinLTO module per bitcode file",
802 inconvertibleErrorCode());
803
804 return Error::success();
805 }
806
getMaxTasks() const807 unsigned LTO::getMaxTasks() const {
808 CalledGetMaxTasks = true;
809 return RegularLTO.ParallelCodeGenParallelismLevel + ThinLTO.ModuleMap.size();
810 }
811
run(AddStreamFn AddStream,NativeObjectCache Cache)812 Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) {
813 // Compute "dead" symbols, we don't want to import/export these!
814 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
815 DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
816 for (auto &Res : GlobalResolutions) {
817 // Normally resolution have IR name of symbol. We can do nothing here
818 // otherwise. See comments in GlobalResolution struct for more details.
819 if (Res.second.IRName.empty())
820 continue;
821
822 GlobalValue::GUID GUID = GlobalValue::getGUID(
823 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
824
825 if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)
826 GUIDPreservedSymbols.insert(GlobalValue::getGUID(
827 GlobalValue::dropLLVMManglingEscape(Res.second.IRName)));
828
829 GUIDPrevailingResolutions[GUID] =
830 Res.second.Prevailing ? PrevailingType::Yes : PrevailingType::No;
831 }
832
833 auto isPrevailing = [&](GlobalValue::GUID G) {
834 auto It = GUIDPrevailingResolutions.find(G);
835 if (It == GUIDPrevailingResolutions.end())
836 return PrevailingType::Unknown;
837 return It->second;
838 };
839 computeDeadSymbolsWithConstProp(ThinLTO.CombinedIndex, GUIDPreservedSymbols,
840 isPrevailing, Conf.OptLevel > 0);
841
842 // Setup output file to emit statistics.
843 std::unique_ptr<ToolOutputFile> StatsFile = nullptr;
844 if (!Conf.StatsFile.empty()) {
845 EnableStatistics(false);
846 std::error_code EC;
847 StatsFile =
848 llvm::make_unique<ToolOutputFile>(Conf.StatsFile, EC, sys::fs::F_None);
849 if (EC)
850 return errorCodeToError(EC);
851 StatsFile->keep();
852 }
853
854 Error Result = runRegularLTO(AddStream);
855 if (!Result)
856 Result = runThinLTO(AddStream, Cache);
857
858 if (StatsFile)
859 PrintStatisticsJSON(StatsFile->os());
860
861 return Result;
862 }
863
runRegularLTO(AddStreamFn AddStream)864 Error LTO::runRegularLTO(AddStreamFn AddStream) {
865 for (auto &M : RegularLTO.ModsWithSummaries)
866 if (Error Err = linkRegularLTO(std::move(M),
867 /*LivenessFromIndex=*/true))
868 return Err;
869
870 // Make sure commons have the right size/alignment: we kept the largest from
871 // all the prevailing when adding the inputs, and we apply it here.
872 const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
873 for (auto &I : RegularLTO.Commons) {
874 if (!I.second.Prevailing)
875 // Don't do anything if no instance of this common was prevailing.
876 continue;
877 GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(I.first);
878 if (OldGV && DL.getTypeAllocSize(OldGV->getValueType()) == I.second.Size) {
879 // Don't create a new global if the type is already correct, just make
880 // sure the alignment is correct.
881 OldGV->setAlignment(I.second.Align);
882 continue;
883 }
884 ArrayType *Ty =
885 ArrayType::get(Type::getInt8Ty(RegularLTO.Ctx), I.second.Size);
886 auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
887 GlobalValue::CommonLinkage,
888 ConstantAggregateZero::get(Ty), "");
889 GV->setAlignment(I.second.Align);
890 if (OldGV) {
891 OldGV->replaceAllUsesWith(ConstantExpr::getBitCast(GV, OldGV->getType()));
892 GV->takeName(OldGV);
893 OldGV->eraseFromParent();
894 } else {
895 GV->setName(I.first);
896 }
897 }
898
899 if (Conf.PreOptModuleHook &&
900 !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
901 return Error::success();
902
903 if (!Conf.CodeGenOnly) {
904 for (const auto &R : GlobalResolutions) {
905 if (!R.second.isPrevailingIRSymbol())
906 continue;
907 if (R.second.Partition != 0 &&
908 R.second.Partition != GlobalResolution::External)
909 continue;
910
911 GlobalValue *GV =
912 RegularLTO.CombinedModule->getNamedValue(R.second.IRName);
913 // Ignore symbols defined in other partitions.
914 // Also skip declarations, which are not allowed to have internal linkage.
915 if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())
916 continue;
917 GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
918 : GlobalValue::UnnamedAddr::None);
919 if (EnableLTOInternalization && R.second.Partition == 0)
920 GV->setLinkage(GlobalValue::InternalLinkage);
921 }
922
923 if (Conf.PostInternalizeModuleHook &&
924 !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
925 return Error::success();
926 }
927 return backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
928 std::move(RegularLTO.CombinedModule), ThinLTO.CombinedIndex);
929 }
930
931 /// This class defines the interface to the ThinLTO backend.
932 class lto::ThinBackendProc {
933 protected:
934 Config &Conf;
935 ModuleSummaryIndex &CombinedIndex;
936 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries;
937
938 public:
ThinBackendProc(Config & Conf,ModuleSummaryIndex & CombinedIndex,const StringMap<GVSummaryMapTy> & ModuleToDefinedGVSummaries)939 ThinBackendProc(Config &Conf, ModuleSummaryIndex &CombinedIndex,
940 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries)
941 : Conf(Conf), CombinedIndex(CombinedIndex),
942 ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries) {}
943
~ThinBackendProc()944 virtual ~ThinBackendProc() {}
945 virtual Error start(
946 unsigned Task, BitcodeModule BM,
947 const FunctionImporter::ImportMapTy &ImportList,
948 const FunctionImporter::ExportSetTy &ExportList,
949 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
950 MapVector<StringRef, BitcodeModule> &ModuleMap) = 0;
951 virtual Error wait() = 0;
952 };
953
954 namespace {
955 class InProcessThinBackend : public ThinBackendProc {
956 ThreadPool BackendThreadPool;
957 AddStreamFn AddStream;
958 NativeObjectCache Cache;
959 std::set<GlobalValue::GUID> CfiFunctionDefs;
960 std::set<GlobalValue::GUID> CfiFunctionDecls;
961
962 Optional<Error> Err;
963 std::mutex ErrMu;
964
965 public:
InProcessThinBackend(Config & Conf,ModuleSummaryIndex & CombinedIndex,unsigned ThinLTOParallelismLevel,const StringMap<GVSummaryMapTy> & ModuleToDefinedGVSummaries,AddStreamFn AddStream,NativeObjectCache Cache)966 InProcessThinBackend(
967 Config &Conf, ModuleSummaryIndex &CombinedIndex,
968 unsigned ThinLTOParallelismLevel,
969 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
970 AddStreamFn AddStream, NativeObjectCache Cache)
971 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
972 BackendThreadPool(ThinLTOParallelismLevel),
973 AddStream(std::move(AddStream)), Cache(std::move(Cache)) {
974 for (auto &Name : CombinedIndex.cfiFunctionDefs())
975 CfiFunctionDefs.insert(
976 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
977 for (auto &Name : CombinedIndex.cfiFunctionDecls())
978 CfiFunctionDecls.insert(
979 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
980 }
981
runThinLTOBackendThread(AddStreamFn AddStream,NativeObjectCache Cache,unsigned Task,BitcodeModule BM,ModuleSummaryIndex & CombinedIndex,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const std::map<GlobalValue::GUID,GlobalValue::LinkageTypes> & ResolvedODR,const GVSummaryMapTy & DefinedGlobals,MapVector<StringRef,BitcodeModule> & ModuleMap)982 Error runThinLTOBackendThread(
983 AddStreamFn AddStream, NativeObjectCache Cache, unsigned Task,
984 BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
985 const FunctionImporter::ImportMapTy &ImportList,
986 const FunctionImporter::ExportSetTy &ExportList,
987 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
988 const GVSummaryMapTy &DefinedGlobals,
989 MapVector<StringRef, BitcodeModule> &ModuleMap) {
990 auto RunThinBackend = [&](AddStreamFn AddStream) {
991 LTOLLVMContext BackendContext(Conf);
992 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);
993 if (!MOrErr)
994 return MOrErr.takeError();
995
996 return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
997 ImportList, DefinedGlobals, ModuleMap);
998 };
999
1000 auto ModuleID = BM.getModuleIdentifier();
1001
1002 if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) ||
1003 all_of(CombinedIndex.getModuleHash(ModuleID),
1004 [](uint32_t V) { return V == 0; }))
1005 // Cache disabled or no entry for this module in the combined index or
1006 // no module hash.
1007 return RunThinBackend(AddStream);
1008
1009 SmallString<40> Key;
1010 // The module may be cached, this helps handling it.
1011 computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList,
1012 ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs,
1013 CfiFunctionDecls);
1014 if (AddStreamFn CacheAddStream = Cache(Task, Key))
1015 return RunThinBackend(CacheAddStream);
1016
1017 return Error::success();
1018 }
1019
start(unsigned Task,BitcodeModule BM,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const std::map<GlobalValue::GUID,GlobalValue::LinkageTypes> & ResolvedODR,MapVector<StringRef,BitcodeModule> & ModuleMap)1020 Error start(
1021 unsigned Task, BitcodeModule BM,
1022 const FunctionImporter::ImportMapTy &ImportList,
1023 const FunctionImporter::ExportSetTy &ExportList,
1024 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1025 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1026 StringRef ModulePath = BM.getModuleIdentifier();
1027 assert(ModuleToDefinedGVSummaries.count(ModulePath));
1028 const GVSummaryMapTy &DefinedGlobals =
1029 ModuleToDefinedGVSummaries.find(ModulePath)->second;
1030 BackendThreadPool.async(
1031 [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1032 const FunctionImporter::ImportMapTy &ImportList,
1033 const FunctionImporter::ExportSetTy &ExportList,
1034 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
1035 &ResolvedODR,
1036 const GVSummaryMapTy &DefinedGlobals,
1037 MapVector<StringRef, BitcodeModule> &ModuleMap) {
1038 Error E = runThinLTOBackendThread(
1039 AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,
1040 ResolvedODR, DefinedGlobals, ModuleMap);
1041 if (E) {
1042 std::unique_lock<std::mutex> L(ErrMu);
1043 if (Err)
1044 Err = joinErrors(std::move(*Err), std::move(E));
1045 else
1046 Err = std::move(E);
1047 }
1048 },
1049 BM, std::ref(CombinedIndex), std::ref(ImportList), std::ref(ExportList),
1050 std::ref(ResolvedODR), std::ref(DefinedGlobals), std::ref(ModuleMap));
1051 return Error::success();
1052 }
1053
wait()1054 Error wait() override {
1055 BackendThreadPool.wait();
1056 if (Err)
1057 return std::move(*Err);
1058 else
1059 return Error::success();
1060 }
1061 };
1062 } // end anonymous namespace
1063
createInProcessThinBackend(unsigned ParallelismLevel)1064 ThinBackend lto::createInProcessThinBackend(unsigned ParallelismLevel) {
1065 return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,
1066 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1067 AddStreamFn AddStream, NativeObjectCache Cache) {
1068 return llvm::make_unique<InProcessThinBackend>(
1069 Conf, CombinedIndex, ParallelismLevel, ModuleToDefinedGVSummaries,
1070 AddStream, Cache);
1071 };
1072 }
1073
1074 // Given the original \p Path to an output file, replace any path
1075 // prefix matching \p OldPrefix with \p NewPrefix. Also, create the
1076 // resulting directory if it does not yet exist.
getThinLTOOutputFile(const std::string & Path,const std::string & OldPrefix,const std::string & NewPrefix)1077 std::string lto::getThinLTOOutputFile(const std::string &Path,
1078 const std::string &OldPrefix,
1079 const std::string &NewPrefix) {
1080 if (OldPrefix.empty() && NewPrefix.empty())
1081 return Path;
1082 SmallString<128> NewPath(Path);
1083 llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
1084 StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
1085 if (!ParentPath.empty()) {
1086 // Make sure the new directory exists, creating it if necessary.
1087 if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
1088 llvm::errs() << "warning: could not create directory '" << ParentPath
1089 << "': " << EC.message() << '\n';
1090 }
1091 return NewPath.str();
1092 }
1093
1094 namespace {
1095 class WriteIndexesThinBackend : public ThinBackendProc {
1096 std::string OldPrefix, NewPrefix;
1097 bool ShouldEmitImportsFiles;
1098 raw_fd_ostream *LinkedObjectsFile;
1099 lto::IndexWriteCallback OnWrite;
1100
1101 public:
WriteIndexesThinBackend(Config & Conf,ModuleSummaryIndex & CombinedIndex,const StringMap<GVSummaryMapTy> & ModuleToDefinedGVSummaries,std::string OldPrefix,std::string NewPrefix,bool ShouldEmitImportsFiles,raw_fd_ostream * LinkedObjectsFile,lto::IndexWriteCallback OnWrite)1102 WriteIndexesThinBackend(
1103 Config &Conf, ModuleSummaryIndex &CombinedIndex,
1104 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1105 std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
1106 raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
1107 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
1108 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
1109 ShouldEmitImportsFiles(ShouldEmitImportsFiles),
1110 LinkedObjectsFile(LinkedObjectsFile), OnWrite(OnWrite) {}
1111
start(unsigned Task,BitcodeModule BM,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const std::map<GlobalValue::GUID,GlobalValue::LinkageTypes> & ResolvedODR,MapVector<StringRef,BitcodeModule> & ModuleMap)1112 Error start(
1113 unsigned Task, BitcodeModule BM,
1114 const FunctionImporter::ImportMapTy &ImportList,
1115 const FunctionImporter::ExportSetTy &ExportList,
1116 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1117 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1118 StringRef ModulePath = BM.getModuleIdentifier();
1119 std::string NewModulePath =
1120 getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
1121
1122 if (LinkedObjectsFile)
1123 *LinkedObjectsFile << NewModulePath << '\n';
1124
1125 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
1126 gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
1127 ImportList, ModuleToSummariesForIndex);
1128
1129 std::error_code EC;
1130 raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
1131 sys::fs::OpenFlags::F_None);
1132 if (EC)
1133 return errorCodeToError(EC);
1134 WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
1135
1136 if (ShouldEmitImportsFiles) {
1137 EC = EmitImportsFiles(ModulePath, NewModulePath + ".imports",
1138 ModuleToSummariesForIndex);
1139 if (EC)
1140 return errorCodeToError(EC);
1141 }
1142
1143 if (OnWrite)
1144 OnWrite(ModulePath);
1145 return Error::success();
1146 }
1147
wait()1148 Error wait() override { return Error::success(); }
1149 };
1150 } // end anonymous namespace
1151
createWriteIndexesThinBackend(std::string OldPrefix,std::string NewPrefix,bool ShouldEmitImportsFiles,raw_fd_ostream * LinkedObjectsFile,IndexWriteCallback OnWrite)1152 ThinBackend lto::createWriteIndexesThinBackend(
1153 std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
1154 raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
1155 return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,
1156 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1157 AddStreamFn AddStream, NativeObjectCache Cache) {
1158 return llvm::make_unique<WriteIndexesThinBackend>(
1159 Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix,
1160 ShouldEmitImportsFiles, LinkedObjectsFile, OnWrite);
1161 };
1162 }
1163
runThinLTO(AddStreamFn AddStream,NativeObjectCache Cache)1164 Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache) {
1165 if (ThinLTO.ModuleMap.empty())
1166 return Error::success();
1167
1168 if (Conf.CombinedIndexHook && !Conf.CombinedIndexHook(ThinLTO.CombinedIndex))
1169 return Error::success();
1170
1171 // Collect for each module the list of function it defines (GUID ->
1172 // Summary).
1173 StringMap<GVSummaryMapTy>
1174 ModuleToDefinedGVSummaries(ThinLTO.ModuleMap.size());
1175 ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
1176 ModuleToDefinedGVSummaries);
1177 // Create entries for any modules that didn't have any GV summaries
1178 // (either they didn't have any GVs to start with, or we suppressed
1179 // generation of the summaries because they e.g. had inline assembly
1180 // uses that couldn't be promoted/renamed on export). This is so
1181 // InProcessThinBackend::start can still launch a backend thread, which
1182 // is passed the map of summaries for the module, without any special
1183 // handling for this case.
1184 for (auto &Mod : ThinLTO.ModuleMap)
1185 if (!ModuleToDefinedGVSummaries.count(Mod.first))
1186 ModuleToDefinedGVSummaries.try_emplace(Mod.first);
1187
1188 // Synthesize entry counts for functions in the CombinedIndex.
1189 computeSyntheticCounts(ThinLTO.CombinedIndex);
1190
1191 StringMap<FunctionImporter::ImportMapTy> ImportLists(
1192 ThinLTO.ModuleMap.size());
1193 StringMap<FunctionImporter::ExportSetTy> ExportLists(
1194 ThinLTO.ModuleMap.size());
1195 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1196
1197 if (DumpThinCGSCCs)
1198 ThinLTO.CombinedIndex.dumpSCCs(outs());
1199
1200 if (Conf.OptLevel > 0)
1201 ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1202 ImportLists, ExportLists);
1203
1204 // Figure out which symbols need to be internalized. This also needs to happen
1205 // at -O0 because summary-based DCE is implemented using internalization, and
1206 // we must apply DCE consistently with the full LTO module in order to avoid
1207 // undefined references during the final link.
1208 std::set<GlobalValue::GUID> ExportedGUIDs;
1209 for (auto &Res : GlobalResolutions) {
1210 // If the symbol does not have external references or it is not prevailing,
1211 // then not need to mark it as exported from a ThinLTO partition.
1212 if (Res.second.Partition != GlobalResolution::External ||
1213 !Res.second.isPrevailingIRSymbol())
1214 continue;
1215 auto GUID = GlobalValue::getGUID(
1216 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1217 // Mark exported unless index-based analysis determined it to be dead.
1218 if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
1219 ExportedGUIDs.insert(GUID);
1220 }
1221
1222 // Any functions referenced by the jump table in the regular LTO object must
1223 // be exported.
1224 for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs())
1225 ExportedGUIDs.insert(
1226 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Def)));
1227
1228 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
1229 const auto &ExportList = ExportLists.find(ModuleIdentifier);
1230 return (ExportList != ExportLists.end() &&
1231 ExportList->second.count(GUID)) ||
1232 ExportedGUIDs.count(GUID);
1233 };
1234 thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported);
1235
1236 auto isPrevailing = [&](GlobalValue::GUID GUID,
1237 const GlobalValueSummary *S) {
1238 return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
1239 };
1240 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1241 GlobalValue::GUID GUID,
1242 GlobalValue::LinkageTypes NewLinkage) {
1243 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1244 };
1245 thinLTOResolvePrevailingInIndex(ThinLTO.CombinedIndex, isPrevailing,
1246 recordNewLinkage);
1247
1248 std::unique_ptr<ThinBackendProc> BackendProc =
1249 ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1250 AddStream, Cache);
1251
1252 // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for combined
1253 // module and parallel code generation partitions.
1254 unsigned Task = RegularLTO.ParallelCodeGenParallelismLevel;
1255 for (auto &Mod : ThinLTO.ModuleMap) {
1256 if (Error E = BackendProc->start(Task, Mod.second, ImportLists[Mod.first],
1257 ExportLists[Mod.first],
1258 ResolvedODR[Mod.first], ThinLTO.ModuleMap))
1259 return E;
1260 ++Task;
1261 }
1262
1263 return BackendProc->wait();
1264 }
1265
1266 Expected<std::unique_ptr<ToolOutputFile>>
setupOptimizationRemarks(LLVMContext & Context,StringRef LTORemarksFilename,bool LTOPassRemarksWithHotness,int Count)1267 lto::setupOptimizationRemarks(LLVMContext &Context,
1268 StringRef LTORemarksFilename,
1269 bool LTOPassRemarksWithHotness, int Count) {
1270 if (LTOPassRemarksWithHotness)
1271 Context.setDiagnosticsHotnessRequested(true);
1272 if (LTORemarksFilename.empty())
1273 return nullptr;
1274
1275 std::string Filename = LTORemarksFilename;
1276 if (Count != -1)
1277 Filename += ".thin." + llvm::utostr(Count) + ".yaml";
1278
1279 std::error_code EC;
1280 auto DiagnosticFile =
1281 llvm::make_unique<ToolOutputFile>(Filename, EC, sys::fs::F_None);
1282 if (EC)
1283 return errorCodeToError(EC);
1284 Context.setDiagnosticsOutputFile(
1285 llvm::make_unique<yaml::Output>(DiagnosticFile->os()));
1286 DiagnosticFile->keep();
1287 return std::move(DiagnosticFile);
1288 }
1289