1 //===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements functions and classes used to support LTO.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/LTO/LTO.h"
14 #include "llvm/ADT/SmallSet.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/ADT/StringExtras.h"
17 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
18 #include "llvm/Analysis/StackSafetyAnalysis.h"
19 #include "llvm/Analysis/TargetLibraryInfo.h"
20 #include "llvm/Analysis/TargetTransformInfo.h"
21 #include "llvm/Bitcode/BitcodeReader.h"
22 #include "llvm/Bitcode/BitcodeWriter.h"
23 #include "llvm/CodeGen/Analysis.h"
24 #include "llvm/Config/llvm-config.h"
25 #include "llvm/IR/AutoUpgrade.h"
26 #include "llvm/IR/DiagnosticPrinter.h"
27 #include "llvm/IR/Intrinsics.h"
28 #include "llvm/IR/LLVMRemarkStreamer.h"
29 #include "llvm/IR/LegacyPassManager.h"
30 #include "llvm/IR/Mangler.h"
31 #include "llvm/IR/Metadata.h"
32 #include "llvm/LTO/LTOBackend.h"
33 #include "llvm/LTO/SummaryBasedOptimizations.h"
34 #include "llvm/Linker/IRMover.h"
35 #include "llvm/Object/IRObjectFile.h"
36 #include "llvm/Support/CommandLine.h"
37 #include "llvm/Support/Error.h"
38 #include "llvm/Support/FileSystem.h"
39 #include "llvm/Support/ManagedStatic.h"
40 #include "llvm/Support/MemoryBuffer.h"
41 #include "llvm/Support/Path.h"
42 #include "llvm/Support/SHA1.h"
43 #include "llvm/Support/SourceMgr.h"
44 #include "llvm/Support/TargetRegistry.h"
45 #include "llvm/Support/ThreadPool.h"
46 #include "llvm/Support/Threading.h"
47 #include "llvm/Support/TimeProfiler.h"
48 #include "llvm/Support/VCSRevision.h"
49 #include "llvm/Support/raw_ostream.h"
50 #include "llvm/Target/TargetMachine.h"
51 #include "llvm/Target/TargetOptions.h"
52 #include "llvm/Transforms/IPO.h"
53 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
54 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
55 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
56 #include "llvm/Transforms/Utils/SplitModule.h"
57
58 #include <set>
59
60 using namespace llvm;
61 using namespace lto;
62 using namespace object;
63
64 #define DEBUG_TYPE "lto"
65
66 static cl::opt<bool>
67 DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden,
68 cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
69
70 /// Enable global value internalization in LTO.
71 cl::opt<bool> EnableLTOInternalization(
72 "enable-lto-internalization", cl::init(true), cl::Hidden,
73 cl::desc("Enable global value internalization in LTO"));
74
75 // Computes a unique hash for the Module considering the current list of
76 // export/import and other global analysis results.
77 // The hash is produced in \p Key.
computeLTOCacheKey(SmallString<40> & Key,const Config & Conf,const ModuleSummaryIndex & Index,StringRef ModuleID,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const std::map<GlobalValue::GUID,GlobalValue::LinkageTypes> & ResolvedODR,const GVSummaryMapTy & DefinedGlobals,const std::set<GlobalValue::GUID> & CfiFunctionDefs,const std::set<GlobalValue::GUID> & CfiFunctionDecls)78 void llvm::computeLTOCacheKey(
79 SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index,
80 StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
81 const FunctionImporter::ExportSetTy &ExportList,
82 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
83 const GVSummaryMapTy &DefinedGlobals,
84 const std::set<GlobalValue::GUID> &CfiFunctionDefs,
85 const std::set<GlobalValue::GUID> &CfiFunctionDecls) {
86 // Compute the unique hash for this entry.
87 // This is based on the current compiler version, the module itself, the
88 // export list, the hash for every single module in the import list, the
89 // list of ResolvedODR for the module, and the list of preserved symbols.
90 SHA1 Hasher;
91
92 // Start with the compiler revision
93 Hasher.update(LLVM_VERSION_STRING);
94 #ifdef LLVM_REVISION
95 Hasher.update(LLVM_REVISION);
96 #endif
97
98 // Include the parts of the LTO configuration that affect code generation.
99 auto AddString = [&](StringRef Str) {
100 Hasher.update(Str);
101 Hasher.update(ArrayRef<uint8_t>{0});
102 };
103 auto AddUnsigned = [&](unsigned I) {
104 uint8_t Data[4];
105 support::endian::write32le(Data, I);
106 Hasher.update(ArrayRef<uint8_t>{Data, 4});
107 };
108 auto AddUint64 = [&](uint64_t I) {
109 uint8_t Data[8];
110 support::endian::write64le(Data, I);
111 Hasher.update(ArrayRef<uint8_t>{Data, 8});
112 };
113 AddString(Conf.CPU);
114 // FIXME: Hash more of Options. For now all clients initialize Options from
115 // command-line flags (which is unsupported in production), but may set
116 // RelaxELFRelocations. The clang driver can also pass FunctionSections,
117 // DataSections and DebuggerTuning via command line flags.
118 AddUnsigned(Conf.Options.RelaxELFRelocations);
119 AddUnsigned(Conf.Options.FunctionSections);
120 AddUnsigned(Conf.Options.DataSections);
121 AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
122 for (auto &A : Conf.MAttrs)
123 AddString(A);
124 if (Conf.RelocModel)
125 AddUnsigned(*Conf.RelocModel);
126 else
127 AddUnsigned(-1);
128 if (Conf.CodeModel)
129 AddUnsigned(*Conf.CodeModel);
130 else
131 AddUnsigned(-1);
132 AddUnsigned(Conf.CGOptLevel);
133 AddUnsigned(Conf.CGFileType);
134 AddUnsigned(Conf.OptLevel);
135 AddUnsigned(Conf.UseNewPM);
136 AddUnsigned(Conf.Freestanding);
137 AddString(Conf.OptPipeline);
138 AddString(Conf.AAPipeline);
139 AddString(Conf.OverrideTriple);
140 AddString(Conf.DefaultTriple);
141 AddString(Conf.DwoDir);
142
143 // Include the hash for the current module
144 auto ModHash = Index.getModuleHash(ModuleID);
145 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
146
147 std::vector<uint64_t> ExportsGUID;
148 ExportsGUID.reserve(ExportList.size());
149 for (const auto &VI : ExportList) {
150 auto GUID = VI.getGUID();
151 ExportsGUID.push_back(GUID);
152 }
153
154 // Sort the export list elements GUIDs.
155 llvm::sort(ExportsGUID);
156 for (uint64_t GUID : ExportsGUID) {
157 // The export list can impact the internalization, be conservative here
158 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&GUID, sizeof(GUID)));
159 }
160
161 // Include the hash for every module we import functions from. The set of
162 // imported symbols for each module may affect code generation and is
163 // sensitive to link order, so include that as well.
164 using ImportMapIteratorTy = FunctionImporter::ImportMapTy::const_iterator;
165 std::vector<ImportMapIteratorTy> ImportModulesVector;
166 ImportModulesVector.reserve(ImportList.size());
167
168 for (ImportMapIteratorTy It = ImportList.begin(); It != ImportList.end();
169 ++It) {
170 ImportModulesVector.push_back(It);
171 }
172 llvm::sort(ImportModulesVector,
173 [](const ImportMapIteratorTy &Lhs, const ImportMapIteratorTy &Rhs)
174 -> bool { return Lhs->getKey() < Rhs->getKey(); });
175 for (const ImportMapIteratorTy &EntryIt : ImportModulesVector) {
176 auto ModHash = Index.getModuleHash(EntryIt->first());
177 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
178
179 AddUint64(EntryIt->second.size());
180 for (auto &Fn : EntryIt->second)
181 AddUint64(Fn);
182 }
183
184 // Include the hash for the resolved ODR.
185 for (auto &Entry : ResolvedODR) {
186 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
187 sizeof(GlobalValue::GUID)));
188 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
189 sizeof(GlobalValue::LinkageTypes)));
190 }
191
192 // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
193 // defined in this module.
194 std::set<GlobalValue::GUID> UsedCfiDefs;
195 std::set<GlobalValue::GUID> UsedCfiDecls;
196
197 // Typeids used in this module.
198 std::set<GlobalValue::GUID> UsedTypeIds;
199
200 auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
201 if (CfiFunctionDefs.count(ValueGUID))
202 UsedCfiDefs.insert(ValueGUID);
203 if (CfiFunctionDecls.count(ValueGUID))
204 UsedCfiDecls.insert(ValueGUID);
205 };
206
207 auto AddUsedThings = [&](GlobalValueSummary *GS) {
208 if (!GS) return;
209 AddUnsigned(GS->getVisibility());
210 AddUnsigned(GS->isLive());
211 AddUnsigned(GS->canAutoHide());
212 for (const ValueInfo &VI : GS->refs()) {
213 AddUnsigned(VI.isDSOLocal(Index.withDSOLocalPropagation()));
214 AddUsedCfiGlobal(VI.getGUID());
215 }
216 if (auto *GVS = dyn_cast<GlobalVarSummary>(GS)) {
217 AddUnsigned(GVS->maybeReadOnly());
218 AddUnsigned(GVS->maybeWriteOnly());
219 }
220 if (auto *FS = dyn_cast<FunctionSummary>(GS)) {
221 for (auto &TT : FS->type_tests())
222 UsedTypeIds.insert(TT);
223 for (auto &TT : FS->type_test_assume_vcalls())
224 UsedTypeIds.insert(TT.GUID);
225 for (auto &TT : FS->type_checked_load_vcalls())
226 UsedTypeIds.insert(TT.GUID);
227 for (auto &TT : FS->type_test_assume_const_vcalls())
228 UsedTypeIds.insert(TT.VFunc.GUID);
229 for (auto &TT : FS->type_checked_load_const_vcalls())
230 UsedTypeIds.insert(TT.VFunc.GUID);
231 for (auto &ET : FS->calls()) {
232 AddUnsigned(ET.first.isDSOLocal(Index.withDSOLocalPropagation()));
233 AddUsedCfiGlobal(ET.first.getGUID());
234 }
235 }
236 };
237
238 // Include the hash for the linkage type to reflect internalization and weak
239 // resolution, and collect any used type identifier resolutions.
240 for (auto &GS : DefinedGlobals) {
241 GlobalValue::LinkageTypes Linkage = GS.second->linkage();
242 Hasher.update(
243 ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
244 AddUsedCfiGlobal(GS.first);
245 AddUsedThings(GS.second);
246 }
247
248 // Imported functions may introduce new uses of type identifier resolutions,
249 // so we need to collect their used resolutions as well.
250 for (auto &ImpM : ImportList)
251 for (auto &ImpF : ImpM.second) {
252 GlobalValueSummary *S = Index.findSummaryInModule(ImpF, ImpM.first());
253 AddUsedThings(S);
254 // If this is an alias, we also care about any types/etc. that the aliasee
255 // may reference.
256 if (auto *AS = dyn_cast_or_null<AliasSummary>(S))
257 AddUsedThings(AS->getBaseObject());
258 }
259
260 auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
261 AddString(TId);
262
263 AddUnsigned(S.TTRes.TheKind);
264 AddUnsigned(S.TTRes.SizeM1BitWidth);
265
266 AddUint64(S.TTRes.AlignLog2);
267 AddUint64(S.TTRes.SizeM1);
268 AddUint64(S.TTRes.BitMask);
269 AddUint64(S.TTRes.InlineBits);
270
271 AddUint64(S.WPDRes.size());
272 for (auto &WPD : S.WPDRes) {
273 AddUnsigned(WPD.first);
274 AddUnsigned(WPD.second.TheKind);
275 AddString(WPD.second.SingleImplName);
276
277 AddUint64(WPD.second.ResByArg.size());
278 for (auto &ByArg : WPD.second.ResByArg) {
279 AddUint64(ByArg.first.size());
280 for (uint64_t Arg : ByArg.first)
281 AddUint64(Arg);
282 AddUnsigned(ByArg.second.TheKind);
283 AddUint64(ByArg.second.Info);
284 AddUnsigned(ByArg.second.Byte);
285 AddUnsigned(ByArg.second.Bit);
286 }
287 }
288 };
289
290 // Include the hash for all type identifiers used by this module.
291 for (GlobalValue::GUID TId : UsedTypeIds) {
292 auto TidIter = Index.typeIds().equal_range(TId);
293 for (auto It = TidIter.first; It != TidIter.second; ++It)
294 AddTypeIdSummary(It->second.first, It->second.second);
295 }
296
297 AddUnsigned(UsedCfiDefs.size());
298 for (auto &V : UsedCfiDefs)
299 AddUint64(V);
300
301 AddUnsigned(UsedCfiDecls.size());
302 for (auto &V : UsedCfiDecls)
303 AddUint64(V);
304
305 if (!Conf.SampleProfile.empty()) {
306 auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
307 if (FileOrErr) {
308 Hasher.update(FileOrErr.get()->getBuffer());
309
310 if (!Conf.ProfileRemapping.empty()) {
311 FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);
312 if (FileOrErr)
313 Hasher.update(FileOrErr.get()->getBuffer());
314 }
315 }
316 }
317
318 Key = toHex(Hasher.result());
319 }
320
thinLTOResolvePrevailingGUID(const Config & C,ValueInfo VI,DenseSet<GlobalValueSummary * > & GlobalInvolvedWithAlias,function_ref<bool (GlobalValue::GUID,const GlobalValueSummary *)> isPrevailing,function_ref<void (StringRef,GlobalValue::GUID,GlobalValue::LinkageTypes)> recordNewLinkage,const DenseSet<GlobalValue::GUID> & GUIDPreservedSymbols)321 static void thinLTOResolvePrevailingGUID(
322 const Config &C, ValueInfo VI,
323 DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
324 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
325 isPrevailing,
326 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
327 recordNewLinkage,
328 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
329 GlobalValue::VisibilityTypes Visibility =
330 C.VisibilityScheme == Config::ELF ? VI.getELFVisibility()
331 : GlobalValue::DefaultVisibility;
332 for (auto &S : VI.getSummaryList()) {
333 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
334 // Ignore local and appending linkage values since the linker
335 // doesn't resolve them.
336 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
337 GlobalValue::isAppendingLinkage(S->linkage()))
338 continue;
339 // We need to emit only one of these. The prevailing module will keep it,
340 // but turned into a weak, while the others will drop it when possible.
341 // This is both a compile-time optimization and a correctness
342 // transformation. This is necessary for correctness when we have exported
343 // a reference - we need to convert the linkonce to weak to
344 // ensure a copy is kept to satisfy the exported reference.
345 // FIXME: We may want to split the compile time and correctness
346 // aspects into separate routines.
347 if (isPrevailing(VI.getGUID(), S.get())) {
348 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage)) {
349 S->setLinkage(GlobalValue::getWeakLinkage(
350 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
351 // The kept copy is eligible for auto-hiding (hidden visibility) if all
352 // copies were (i.e. they were all linkonce_odr global unnamed addr).
353 // If any copy is not (e.g. it was originally weak_odr), then the symbol
354 // must remain externally available (e.g. a weak_odr from an explicitly
355 // instantiated template). Additionally, if it is in the
356 // GUIDPreservedSymbols set, that means that it is visibile outside
357 // the summary (e.g. in a native object or a bitcode file without
358 // summary), and in that case we cannot hide it as it isn't possible to
359 // check all copies.
360 S->setCanAutoHide(VI.canAutoHide() &&
361 !GUIDPreservedSymbols.count(VI.getGUID()));
362 }
363 if (C.VisibilityScheme == Config::FromPrevailing)
364 Visibility = S->getVisibility();
365 }
366 // Alias and aliasee can't be turned into available_externally.
367 else if (!isa<AliasSummary>(S.get()) &&
368 !GlobalInvolvedWithAlias.count(S.get()))
369 S->setLinkage(GlobalValue::AvailableExternallyLinkage);
370
371 // For ELF, set visibility to the computed visibility from summaries. We
372 // don't track visibility from declarations so this may be more relaxed than
373 // the most constraining one.
374 if (C.VisibilityScheme == Config::ELF)
375 S->setVisibility(Visibility);
376
377 if (S->linkage() != OriginalLinkage)
378 recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage());
379 }
380
381 if (C.VisibilityScheme == Config::FromPrevailing) {
382 for (auto &S : VI.getSummaryList()) {
383 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
384 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
385 GlobalValue::isAppendingLinkage(S->linkage()))
386 continue;
387 S->setVisibility(Visibility);
388 }
389 }
390 }
391
392 /// Resolve linkage for prevailing symbols in the \p Index.
393 //
394 // We'd like to drop these functions if they are no longer referenced in the
395 // current module. However there is a chance that another module is still
396 // referencing them because of the import. We make sure we always emit at least
397 // one copy.
thinLTOResolvePrevailingInIndex(const Config & C,ModuleSummaryIndex & Index,function_ref<bool (GlobalValue::GUID,const GlobalValueSummary *)> isPrevailing,function_ref<void (StringRef,GlobalValue::GUID,GlobalValue::LinkageTypes)> recordNewLinkage,const DenseSet<GlobalValue::GUID> & GUIDPreservedSymbols)398 void llvm::thinLTOResolvePrevailingInIndex(
399 const Config &C, ModuleSummaryIndex &Index,
400 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
401 isPrevailing,
402 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
403 recordNewLinkage,
404 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
405 // We won't optimize the globals that are referenced by an alias for now
406 // Ideally we should turn the alias into a global and duplicate the definition
407 // when needed.
408 DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
409 for (auto &I : Index)
410 for (auto &S : I.second.SummaryList)
411 if (auto AS = dyn_cast<AliasSummary>(S.get()))
412 GlobalInvolvedWithAlias.insert(&AS->getAliasee());
413
414 for (auto &I : Index)
415 thinLTOResolvePrevailingGUID(C, Index.getValueInfo(I),
416 GlobalInvolvedWithAlias, isPrevailing,
417 recordNewLinkage, GUIDPreservedSymbols);
418 }
419
isWeakObjectWithRWAccess(GlobalValueSummary * GVS)420 static bool isWeakObjectWithRWAccess(GlobalValueSummary *GVS) {
421 if (auto *VarSummary = dyn_cast<GlobalVarSummary>(GVS->getBaseObject()))
422 return !VarSummary->maybeReadOnly() && !VarSummary->maybeWriteOnly() &&
423 (VarSummary->linkage() == GlobalValue::WeakODRLinkage ||
424 VarSummary->linkage() == GlobalValue::LinkOnceODRLinkage);
425 return false;
426 }
427
thinLTOInternalizeAndPromoteGUID(ValueInfo VI,function_ref<bool (StringRef,ValueInfo)> isExported,function_ref<bool (GlobalValue::GUID,const GlobalValueSummary *)> isPrevailing)428 static void thinLTOInternalizeAndPromoteGUID(
429 ValueInfo VI, function_ref<bool(StringRef, ValueInfo)> isExported,
430 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
431 isPrevailing) {
432 for (auto &S : VI.getSummaryList()) {
433 if (isExported(S->modulePath(), VI)) {
434 if (GlobalValue::isLocalLinkage(S->linkage()))
435 S->setLinkage(GlobalValue::ExternalLinkage);
436 } else if (EnableLTOInternalization &&
437 // Ignore local and appending linkage values since the linker
438 // doesn't resolve them.
439 !GlobalValue::isLocalLinkage(S->linkage()) &&
440 (!GlobalValue::isInterposableLinkage(S->linkage()) ||
441 isPrevailing(VI.getGUID(), S.get())) &&
442 S->linkage() != GlobalValue::AppendingLinkage &&
443 // We can't internalize available_externally globals because this
444 // can break function pointer equality.
445 S->linkage() != GlobalValue::AvailableExternallyLinkage &&
446 // Functions and read-only variables with linkonce_odr and
447 // weak_odr linkage can be internalized. We can't internalize
448 // linkonce_odr and weak_odr variables which are both modified
449 // and read somewhere in the program because reads and writes
450 // will become inconsistent.
451 !isWeakObjectWithRWAccess(S.get()))
452 S->setLinkage(GlobalValue::InternalLinkage);
453 }
454 }
455
456 // Update the linkages in the given \p Index to mark exported values
457 // as external and non-exported values as internal.
thinLTOInternalizeAndPromoteInIndex(ModuleSummaryIndex & Index,function_ref<bool (StringRef,ValueInfo)> isExported,function_ref<bool (GlobalValue::GUID,const GlobalValueSummary *)> isPrevailing)458 void llvm::thinLTOInternalizeAndPromoteInIndex(
459 ModuleSummaryIndex &Index,
460 function_ref<bool(StringRef, ValueInfo)> isExported,
461 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
462 isPrevailing) {
463 for (auto &I : Index)
464 thinLTOInternalizeAndPromoteGUID(Index.getValueInfo(I), isExported,
465 isPrevailing);
466 }
467
468 // Requires a destructor for std::vector<InputModule>.
469 InputFile::~InputFile() = default;
470
create(MemoryBufferRef Object)471 Expected<std::unique_ptr<InputFile>> InputFile::create(MemoryBufferRef Object) {
472 std::unique_ptr<InputFile> File(new InputFile);
473
474 Expected<IRSymtabFile> FOrErr = readIRSymtab(Object);
475 if (!FOrErr)
476 return FOrErr.takeError();
477
478 File->TargetTriple = FOrErr->TheReader.getTargetTriple();
479 File->SourceFileName = FOrErr->TheReader.getSourceFileName();
480 File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();
481 File->DependentLibraries = FOrErr->TheReader.getDependentLibraries();
482 File->ComdatTable = FOrErr->TheReader.getComdatTable();
483
484 for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {
485 size_t Begin = File->Symbols.size();
486 for (const irsymtab::Reader::SymbolRef &Sym :
487 FOrErr->TheReader.module_symbols(I))
488 // Skip symbols that are irrelevant to LTO. Note that this condition needs
489 // to match the one in Skip() in LTO::addRegularLTO().
490 if (Sym.isGlobal() && !Sym.isFormatSpecific())
491 File->Symbols.push_back(Sym);
492 File->ModuleSymIndices.push_back({Begin, File->Symbols.size()});
493 }
494
495 File->Mods = FOrErr->Mods;
496 File->Strtab = std::move(FOrErr->Strtab);
497 return std::move(File);
498 }
499
getName() const500 StringRef InputFile::getName() const {
501 return Mods[0].getModuleIdentifier();
502 }
503
getSingleBitcodeModule()504 BitcodeModule &InputFile::getSingleBitcodeModule() {
505 assert(Mods.size() == 1 && "Expect only one bitcode module");
506 return Mods[0];
507 }
508
RegularLTOState(unsigned ParallelCodeGenParallelismLevel,const Config & Conf)509 LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
510 const Config &Conf)
511 : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
512 Ctx(Conf), CombinedModule(std::make_unique<Module>("ld-temp.o", Ctx)),
513 Mover(std::make_unique<IRMover>(*CombinedModule)) {}
514
ThinLTOState(ThinBackend Backend)515 LTO::ThinLTOState::ThinLTOState(ThinBackend Backend)
516 : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) {
517 if (!Backend)
518 this->Backend =
519 createInProcessThinBackend(llvm::heavyweight_hardware_concurrency());
520 }
521
LTO(Config Conf,ThinBackend Backend,unsigned ParallelCodeGenParallelismLevel)522 LTO::LTO(Config Conf, ThinBackend Backend,
523 unsigned ParallelCodeGenParallelismLevel)
524 : Conf(std::move(Conf)),
525 RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
526 ThinLTO(std::move(Backend)) {}
527
528 // Requires a destructor for MapVector<BitcodeModule>.
529 LTO::~LTO() = default;
530
531 // Add the symbols in the given module to the GlobalResolutions map, and resolve
532 // their partitions.
addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,ArrayRef<SymbolResolution> Res,unsigned Partition,bool InSummary)533 void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
534 ArrayRef<SymbolResolution> Res,
535 unsigned Partition, bool InSummary) {
536 auto *ResI = Res.begin();
537 auto *ResE = Res.end();
538 (void)ResE;
539 for (const InputFile::Symbol &Sym : Syms) {
540 assert(ResI != ResE);
541 SymbolResolution Res = *ResI++;
542
543 StringRef Name = Sym.getName();
544 Triple TT(RegularLTO.CombinedModule->getTargetTriple());
545 // Strip the __imp_ prefix from COFF dllimport symbols (similar to the
546 // way they are handled by lld), otherwise we can end up with two
547 // global resolutions (one with and one for a copy of the symbol without).
548 if (TT.isOSBinFormatCOFF() && Name.startswith("__imp_"))
549 Name = Name.substr(strlen("__imp_"));
550 auto &GlobalRes = GlobalResolutions[Name];
551 GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
552 if (Res.Prevailing) {
553 assert(!GlobalRes.Prevailing &&
554 "Multiple prevailing defs are not allowed");
555 GlobalRes.Prevailing = true;
556 GlobalRes.IRName = std::string(Sym.getIRName());
557 } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {
558 // Sometimes it can be two copies of symbol in a module and prevailing
559 // symbol can have no IR name. That might happen if symbol is defined in
560 // module level inline asm block. In case we have multiple modules with
561 // the same symbol we want to use IR name of the prevailing symbol.
562 // Otherwise, if we haven't seen a prevailing symbol, set the name so that
563 // we can later use it to check if there is any prevailing copy in IR.
564 GlobalRes.IRName = std::string(Sym.getIRName());
565 }
566
567 // Set the partition to external if we know it is re-defined by the linker
568 // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
569 // regular object, is referenced from llvm.compiler.used/llvm.used, or was
570 // already recorded as being referenced from a different partition.
571 if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||
572 (GlobalRes.Partition != GlobalResolution::Unknown &&
573 GlobalRes.Partition != Partition)) {
574 GlobalRes.Partition = GlobalResolution::External;
575 } else
576 // First recorded reference, save the current partition.
577 GlobalRes.Partition = Partition;
578
579 // Flag as visible outside of summary if visible from a regular object or
580 // from a module that does not have a summary.
581 GlobalRes.VisibleOutsideSummary |=
582 (Res.VisibleToRegularObj || Sym.isUsed() || !InSummary);
583
584 GlobalRes.ExportDynamic |= Res.ExportDynamic;
585 }
586 }
587
writeToResolutionFile(raw_ostream & OS,InputFile * Input,ArrayRef<SymbolResolution> Res)588 static void writeToResolutionFile(raw_ostream &OS, InputFile *Input,
589 ArrayRef<SymbolResolution> Res) {
590 StringRef Path = Input->getName();
591 OS << Path << '\n';
592 auto ResI = Res.begin();
593 for (const InputFile::Symbol &Sym : Input->symbols()) {
594 assert(ResI != Res.end());
595 SymbolResolution Res = *ResI++;
596
597 OS << "-r=" << Path << ',' << Sym.getName() << ',';
598 if (Res.Prevailing)
599 OS << 'p';
600 if (Res.FinalDefinitionInLinkageUnit)
601 OS << 'l';
602 if (Res.VisibleToRegularObj)
603 OS << 'x';
604 if (Res.LinkerRedefined)
605 OS << 'r';
606 OS << '\n';
607 }
608 OS.flush();
609 assert(ResI == Res.end());
610 }
611
add(std::unique_ptr<InputFile> Input,ArrayRef<SymbolResolution> Res)612 Error LTO::add(std::unique_ptr<InputFile> Input,
613 ArrayRef<SymbolResolution> Res) {
614 assert(!CalledGetMaxTasks);
615
616 if (Conf.ResolutionFile)
617 writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res);
618
619 if (RegularLTO.CombinedModule->getTargetTriple().empty()) {
620 RegularLTO.CombinedModule->setTargetTriple(Input->getTargetTriple());
621 if (Triple(Input->getTargetTriple()).isOSBinFormatELF())
622 Conf.VisibilityScheme = Config::ELF;
623 }
624
625 const SymbolResolution *ResI = Res.begin();
626 for (unsigned I = 0; I != Input->Mods.size(); ++I)
627 if (Error Err = addModule(*Input, I, ResI, Res.end()))
628 return Err;
629
630 assert(ResI == Res.end());
631 return Error::success();
632 }
633
addModule(InputFile & Input,unsigned ModI,const SymbolResolution * & ResI,const SymbolResolution * ResE)634 Error LTO::addModule(InputFile &Input, unsigned ModI,
635 const SymbolResolution *&ResI,
636 const SymbolResolution *ResE) {
637 Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();
638 if (!LTOInfo)
639 return LTOInfo.takeError();
640
641 if (EnableSplitLTOUnit.hasValue()) {
642 // If only some modules were split, flag this in the index so that
643 // we can skip or error on optimizations that need consistently split
644 // modules (whole program devirt and lower type tests).
645 if (EnableSplitLTOUnit.getValue() != LTOInfo->EnableSplitLTOUnit)
646 ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
647 } else
648 EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
649
650 BitcodeModule BM = Input.Mods[ModI];
651 auto ModSyms = Input.module_symbols(ModI);
652 addModuleToGlobalRes(ModSyms, {ResI, ResE},
653 LTOInfo->IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,
654 LTOInfo->HasSummary);
655
656 if (LTOInfo->IsThinLTO)
657 return addThinLTO(BM, ModSyms, ResI, ResE);
658
659 RegularLTO.EmptyCombinedModule = false;
660 Expected<RegularLTOState::AddedModule> ModOrErr =
661 addRegularLTO(BM, ModSyms, ResI, ResE);
662 if (!ModOrErr)
663 return ModOrErr.takeError();
664
665 if (!LTOInfo->HasSummary)
666 return linkRegularLTO(std::move(*ModOrErr), /*LivenessFromIndex=*/false);
667
668 // Regular LTO module summaries are added to a dummy module that represents
669 // the combined regular LTO module.
670 if (Error Err = BM.readSummary(ThinLTO.CombinedIndex, "", -1ull))
671 return Err;
672 RegularLTO.ModsWithSummaries.push_back(std::move(*ModOrErr));
673 return Error::success();
674 }
675
676 // Checks whether the given global value is in a non-prevailing comdat
677 // (comdat containing values the linker indicated were not prevailing,
678 // which we then dropped to available_externally), and if so, removes
679 // it from the comdat. This is called for all global values to ensure the
680 // comdat is empty rather than leaving an incomplete comdat. It is needed for
681 // regular LTO modules, in case we are in a mixed-LTO mode (both regular
682 // and thin LTO modules) compilation. Since the regular LTO module will be
683 // linked first in the final native link, we want to make sure the linker
684 // doesn't select any of these incomplete comdats that would be left
685 // in the regular LTO module without this cleanup.
686 static void
handleNonPrevailingComdat(GlobalValue & GV,std::set<const Comdat * > & NonPrevailingComdats)687 handleNonPrevailingComdat(GlobalValue &GV,
688 std::set<const Comdat *> &NonPrevailingComdats) {
689 Comdat *C = GV.getComdat();
690 if (!C)
691 return;
692
693 if (!NonPrevailingComdats.count(C))
694 return;
695
696 // Additionally need to drop externally visible global values from the comdat
697 // to available_externally, so that there aren't multiply defined linker
698 // errors.
699 if (!GV.hasLocalLinkage())
700 GV.setLinkage(GlobalValue::AvailableExternallyLinkage);
701
702 if (auto GO = dyn_cast<GlobalObject>(&GV))
703 GO->setComdat(nullptr);
704 }
705
706 // Add a regular LTO object to the link.
707 // The resulting module needs to be linked into the combined LTO module with
708 // linkRegularLTO.
709 Expected<LTO::RegularLTOState::AddedModule>
addRegularLTO(BitcodeModule BM,ArrayRef<InputFile::Symbol> Syms,const SymbolResolution * & ResI,const SymbolResolution * ResE)710 LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
711 const SymbolResolution *&ResI,
712 const SymbolResolution *ResE) {
713 RegularLTOState::AddedModule Mod;
714 Expected<std::unique_ptr<Module>> MOrErr =
715 BM.getLazyModule(RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
716 /*IsImporting*/ false);
717 if (!MOrErr)
718 return MOrErr.takeError();
719 Module &M = **MOrErr;
720 Mod.M = std::move(*MOrErr);
721
722 if (Error Err = M.materializeMetadata())
723 return std::move(Err);
724 UpgradeDebugInfo(M);
725
726 ModuleSymbolTable SymTab;
727 SymTab.addModule(&M);
728
729 for (GlobalVariable &GV : M.globals())
730 if (GV.hasAppendingLinkage())
731 Mod.Keep.push_back(&GV);
732
733 DenseSet<GlobalObject *> AliasedGlobals;
734 for (auto &GA : M.aliases())
735 if (GlobalObject *GO = GA.getBaseObject())
736 AliasedGlobals.insert(GO);
737
738 // In this function we need IR GlobalValues matching the symbols in Syms
739 // (which is not backed by a module), so we need to enumerate them in the same
740 // order. The symbol enumeration order of a ModuleSymbolTable intentionally
741 // matches the order of an irsymtab, but when we read the irsymtab in
742 // InputFile::create we omit some symbols that are irrelevant to LTO. The
743 // Skip() function skips the same symbols from the module as InputFile does
744 // from the symbol table.
745 auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();
746 auto Skip = [&]() {
747 while (MsymI != MsymE) {
748 auto Flags = SymTab.getSymbolFlags(*MsymI);
749 if ((Flags & object::BasicSymbolRef::SF_Global) &&
750 !(Flags & object::BasicSymbolRef::SF_FormatSpecific))
751 return;
752 ++MsymI;
753 }
754 };
755 Skip();
756
757 std::set<const Comdat *> NonPrevailingComdats;
758 SmallSet<StringRef, 2> NonPrevailingAsmSymbols;
759 for (const InputFile::Symbol &Sym : Syms) {
760 assert(ResI != ResE);
761 SymbolResolution Res = *ResI++;
762
763 assert(MsymI != MsymE);
764 ModuleSymbolTable::Symbol Msym = *MsymI++;
765 Skip();
766
767 if (GlobalValue *GV = Msym.dyn_cast<GlobalValue *>()) {
768 if (Res.Prevailing) {
769 if (Sym.isUndefined())
770 continue;
771 Mod.Keep.push_back(GV);
772 // For symbols re-defined with linker -wrap and -defsym options,
773 // set the linkage to weak to inhibit IPO. The linkage will be
774 // restored by the linker.
775 if (Res.LinkerRedefined)
776 GV->setLinkage(GlobalValue::WeakAnyLinkage);
777
778 GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();
779 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))
780 GV->setLinkage(GlobalValue::getWeakLinkage(
781 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
782 } else if (isa<GlobalObject>(GV) &&
783 (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||
784 GV->hasAvailableExternallyLinkage()) &&
785 !AliasedGlobals.count(cast<GlobalObject>(GV))) {
786 // Any of the above three types of linkage indicates that the
787 // chosen prevailing symbol will have the same semantics as this copy of
788 // the symbol, so we may be able to link it with available_externally
789 // linkage. We will decide later whether to do that when we link this
790 // module (in linkRegularLTO), based on whether it is undefined.
791 Mod.Keep.push_back(GV);
792 GV->setLinkage(GlobalValue::AvailableExternallyLinkage);
793 if (GV->hasComdat())
794 NonPrevailingComdats.insert(GV->getComdat());
795 cast<GlobalObject>(GV)->setComdat(nullptr);
796 }
797
798 // Set the 'local' flag based on the linker resolution for this symbol.
799 if (Res.FinalDefinitionInLinkageUnit) {
800 GV->setDSOLocal(true);
801 if (GV->hasDLLImportStorageClass())
802 GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
803 DefaultStorageClass);
804 }
805 } else if (auto *AS = Msym.dyn_cast<ModuleSymbolTable::AsmSymbol *>()) {
806 // Collect non-prevailing symbols.
807 if (!Res.Prevailing)
808 NonPrevailingAsmSymbols.insert(AS->first);
809 } else {
810 llvm_unreachable("unknown symbol type");
811 }
812
813 // Common resolution: collect the maximum size/alignment over all commons.
814 // We also record if we see an instance of a common as prevailing, so that
815 // if none is prevailing we can ignore it later.
816 if (Sym.isCommon()) {
817 // FIXME: We should figure out what to do about commons defined by asm.
818 // For now they aren't reported correctly by ModuleSymbolTable.
819 auto &CommonRes = RegularLTO.Commons[std::string(Sym.getIRName())];
820 CommonRes.Size = std::max(CommonRes.Size, Sym.getCommonSize());
821 MaybeAlign SymAlign(Sym.getCommonAlignment());
822 if (SymAlign)
823 CommonRes.Align = max(*SymAlign, CommonRes.Align);
824 CommonRes.Prevailing |= Res.Prevailing;
825 }
826 }
827
828 if (!M.getComdatSymbolTable().empty())
829 for (GlobalValue &GV : M.global_values())
830 handleNonPrevailingComdat(GV, NonPrevailingComdats);
831
832 // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm
833 // block.
834 if (!M.getModuleInlineAsm().empty()) {
835 std::string NewIA = ".lto_discard";
836 if (!NonPrevailingAsmSymbols.empty()) {
837 // Don't dicard a symbol if there is a live .symver for it.
838 ModuleSymbolTable::CollectAsmSymvers(
839 M, [&](StringRef Name, StringRef Alias) {
840 if (!NonPrevailingAsmSymbols.count(Alias))
841 NonPrevailingAsmSymbols.erase(Name);
842 });
843 NewIA += " " + llvm::join(NonPrevailingAsmSymbols, ", ");
844 }
845 NewIA += "\n";
846 M.setModuleInlineAsm(NewIA + M.getModuleInlineAsm());
847 }
848
849 assert(MsymI == MsymE);
850 return std::move(Mod);
851 }
852
linkRegularLTO(RegularLTOState::AddedModule Mod,bool LivenessFromIndex)853 Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
854 bool LivenessFromIndex) {
855 std::vector<GlobalValue *> Keep;
856 for (GlobalValue *GV : Mod.Keep) {
857 if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GV->getGUID())) {
858 if (Function *F = dyn_cast<Function>(GV)) {
859 OptimizationRemarkEmitter ORE(F, nullptr);
860 ORE.emit(OptimizationRemark(DEBUG_TYPE, "deadfunction", F)
861 << ore::NV("Function", F)
862 << " not added to the combined module ");
863 }
864 continue;
865 }
866
867 if (!GV->hasAvailableExternallyLinkage()) {
868 Keep.push_back(GV);
869 continue;
870 }
871
872 // Only link available_externally definitions if we don't already have a
873 // definition.
874 GlobalValue *CombinedGV =
875 RegularLTO.CombinedModule->getNamedValue(GV->getName());
876 if (CombinedGV && !CombinedGV->isDeclaration())
877 continue;
878
879 Keep.push_back(GV);
880 }
881
882 return RegularLTO.Mover->move(std::move(Mod.M), Keep,
883 [](GlobalValue &, IRMover::ValueAdder) {},
884 /* IsPerformingImport */ false);
885 }
886
887 // Add a ThinLTO module to the link.
addThinLTO(BitcodeModule BM,ArrayRef<InputFile::Symbol> Syms,const SymbolResolution * & ResI,const SymbolResolution * ResE)888 Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
889 const SymbolResolution *&ResI,
890 const SymbolResolution *ResE) {
891 if (Error Err =
892 BM.readSummary(ThinLTO.CombinedIndex, BM.getModuleIdentifier(),
893 ThinLTO.ModuleMap.size()))
894 return Err;
895
896 for (const InputFile::Symbol &Sym : Syms) {
897 assert(ResI != ResE);
898 SymbolResolution Res = *ResI++;
899
900 if (!Sym.getIRName().empty()) {
901 auto GUID = GlobalValue::getGUID(GlobalValue::getGlobalIdentifier(
902 Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
903 if (Res.Prevailing) {
904 ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier();
905
906 // For linker redefined symbols (via --wrap or --defsym) we want to
907 // switch the linkage to `weak` to prevent IPOs from happening.
908 // Find the summary in the module for this very GV and record the new
909 // linkage so that we can switch it when we import the GV.
910 if (Res.LinkerRedefined)
911 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
912 GUID, BM.getModuleIdentifier()))
913 S->setLinkage(GlobalValue::WeakAnyLinkage);
914 }
915
916 // If the linker resolved the symbol to a local definition then mark it
917 // as local in the summary for the module we are adding.
918 if (Res.FinalDefinitionInLinkageUnit) {
919 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
920 GUID, BM.getModuleIdentifier())) {
921 S->setDSOLocal(true);
922 }
923 }
924 }
925 }
926
927 if (!ThinLTO.ModuleMap.insert({BM.getModuleIdentifier(), BM}).second)
928 return make_error<StringError>(
929 "Expected at most one ThinLTO module per bitcode file",
930 inconvertibleErrorCode());
931
932 if (!Conf.ThinLTOModulesToCompile.empty()) {
933 if (!ThinLTO.ModulesToCompile)
934 ThinLTO.ModulesToCompile = ModuleMapType();
935 // This is a fuzzy name matching where only modules with name containing the
936 // specified switch values are going to be compiled.
937 for (const std::string &Name : Conf.ThinLTOModulesToCompile) {
938 if (BM.getModuleIdentifier().contains(Name)) {
939 ThinLTO.ModulesToCompile->insert({BM.getModuleIdentifier(), BM});
940 llvm::errs() << "[ThinLTO] Selecting " << BM.getModuleIdentifier()
941 << " to compile\n";
942 }
943 }
944 }
945
946 return Error::success();
947 }
948
getMaxTasks() const949 unsigned LTO::getMaxTasks() const {
950 CalledGetMaxTasks = true;
951 auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()
952 : ThinLTO.ModuleMap.size();
953 return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;
954 }
955
956 // If only some of the modules were split, we cannot correctly handle
957 // code that contains type tests or type checked loads.
checkPartiallySplit()958 Error LTO::checkPartiallySplit() {
959 if (!ThinLTO.CombinedIndex.partiallySplitLTOUnits())
960 return Error::success();
961
962 Function *TypeTestFunc = RegularLTO.CombinedModule->getFunction(
963 Intrinsic::getName(Intrinsic::type_test));
964 Function *TypeCheckedLoadFunc = RegularLTO.CombinedModule->getFunction(
965 Intrinsic::getName(Intrinsic::type_checked_load));
966
967 // First check if there are type tests / type checked loads in the
968 // merged regular LTO module IR.
969 if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
970 (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()))
971 return make_error<StringError>(
972 "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
973 inconvertibleErrorCode());
974
975 // Otherwise check if there are any recorded in the combined summary from the
976 // ThinLTO modules.
977 for (auto &P : ThinLTO.CombinedIndex) {
978 for (auto &S : P.second.SummaryList) {
979 auto *FS = dyn_cast<FunctionSummary>(S.get());
980 if (!FS)
981 continue;
982 if (!FS->type_test_assume_vcalls().empty() ||
983 !FS->type_checked_load_vcalls().empty() ||
984 !FS->type_test_assume_const_vcalls().empty() ||
985 !FS->type_checked_load_const_vcalls().empty() ||
986 !FS->type_tests().empty())
987 return make_error<StringError>(
988 "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
989 inconvertibleErrorCode());
990 }
991 }
992 return Error::success();
993 }
994
run(AddStreamFn AddStream,NativeObjectCache Cache)995 Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) {
996 // Compute "dead" symbols, we don't want to import/export these!
997 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
998 DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
999 for (auto &Res : GlobalResolutions) {
1000 // Normally resolution have IR name of symbol. We can do nothing here
1001 // otherwise. See comments in GlobalResolution struct for more details.
1002 if (Res.second.IRName.empty())
1003 continue;
1004
1005 GlobalValue::GUID GUID = GlobalValue::getGUID(
1006 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1007
1008 if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)
1009 GUIDPreservedSymbols.insert(GUID);
1010
1011 if (Res.second.ExportDynamic)
1012 DynamicExportSymbols.insert(GUID);
1013
1014 GUIDPrevailingResolutions[GUID] =
1015 Res.second.Prevailing ? PrevailingType::Yes : PrevailingType::No;
1016 }
1017
1018 auto isPrevailing = [&](GlobalValue::GUID G) {
1019 auto It = GUIDPrevailingResolutions.find(G);
1020 if (It == GUIDPrevailingResolutions.end())
1021 return PrevailingType::Unknown;
1022 return It->second;
1023 };
1024 computeDeadSymbolsWithConstProp(ThinLTO.CombinedIndex, GUIDPreservedSymbols,
1025 isPrevailing, Conf.OptLevel > 0);
1026
1027 // Setup output file to emit statistics.
1028 auto StatsFileOrErr = setupStatsFile(Conf.StatsFile);
1029 if (!StatsFileOrErr)
1030 return StatsFileOrErr.takeError();
1031 std::unique_ptr<ToolOutputFile> StatsFile = std::move(StatsFileOrErr.get());
1032
1033 Error Result = runRegularLTO(AddStream);
1034 if (!Result)
1035 Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);
1036
1037 if (StatsFile)
1038 PrintStatisticsJSON(StatsFile->os());
1039
1040 return Result;
1041 }
1042
runRegularLTO(AddStreamFn AddStream)1043 Error LTO::runRegularLTO(AddStreamFn AddStream) {
1044 // Setup optimization remarks.
1045 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1046 RegularLTO.CombinedModule->getContext(), Conf.RemarksFilename,
1047 Conf.RemarksPasses, Conf.RemarksFormat, Conf.RemarksWithHotness,
1048 Conf.RemarksHotnessThreshold);
1049 if (!DiagFileOrErr)
1050 return DiagFileOrErr.takeError();
1051
1052 // Finalize linking of regular LTO modules containing summaries now that
1053 // we have computed liveness information.
1054 for (auto &M : RegularLTO.ModsWithSummaries)
1055 if (Error Err = linkRegularLTO(std::move(M),
1056 /*LivenessFromIndex=*/true))
1057 return Err;
1058
1059 // Ensure we don't have inconsistently split LTO units with type tests.
1060 // FIXME: this checks both LTO and ThinLTO. It happens to work as we take
1061 // this path both cases but eventually this should be split into two and
1062 // do the ThinLTO checks in `runThinLTO`.
1063 if (Error Err = checkPartiallySplit())
1064 return Err;
1065
1066 // Make sure commons have the right size/alignment: we kept the largest from
1067 // all the prevailing when adding the inputs, and we apply it here.
1068 const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
1069 for (auto &I : RegularLTO.Commons) {
1070 if (!I.second.Prevailing)
1071 // Don't do anything if no instance of this common was prevailing.
1072 continue;
1073 GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(I.first);
1074 if (OldGV && DL.getTypeAllocSize(OldGV->getValueType()) == I.second.Size) {
1075 // Don't create a new global if the type is already correct, just make
1076 // sure the alignment is correct.
1077 OldGV->setAlignment(I.second.Align);
1078 continue;
1079 }
1080 ArrayType *Ty =
1081 ArrayType::get(Type::getInt8Ty(RegularLTO.Ctx), I.second.Size);
1082 auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
1083 GlobalValue::CommonLinkage,
1084 ConstantAggregateZero::get(Ty), "");
1085 GV->setAlignment(I.second.Align);
1086 if (OldGV) {
1087 OldGV->replaceAllUsesWith(ConstantExpr::getBitCast(GV, OldGV->getType()));
1088 GV->takeName(OldGV);
1089 OldGV->eraseFromParent();
1090 } else {
1091 GV->setName(I.first);
1092 }
1093 }
1094
1095 // If allowed, upgrade public vcall visibility metadata to linkage unit
1096 // visibility before whole program devirtualization in the optimizer.
1097 updateVCallVisibilityInModule(*RegularLTO.CombinedModule,
1098 Conf.HasWholeProgramVisibility,
1099 DynamicExportSymbols);
1100
1101 if (Conf.PreOptModuleHook &&
1102 !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
1103 return Error::success();
1104
1105 if (!Conf.CodeGenOnly) {
1106 for (const auto &R : GlobalResolutions) {
1107 if (!R.second.isPrevailingIRSymbol())
1108 continue;
1109 if (R.second.Partition != 0 &&
1110 R.second.Partition != GlobalResolution::External)
1111 continue;
1112
1113 GlobalValue *GV =
1114 RegularLTO.CombinedModule->getNamedValue(R.second.IRName);
1115 // Ignore symbols defined in other partitions.
1116 // Also skip declarations, which are not allowed to have internal linkage.
1117 if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())
1118 continue;
1119 GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
1120 : GlobalValue::UnnamedAddr::None);
1121 if (EnableLTOInternalization && R.second.Partition == 0)
1122 GV->setLinkage(GlobalValue::InternalLinkage);
1123 }
1124
1125 RegularLTO.CombinedModule->addModuleFlag(Module::Error, "LTOPostLink", 1);
1126
1127 if (Conf.PostInternalizeModuleHook &&
1128 !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
1129 return Error::success();
1130 }
1131
1132 if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
1133 if (Error Err =
1134 backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
1135 *RegularLTO.CombinedModule, ThinLTO.CombinedIndex))
1136 return Err;
1137 }
1138
1139 return finalizeOptimizationRemarks(std::move(*DiagFileOrErr));
1140 }
1141
1142 static const char *libcallRoutineNames[] = {
1143 #define HANDLE_LIBCALL(code, name) name,
1144 #include "llvm/IR/RuntimeLibcalls.def"
1145 #undef HANDLE_LIBCALL
1146 };
1147
getRuntimeLibcallSymbols()1148 ArrayRef<const char*> LTO::getRuntimeLibcallSymbols() {
1149 return makeArrayRef(libcallRoutineNames);
1150 }
1151
1152 /// This class defines the interface to the ThinLTO backend.
1153 class lto::ThinBackendProc {
1154 protected:
1155 const Config &Conf;
1156 ModuleSummaryIndex &CombinedIndex;
1157 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries;
1158
1159 public:
ThinBackendProc(const Config & Conf,ModuleSummaryIndex & CombinedIndex,const StringMap<GVSummaryMapTy> & ModuleToDefinedGVSummaries)1160 ThinBackendProc(const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1161 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries)
1162 : Conf(Conf), CombinedIndex(CombinedIndex),
1163 ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries) {}
1164
~ThinBackendProc()1165 virtual ~ThinBackendProc() {}
1166 virtual Error start(
1167 unsigned Task, BitcodeModule BM,
1168 const FunctionImporter::ImportMapTy &ImportList,
1169 const FunctionImporter::ExportSetTy &ExportList,
1170 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1171 MapVector<StringRef, BitcodeModule> &ModuleMap) = 0;
1172 virtual Error wait() = 0;
1173 virtual unsigned getThreadCount() = 0;
1174 };
1175
1176 namespace {
1177 class InProcessThinBackend : public ThinBackendProc {
1178 ThreadPool BackendThreadPool;
1179 AddStreamFn AddStream;
1180 NativeObjectCache Cache;
1181 std::set<GlobalValue::GUID> CfiFunctionDefs;
1182 std::set<GlobalValue::GUID> CfiFunctionDecls;
1183
1184 Optional<Error> Err;
1185 std::mutex ErrMu;
1186
1187 public:
InProcessThinBackend(const Config & Conf,ModuleSummaryIndex & CombinedIndex,ThreadPoolStrategy ThinLTOParallelism,const StringMap<GVSummaryMapTy> & ModuleToDefinedGVSummaries,AddStreamFn AddStream,NativeObjectCache Cache)1188 InProcessThinBackend(
1189 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1190 ThreadPoolStrategy ThinLTOParallelism,
1191 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1192 AddStreamFn AddStream, NativeObjectCache Cache)
1193 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
1194 BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)),
1195 Cache(std::move(Cache)) {
1196 for (auto &Name : CombinedIndex.cfiFunctionDefs())
1197 CfiFunctionDefs.insert(
1198 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1199 for (auto &Name : CombinedIndex.cfiFunctionDecls())
1200 CfiFunctionDecls.insert(
1201 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1202 }
1203
runThinLTOBackendThread(AddStreamFn AddStream,NativeObjectCache Cache,unsigned Task,BitcodeModule BM,ModuleSummaryIndex & CombinedIndex,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const std::map<GlobalValue::GUID,GlobalValue::LinkageTypes> & ResolvedODR,const GVSummaryMapTy & DefinedGlobals,MapVector<StringRef,BitcodeModule> & ModuleMap)1204 Error runThinLTOBackendThread(
1205 AddStreamFn AddStream, NativeObjectCache Cache, unsigned Task,
1206 BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1207 const FunctionImporter::ImportMapTy &ImportList,
1208 const FunctionImporter::ExportSetTy &ExportList,
1209 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1210 const GVSummaryMapTy &DefinedGlobals,
1211 MapVector<StringRef, BitcodeModule> &ModuleMap) {
1212 auto RunThinBackend = [&](AddStreamFn AddStream) {
1213 LTOLLVMContext BackendContext(Conf);
1214 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);
1215 if (!MOrErr)
1216 return MOrErr.takeError();
1217
1218 return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
1219 ImportList, DefinedGlobals, &ModuleMap);
1220 };
1221
1222 auto ModuleID = BM.getModuleIdentifier();
1223
1224 if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) ||
1225 all_of(CombinedIndex.getModuleHash(ModuleID),
1226 [](uint32_t V) { return V == 0; }))
1227 // Cache disabled or no entry for this module in the combined index or
1228 // no module hash.
1229 return RunThinBackend(AddStream);
1230
1231 SmallString<40> Key;
1232 // The module may be cached, this helps handling it.
1233 computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList,
1234 ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs,
1235 CfiFunctionDecls);
1236 if (AddStreamFn CacheAddStream = Cache(Task, Key))
1237 return RunThinBackend(CacheAddStream);
1238
1239 return Error::success();
1240 }
1241
start(unsigned Task,BitcodeModule BM,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const std::map<GlobalValue::GUID,GlobalValue::LinkageTypes> & ResolvedODR,MapVector<StringRef,BitcodeModule> & ModuleMap)1242 Error start(
1243 unsigned Task, BitcodeModule BM,
1244 const FunctionImporter::ImportMapTy &ImportList,
1245 const FunctionImporter::ExportSetTy &ExportList,
1246 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1247 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1248 StringRef ModulePath = BM.getModuleIdentifier();
1249 assert(ModuleToDefinedGVSummaries.count(ModulePath));
1250 const GVSummaryMapTy &DefinedGlobals =
1251 ModuleToDefinedGVSummaries.find(ModulePath)->second;
1252 BackendThreadPool.async(
1253 [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1254 const FunctionImporter::ImportMapTy &ImportList,
1255 const FunctionImporter::ExportSetTy &ExportList,
1256 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
1257 &ResolvedODR,
1258 const GVSummaryMapTy &DefinedGlobals,
1259 MapVector<StringRef, BitcodeModule> &ModuleMap) {
1260 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1261 timeTraceProfilerInitialize(Conf.TimeTraceGranularity,
1262 "thin backend");
1263 Error E = runThinLTOBackendThread(
1264 AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,
1265 ResolvedODR, DefinedGlobals, ModuleMap);
1266 if (E) {
1267 std::unique_lock<std::mutex> L(ErrMu);
1268 if (Err)
1269 Err = joinErrors(std::move(*Err), std::move(E));
1270 else
1271 Err = std::move(E);
1272 }
1273 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1274 timeTraceProfilerFinishThread();
1275 },
1276 BM, std::ref(CombinedIndex), std::ref(ImportList), std::ref(ExportList),
1277 std::ref(ResolvedODR), std::ref(DefinedGlobals), std::ref(ModuleMap));
1278 return Error::success();
1279 }
1280
wait()1281 Error wait() override {
1282 BackendThreadPool.wait();
1283 if (Err)
1284 return std::move(*Err);
1285 else
1286 return Error::success();
1287 }
1288
getThreadCount()1289 unsigned getThreadCount() override {
1290 return BackendThreadPool.getThreadCount();
1291 }
1292 };
1293 } // end anonymous namespace
1294
createInProcessThinBackend(ThreadPoolStrategy Parallelism)1295 ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism) {
1296 return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1297 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1298 AddStreamFn AddStream, NativeObjectCache Cache) {
1299 return std::make_unique<InProcessThinBackend>(
1300 Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, AddStream,
1301 Cache);
1302 };
1303 }
1304
1305 // Given the original \p Path to an output file, replace any path
1306 // prefix matching \p OldPrefix with \p NewPrefix. Also, create the
1307 // resulting directory if it does not yet exist.
getThinLTOOutputFile(const std::string & Path,const std::string & OldPrefix,const std::string & NewPrefix)1308 std::string lto::getThinLTOOutputFile(const std::string &Path,
1309 const std::string &OldPrefix,
1310 const std::string &NewPrefix) {
1311 if (OldPrefix.empty() && NewPrefix.empty())
1312 return Path;
1313 SmallString<128> NewPath(Path);
1314 llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
1315 StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
1316 if (!ParentPath.empty()) {
1317 // Make sure the new directory exists, creating it if necessary.
1318 if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
1319 llvm::errs() << "warning: could not create directory '" << ParentPath
1320 << "': " << EC.message() << '\n';
1321 }
1322 return std::string(NewPath.str());
1323 }
1324
1325 namespace {
1326 class WriteIndexesThinBackend : public ThinBackendProc {
1327 std::string OldPrefix, NewPrefix;
1328 bool ShouldEmitImportsFiles;
1329 raw_fd_ostream *LinkedObjectsFile;
1330 lto::IndexWriteCallback OnWrite;
1331
1332 public:
WriteIndexesThinBackend(const Config & Conf,ModuleSummaryIndex & CombinedIndex,const StringMap<GVSummaryMapTy> & ModuleToDefinedGVSummaries,std::string OldPrefix,std::string NewPrefix,bool ShouldEmitImportsFiles,raw_fd_ostream * LinkedObjectsFile,lto::IndexWriteCallback OnWrite)1333 WriteIndexesThinBackend(
1334 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1335 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1336 std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
1337 raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
1338 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
1339 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
1340 ShouldEmitImportsFiles(ShouldEmitImportsFiles),
1341 LinkedObjectsFile(LinkedObjectsFile), OnWrite(OnWrite) {}
1342
start(unsigned Task,BitcodeModule BM,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const std::map<GlobalValue::GUID,GlobalValue::LinkageTypes> & ResolvedODR,MapVector<StringRef,BitcodeModule> & ModuleMap)1343 Error start(
1344 unsigned Task, BitcodeModule BM,
1345 const FunctionImporter::ImportMapTy &ImportList,
1346 const FunctionImporter::ExportSetTy &ExportList,
1347 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1348 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1349 StringRef ModulePath = BM.getModuleIdentifier();
1350 std::string NewModulePath =
1351 getThinLTOOutputFile(std::string(ModulePath), OldPrefix, NewPrefix);
1352
1353 if (LinkedObjectsFile)
1354 *LinkedObjectsFile << NewModulePath << '\n';
1355
1356 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
1357 gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
1358 ImportList, ModuleToSummariesForIndex);
1359
1360 std::error_code EC;
1361 raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
1362 sys::fs::OpenFlags::OF_None);
1363 if (EC)
1364 return errorCodeToError(EC);
1365 WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
1366
1367 if (ShouldEmitImportsFiles) {
1368 EC = EmitImportsFiles(ModulePath, NewModulePath + ".imports",
1369 ModuleToSummariesForIndex);
1370 if (EC)
1371 return errorCodeToError(EC);
1372 }
1373
1374 if (OnWrite)
1375 OnWrite(std::string(ModulePath));
1376 return Error::success();
1377 }
1378
wait()1379 Error wait() override { return Error::success(); }
1380
1381 // WriteIndexesThinBackend should always return 1 to prevent module
1382 // re-ordering and avoid non-determinism in the final link.
getThreadCount()1383 unsigned getThreadCount() override { return 1; }
1384 };
1385 } // end anonymous namespace
1386
createWriteIndexesThinBackend(std::string OldPrefix,std::string NewPrefix,bool ShouldEmitImportsFiles,raw_fd_ostream * LinkedObjectsFile,IndexWriteCallback OnWrite)1387 ThinBackend lto::createWriteIndexesThinBackend(
1388 std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
1389 raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
1390 return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1391 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1392 AddStreamFn AddStream, NativeObjectCache Cache) {
1393 return std::make_unique<WriteIndexesThinBackend>(
1394 Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix,
1395 ShouldEmitImportsFiles, LinkedObjectsFile, OnWrite);
1396 };
1397 }
1398
runThinLTO(AddStreamFn AddStream,NativeObjectCache Cache,const DenseSet<GlobalValue::GUID> & GUIDPreservedSymbols)1399 Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
1400 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
1401 if (ThinLTO.ModuleMap.empty())
1402 return Error::success();
1403
1404 if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {
1405 llvm::errs() << "warning: [ThinLTO] No module compiled\n";
1406 return Error::success();
1407 }
1408
1409 if (Conf.CombinedIndexHook &&
1410 !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))
1411 return Error::success();
1412
1413 // Collect for each module the list of function it defines (GUID ->
1414 // Summary).
1415 StringMap<GVSummaryMapTy>
1416 ModuleToDefinedGVSummaries(ThinLTO.ModuleMap.size());
1417 ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
1418 ModuleToDefinedGVSummaries);
1419 // Create entries for any modules that didn't have any GV summaries
1420 // (either they didn't have any GVs to start with, or we suppressed
1421 // generation of the summaries because they e.g. had inline assembly
1422 // uses that couldn't be promoted/renamed on export). This is so
1423 // InProcessThinBackend::start can still launch a backend thread, which
1424 // is passed the map of summaries for the module, without any special
1425 // handling for this case.
1426 for (auto &Mod : ThinLTO.ModuleMap)
1427 if (!ModuleToDefinedGVSummaries.count(Mod.first))
1428 ModuleToDefinedGVSummaries.try_emplace(Mod.first);
1429
1430 // Synthesize entry counts for functions in the CombinedIndex.
1431 computeSyntheticCounts(ThinLTO.CombinedIndex);
1432
1433 StringMap<FunctionImporter::ImportMapTy> ImportLists(
1434 ThinLTO.ModuleMap.size());
1435 StringMap<FunctionImporter::ExportSetTy> ExportLists(
1436 ThinLTO.ModuleMap.size());
1437 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1438
1439 if (DumpThinCGSCCs)
1440 ThinLTO.CombinedIndex.dumpSCCs(outs());
1441
1442 std::set<GlobalValue::GUID> ExportedGUIDs;
1443
1444 // If allowed, upgrade public vcall visibility to linkage unit visibility in
1445 // the summaries before whole program devirtualization below.
1446 updateVCallVisibilityInIndex(ThinLTO.CombinedIndex,
1447 Conf.HasWholeProgramVisibility,
1448 DynamicExportSymbols);
1449
1450 // Perform index-based WPD. This will return immediately if there are
1451 // no index entries in the typeIdMetadata map (e.g. if we are instead
1452 // performing IR-based WPD in hybrid regular/thin LTO mode).
1453 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1454 runWholeProgramDevirtOnIndex(ThinLTO.CombinedIndex, ExportedGUIDs,
1455 LocalWPDTargetsMap);
1456
1457 if (Conf.OptLevel > 0)
1458 ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1459 ImportLists, ExportLists);
1460
1461 // Figure out which symbols need to be internalized. This also needs to happen
1462 // at -O0 because summary-based DCE is implemented using internalization, and
1463 // we must apply DCE consistently with the full LTO module in order to avoid
1464 // undefined references during the final link.
1465 for (auto &Res : GlobalResolutions) {
1466 // If the symbol does not have external references or it is not prevailing,
1467 // then not need to mark it as exported from a ThinLTO partition.
1468 if (Res.second.Partition != GlobalResolution::External ||
1469 !Res.second.isPrevailingIRSymbol())
1470 continue;
1471 auto GUID = GlobalValue::getGUID(
1472 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1473 // Mark exported unless index-based analysis determined it to be dead.
1474 if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
1475 ExportedGUIDs.insert(GUID);
1476 }
1477
1478 // Any functions referenced by the jump table in the regular LTO object must
1479 // be exported.
1480 for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs())
1481 ExportedGUIDs.insert(
1482 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Def)));
1483 for (auto &Decl : ThinLTO.CombinedIndex.cfiFunctionDecls())
1484 ExportedGUIDs.insert(
1485 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Decl)));
1486
1487 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1488 const auto &ExportList = ExportLists.find(ModuleIdentifier);
1489 return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
1490 ExportedGUIDs.count(VI.getGUID());
1491 };
1492
1493 // Update local devirtualized targets that were exported by cross-module
1494 // importing or by other devirtualizations marked in the ExportedGUIDs set.
1495 updateIndexWPDForExports(ThinLTO.CombinedIndex, isExported,
1496 LocalWPDTargetsMap);
1497
1498 auto isPrevailing = [&](GlobalValue::GUID GUID,
1499 const GlobalValueSummary *S) {
1500 return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
1501 };
1502 thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported,
1503 isPrevailing);
1504
1505 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1506 GlobalValue::GUID GUID,
1507 GlobalValue::LinkageTypes NewLinkage) {
1508 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1509 };
1510 thinLTOResolvePrevailingInIndex(Conf, ThinLTO.CombinedIndex, isPrevailing,
1511 recordNewLinkage, GUIDPreservedSymbols);
1512
1513 generateParamAccessSummary(ThinLTO.CombinedIndex);
1514
1515 std::unique_ptr<ThinBackendProc> BackendProc =
1516 ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1517 AddStream, Cache);
1518
1519 auto &ModuleMap =
1520 ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
1521
1522 auto ProcessOneModule = [&](int I) -> Error {
1523 auto &Mod = *(ModuleMap.begin() + I);
1524 // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
1525 // combined module and parallel code generation partitions.
1526 return BackendProc->start(RegularLTO.ParallelCodeGenParallelismLevel + I,
1527 Mod.second, ImportLists[Mod.first],
1528 ExportLists[Mod.first], ResolvedODR[Mod.first],
1529 ThinLTO.ModuleMap);
1530 };
1531
1532 if (BackendProc->getThreadCount() == 1) {
1533 // Process the modules in the order they were provided on the command-line.
1534 // It is important for this codepath to be used for WriteIndexesThinBackend,
1535 // to ensure the emitted LinkedObjectsFile lists ThinLTO objects in the same
1536 // order as the inputs, which otherwise would affect the final link order.
1537 for (int I = 0, E = ModuleMap.size(); I != E; ++I)
1538 if (Error E = ProcessOneModule(I))
1539 return E;
1540 } else {
1541 // When executing in parallel, process largest bitsize modules first to
1542 // improve parallelism, and avoid starving the thread pool near the end.
1543 // This saves about 15 sec on a 36-core machine while link `clang.exe` (out
1544 // of 100 sec).
1545 std::vector<BitcodeModule *> ModulesVec;
1546 ModulesVec.reserve(ModuleMap.size());
1547 for (auto &Mod : ModuleMap)
1548 ModulesVec.push_back(&Mod.second);
1549 for (int I : generateModulesOrdering(ModulesVec))
1550 if (Error E = ProcessOneModule(I))
1551 return E;
1552 }
1553 return BackendProc->wait();
1554 }
1555
setupLLVMOptimizationRemarks(LLVMContext & Context,StringRef RemarksFilename,StringRef RemarksPasses,StringRef RemarksFormat,bool RemarksWithHotness,Optional<uint64_t> RemarksHotnessThreshold,int Count)1556 Expected<std::unique_ptr<ToolOutputFile>> lto::setupLLVMOptimizationRemarks(
1557 LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,
1558 StringRef RemarksFormat, bool RemarksWithHotness,
1559 Optional<uint64_t> RemarksHotnessThreshold, int Count) {
1560 std::string Filename = std::string(RemarksFilename);
1561 // For ThinLTO, file.opt.<format> becomes
1562 // file.opt.<format>.thin.<num>.<format>.
1563 if (!Filename.empty() && Count != -1)
1564 Filename =
1565 (Twine(Filename) + ".thin." + llvm::utostr(Count) + "." + RemarksFormat)
1566 .str();
1567
1568 auto ResultOrErr = llvm::setupLLVMOptimizationRemarks(
1569 Context, Filename, RemarksPasses, RemarksFormat, RemarksWithHotness,
1570 RemarksHotnessThreshold);
1571 if (Error E = ResultOrErr.takeError())
1572 return std::move(E);
1573
1574 if (*ResultOrErr)
1575 (*ResultOrErr)->keep();
1576
1577 return ResultOrErr;
1578 }
1579
1580 Expected<std::unique_ptr<ToolOutputFile>>
setupStatsFile(StringRef StatsFilename)1581 lto::setupStatsFile(StringRef StatsFilename) {
1582 // Setup output file to emit statistics.
1583 if (StatsFilename.empty())
1584 return nullptr;
1585
1586 llvm::EnableStatistics(false);
1587 std::error_code EC;
1588 auto StatsFile =
1589 std::make_unique<ToolOutputFile>(StatsFilename, EC, sys::fs::OF_None);
1590 if (EC)
1591 return errorCodeToError(EC);
1592
1593 StatsFile->keep();
1594 return std::move(StatsFile);
1595 }
1596
1597 // Compute the ordering we will process the inputs: the rough heuristic here
1598 // is to sort them per size so that the largest module get schedule as soon as
1599 // possible. This is purely a compile-time optimization.
generateModulesOrdering(ArrayRef<BitcodeModule * > R)1600 std::vector<int> lto::generateModulesOrdering(ArrayRef<BitcodeModule *> R) {
1601 std::vector<int> ModulesOrdering;
1602 ModulesOrdering.resize(R.size());
1603 std::iota(ModulesOrdering.begin(), ModulesOrdering.end(), 0);
1604 llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
1605 auto LSize = R[LeftIndex]->getBuffer().size();
1606 auto RSize = R[RightIndex]->getBuffer().size();
1607 return LSize > RSize;
1608 });
1609 return ModulesOrdering;
1610 }
1611