1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Thin Link Time Optimization library. This library is
10 // intended to be used by linker to optimize code at link time.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
15 #include "llvm/Support/CommandLine.h"
16
17 #include "llvm/ADT/Statistic.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
20 #include "llvm/Analysis/ProfileSummaryInfo.h"
21 #include "llvm/Analysis/TargetLibraryInfo.h"
22 #include "llvm/Analysis/TargetTransformInfo.h"
23 #include "llvm/Bitcode/BitcodeReader.h"
24 #include "llvm/Bitcode/BitcodeWriter.h"
25 #include "llvm/Bitcode/BitcodeWriterPass.h"
26 #include "llvm/Config/llvm-config.h"
27 #include "llvm/IR/DebugInfo.h"
28 #include "llvm/IR/DiagnosticPrinter.h"
29 #include "llvm/IR/LLVMContext.h"
30 #include "llvm/IR/LegacyPassManager.h"
31 #include "llvm/IR/Mangler.h"
32 #include "llvm/IR/PassTimingInfo.h"
33 #include "llvm/IR/RemarkStreamer.h"
34 #include "llvm/IR/Verifier.h"
35 #include "llvm/IRReader/IRReader.h"
36 #include "llvm/LTO/LTO.h"
37 #include "llvm/LTO/SummaryBasedOptimizations.h"
38 #include "llvm/MC/SubtargetFeature.h"
39 #include "llvm/Object/IRObjectFile.h"
40 #include "llvm/Support/CachePruning.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/Error.h"
43 #include "llvm/Support/FileUtilities.h"
44 #include "llvm/Support/Path.h"
45 #include "llvm/Support/SHA1.h"
46 #include "llvm/Support/SmallVectorMemoryBuffer.h"
47 #include "llvm/Support/TargetRegistry.h"
48 #include "llvm/Support/ThreadPool.h"
49 #include "llvm/Support/Threading.h"
50 #include "llvm/Support/ToolOutputFile.h"
51 #include "llvm/Support/VCSRevision.h"
52 #include "llvm/Target/TargetMachine.h"
53 #include "llvm/Transforms/IPO.h"
54 #include "llvm/Transforms/IPO/FunctionImport.h"
55 #include "llvm/Transforms/IPO/Internalize.h"
56 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
57 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
58 #include "llvm/Transforms/ObjCARC.h"
59 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
60
61 #include <numeric>
62
63 #if !defined(_MSC_VER) && !defined(__MINGW32__)
64 #include <unistd.h>
65 #else
66 #include <io.h>
67 #endif
68
69 using namespace llvm;
70
71 #define DEBUG_TYPE "thinlto"
72
73 namespace llvm {
74 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
75 extern cl::opt<bool> LTODiscardValueNames;
76 extern cl::opt<std::string> RemarksFilename;
77 extern cl::opt<std::string> RemarksPasses;
78 extern cl::opt<bool> RemarksWithHotness;
79 extern cl::opt<std::string> RemarksFormat;
80 }
81
82 namespace {
83
84 static cl::opt<int>
85 ThreadCount("threads", cl::init(llvm::heavyweight_hardware_concurrency()));
86
87 // Simple helper to save temporary files for debug.
saveTempBitcode(const Module & TheModule,StringRef TempDir,unsigned count,StringRef Suffix)88 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
89 unsigned count, StringRef Suffix) {
90 if (TempDir.empty())
91 return;
92 // User asked to save temps, let dump the bitcode file after import.
93 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
94 std::error_code EC;
95 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
96 if (EC)
97 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
98 " to save optimized bitcode\n");
99 WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
100 }
101
102 static const GlobalValueSummary *
getFirstDefinitionForLinker(const GlobalValueSummaryList & GVSummaryList)103 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
104 // If there is any strong definition anywhere, get it.
105 auto StrongDefForLinker = llvm::find_if(
106 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
107 auto Linkage = Summary->linkage();
108 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
109 !GlobalValue::isWeakForLinker(Linkage);
110 });
111 if (StrongDefForLinker != GVSummaryList.end())
112 return StrongDefForLinker->get();
113 // Get the first *linker visible* definition for this global in the summary
114 // list.
115 auto FirstDefForLinker = llvm::find_if(
116 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
117 auto Linkage = Summary->linkage();
118 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
119 });
120 // Extern templates can be emitted as available_externally.
121 if (FirstDefForLinker == GVSummaryList.end())
122 return nullptr;
123 return FirstDefForLinker->get();
124 }
125
126 // Populate map of GUID to the prevailing copy for any multiply defined
127 // symbols. Currently assume first copy is prevailing, or any strong
128 // definition. Can be refined with Linker information in the future.
computePrevailingCopies(const ModuleSummaryIndex & Index,DenseMap<GlobalValue::GUID,const GlobalValueSummary * > & PrevailingCopy)129 static void computePrevailingCopies(
130 const ModuleSummaryIndex &Index,
131 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
132 auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
133 return GVSummaryList.size() > 1;
134 };
135
136 for (auto &I : Index) {
137 if (HasMultipleCopies(I.second.SummaryList))
138 PrevailingCopy[I.first] =
139 getFirstDefinitionForLinker(I.second.SummaryList);
140 }
141 }
142
143 static StringMap<lto::InputFile *>
generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> & Modules)144 generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
145 StringMap<lto::InputFile *> ModuleMap;
146 for (auto &M : Modules) {
147 assert(ModuleMap.find(M->getName()) == ModuleMap.end() &&
148 "Expect unique Buffer Identifier");
149 ModuleMap[M->getName()] = M.get();
150 }
151 return ModuleMap;
152 }
153
promoteModule(Module & TheModule,const ModuleSummaryIndex & Index)154 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index) {
155 if (renameModuleForThinLTO(TheModule, Index))
156 report_fatal_error("renameModuleForThinLTO failed");
157 }
158
159 namespace {
160 class ThinLTODiagnosticInfo : public DiagnosticInfo {
161 const Twine &Msg;
162 public:
ThinLTODiagnosticInfo(const Twine & DiagMsg,DiagnosticSeverity Severity=DS_Error)163 ThinLTODiagnosticInfo(const Twine &DiagMsg,
164 DiagnosticSeverity Severity = DS_Error)
165 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
print(DiagnosticPrinter & DP) const166 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
167 };
168 }
169
170 /// Verify the module and strip broken debug info.
verifyLoadedModule(Module & TheModule)171 static void verifyLoadedModule(Module &TheModule) {
172 bool BrokenDebugInfo = false;
173 if (verifyModule(TheModule, &dbgs(), &BrokenDebugInfo))
174 report_fatal_error("Broken module found, compilation aborted!");
175 if (BrokenDebugInfo) {
176 TheModule.getContext().diagnose(ThinLTODiagnosticInfo(
177 "Invalid debug info found, debug info will be stripped", DS_Warning));
178 StripDebugInfo(TheModule);
179 }
180 }
181
loadModuleFromInput(lto::InputFile * Input,LLVMContext & Context,bool Lazy,bool IsImporting)182 static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
183 LLVMContext &Context,
184 bool Lazy,
185 bool IsImporting) {
186 auto &Mod = Input->getSingleBitcodeModule();
187 SMDiagnostic Err;
188 Expected<std::unique_ptr<Module>> ModuleOrErr =
189 Lazy ? Mod.getLazyModule(Context,
190 /* ShouldLazyLoadMetadata */ true, IsImporting)
191 : Mod.parseModule(Context);
192 if (!ModuleOrErr) {
193 handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
194 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
195 SourceMgr::DK_Error, EIB.message());
196 Err.print("ThinLTO", errs());
197 });
198 report_fatal_error("Can't load module, abort.");
199 }
200 if (!Lazy)
201 verifyLoadedModule(*ModuleOrErr.get());
202 return std::move(*ModuleOrErr);
203 }
204
205 static void
crossImportIntoModule(Module & TheModule,const ModuleSummaryIndex & Index,StringMap<lto::InputFile * > & ModuleMap,const FunctionImporter::ImportMapTy & ImportList)206 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
207 StringMap<lto::InputFile*> &ModuleMap,
208 const FunctionImporter::ImportMapTy &ImportList) {
209 auto Loader = [&](StringRef Identifier) {
210 auto &Input = ModuleMap[Identifier];
211 return loadModuleFromInput(Input, TheModule.getContext(),
212 /*Lazy=*/true, /*IsImporting*/ true);
213 };
214
215 FunctionImporter Importer(Index, Loader);
216 Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
217 if (!Result) {
218 handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
219 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
220 SourceMgr::DK_Error, EIB.message());
221 Err.print("ThinLTO", errs());
222 });
223 report_fatal_error("importFunctions failed");
224 }
225 // Verify again after cross-importing.
226 verifyLoadedModule(TheModule);
227 }
228
optimizeModule(Module & TheModule,TargetMachine & TM,unsigned OptLevel,bool Freestanding,ModuleSummaryIndex * Index)229 static void optimizeModule(Module &TheModule, TargetMachine &TM,
230 unsigned OptLevel, bool Freestanding,
231 ModuleSummaryIndex *Index) {
232 // Populate the PassManager
233 PassManagerBuilder PMB;
234 PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
235 if (Freestanding)
236 PMB.LibraryInfo->disableAllFunctions();
237 PMB.Inliner = createFunctionInliningPass();
238 // FIXME: should get it from the bitcode?
239 PMB.OptLevel = OptLevel;
240 PMB.LoopVectorize = true;
241 PMB.SLPVectorize = true;
242 // Already did this in verifyLoadedModule().
243 PMB.VerifyInput = false;
244 PMB.VerifyOutput = false;
245 PMB.ImportSummary = Index;
246
247 legacy::PassManager PM;
248
249 // Add the TTI (required to inform the vectorizer about register size for
250 // instance)
251 PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
252
253 // Add optimizations
254 PMB.populateThinLTOPassManager(PM);
255
256 PM.run(TheModule);
257 }
258
259 static void
addUsedSymbolToPreservedGUID(const lto::InputFile & File,DenseSet<GlobalValue::GUID> & PreservedGUID)260 addUsedSymbolToPreservedGUID(const lto::InputFile &File,
261 DenseSet<GlobalValue::GUID> &PreservedGUID) {
262 for (const auto &Sym : File.symbols()) {
263 if (Sym.isUsed())
264 PreservedGUID.insert(GlobalValue::getGUID(Sym.getIRName()));
265 }
266 }
267
268 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
269 static DenseSet<GlobalValue::GUID>
computeGUIDPreservedSymbols(const StringSet<> & PreservedSymbols,const Triple & TheTriple)270 computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols,
271 const Triple &TheTriple) {
272 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
273 for (auto &Entry : PreservedSymbols) {
274 StringRef Name = Entry.first();
275 if (TheTriple.isOSBinFormatMachO() && Name.size() > 0 && Name[0] == '_')
276 Name = Name.drop_front();
277 GUIDPreservedSymbols.insert(GlobalValue::getGUID(Name));
278 }
279 return GUIDPreservedSymbols;
280 }
281
codegenModule(Module & TheModule,TargetMachine & TM)282 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
283 TargetMachine &TM) {
284 SmallVector<char, 128> OutputBuffer;
285
286 // CodeGen
287 {
288 raw_svector_ostream OS(OutputBuffer);
289 legacy::PassManager PM;
290
291 // If the bitcode files contain ARC code and were compiled with optimization,
292 // the ObjCARCContractPass must be run, so do it unconditionally here.
293 PM.add(createObjCARCContractPass());
294
295 // Setup the codegen now.
296 if (TM.addPassesToEmitFile(PM, OS, nullptr, CGFT_ObjectFile,
297 /* DisableVerify */ true))
298 report_fatal_error("Failed to setup codegen");
299
300 // Run codegen now. resulting binary is in OutputBuffer.
301 PM.run(TheModule);
302 }
303 return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
304 }
305
306 /// Manage caching for a single Module.
307 class ModuleCacheEntry {
308 SmallString<128> EntryPath;
309
310 public:
311 // Create a cache entry. This compute a unique hash for the Module considering
312 // the current list of export/import, and offer an interface to query to
313 // access the content in the cache.
ModuleCacheEntry(StringRef CachePath,const ModuleSummaryIndex & Index,StringRef ModuleID,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const std::map<GlobalValue::GUID,GlobalValue::LinkageTypes> & ResolvedODR,const GVSummaryMapTy & DefinedGVSummaries,unsigned OptLevel,bool Freestanding,const TargetMachineBuilder & TMBuilder)314 ModuleCacheEntry(
315 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
316 const FunctionImporter::ImportMapTy &ImportList,
317 const FunctionImporter::ExportSetTy &ExportList,
318 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
319 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
320 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
321 if (CachePath.empty())
322 return;
323
324 if (!Index.modulePaths().count(ModuleID))
325 // The module does not have an entry, it can't have a hash at all
326 return;
327
328 if (all_of(Index.getModuleHash(ModuleID),
329 [](uint32_t V) { return V == 0; }))
330 // No hash entry, no caching!
331 return;
332
333 llvm::lto::Config Conf;
334 Conf.OptLevel = OptLevel;
335 Conf.Options = TMBuilder.Options;
336 Conf.CPU = TMBuilder.MCpu;
337 Conf.MAttrs.push_back(TMBuilder.MAttr);
338 Conf.RelocModel = TMBuilder.RelocModel;
339 Conf.CGOptLevel = TMBuilder.CGOptLevel;
340 Conf.Freestanding = Freestanding;
341 SmallString<40> Key;
342 computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
343 ResolvedODR, DefinedGVSummaries);
344
345 // This choice of file name allows the cache to be pruned (see pruneCache()
346 // in include/llvm/Support/CachePruning.h).
347 sys::path::append(EntryPath, CachePath, "llvmcache-" + Key);
348 }
349
350 // Access the path to this entry in the cache.
getEntryPath()351 StringRef getEntryPath() { return EntryPath; }
352
353 // Try loading the buffer for this cache entry.
tryLoadingBuffer()354 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
355 if (EntryPath.empty())
356 return std::error_code();
357 SmallString<64> ResultPath;
358 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
359 Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
360 if (!FDOrErr)
361 return errorToErrorCode(FDOrErr.takeError());
362 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
363 *FDOrErr, EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
364 sys::fs::closeFile(*FDOrErr);
365 return MBOrErr;
366 }
367
368 // Cache the Produced object file
write(const MemoryBuffer & OutputBuffer)369 void write(const MemoryBuffer &OutputBuffer) {
370 if (EntryPath.empty())
371 return;
372
373 // Write to a temporary to avoid race condition
374 SmallString<128> TempFilename;
375 SmallString<128> CachePath(EntryPath);
376 llvm::sys::path::remove_filename(CachePath);
377 sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
378
379 if (auto Err = handleErrors(
380 llvm::writeFileAtomically(TempFilename, EntryPath,
381 OutputBuffer.getBuffer()),
382 [](const llvm::AtomicFileWriteError &E) {
383 std::string ErrorMsgBuffer;
384 llvm::raw_string_ostream S(ErrorMsgBuffer);
385 E.log(S);
386
387 if (E.Error ==
388 llvm::atomic_write_error::failed_to_create_uniq_file) {
389 errs() << "Error: " << ErrorMsgBuffer << "\n";
390 report_fatal_error("ThinLTO: Can't get a temporary file");
391 }
392 })) {
393 // FIXME
394 consumeError(std::move(Err));
395 }
396 }
397 };
398
399 static std::unique_ptr<MemoryBuffer>
ProcessThinLTOModule(Module & TheModule,ModuleSummaryIndex & Index,StringMap<lto::InputFile * > & ModuleMap,TargetMachine & TM,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const DenseSet<GlobalValue::GUID> & GUIDPreservedSymbols,const GVSummaryMapTy & DefinedGlobals,const ThinLTOCodeGenerator::CachingOptions & CacheOptions,bool DisableCodeGen,StringRef SaveTempsDir,bool Freestanding,unsigned OptLevel,unsigned count)400 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
401 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
402 const FunctionImporter::ImportMapTy &ImportList,
403 const FunctionImporter::ExportSetTy &ExportList,
404 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
405 const GVSummaryMapTy &DefinedGlobals,
406 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
407 bool DisableCodeGen, StringRef SaveTempsDir,
408 bool Freestanding, unsigned OptLevel, unsigned count) {
409
410 // "Benchmark"-like optimization: single-source case
411 bool SingleModule = (ModuleMap.size() == 1);
412
413 if (!SingleModule) {
414 promoteModule(TheModule, Index);
415
416 // Apply summary-based prevailing-symbol resolution decisions.
417 thinLTOResolvePrevailingInModule(TheModule, DefinedGlobals);
418
419 // Save temps: after promotion.
420 saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
421 }
422
423 // Be friendly and don't nuke totally the module when the client didn't
424 // supply anything to preserve.
425 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
426 // Apply summary-based internalization decisions.
427 thinLTOInternalizeModule(TheModule, DefinedGlobals);
428 }
429
430 // Save internalized bitcode
431 saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
432
433 if (!SingleModule) {
434 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
435
436 // Save temps: after cross-module import.
437 saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
438 }
439
440 optimizeModule(TheModule, TM, OptLevel, Freestanding, &Index);
441
442 saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
443
444 if (DisableCodeGen) {
445 // Configured to stop before CodeGen, serialize the bitcode and return.
446 SmallVector<char, 128> OutputBuffer;
447 {
448 raw_svector_ostream OS(OutputBuffer);
449 ProfileSummaryInfo PSI(TheModule);
450 auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
451 WriteBitcodeToFile(TheModule, OS, true, &Index);
452 }
453 return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
454 }
455
456 return codegenModule(TheModule, TM);
457 }
458
459 /// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
460 /// for caching, and in the \p Index for application during the ThinLTO
461 /// backends. This is needed for correctness for exported symbols (ensure
462 /// at least one copy kept) and a compile-time optimization (to drop duplicate
463 /// copies when possible).
resolvePrevailingInIndex(ModuleSummaryIndex & Index,StringMap<std::map<GlobalValue::GUID,GlobalValue::LinkageTypes>> & ResolvedODR,const DenseSet<GlobalValue::GUID> & GUIDPreservedSymbols,const DenseMap<GlobalValue::GUID,const GlobalValueSummary * > & PrevailingCopy)464 static void resolvePrevailingInIndex(
465 ModuleSummaryIndex &Index,
466 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
467 &ResolvedODR,
468 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
469 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
470 &PrevailingCopy) {
471
472 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
473 const auto &Prevailing = PrevailingCopy.find(GUID);
474 // Not in map means that there was only one copy, which must be prevailing.
475 if (Prevailing == PrevailingCopy.end())
476 return true;
477 return Prevailing->second == S;
478 };
479
480 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
481 GlobalValue::GUID GUID,
482 GlobalValue::LinkageTypes NewLinkage) {
483 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
484 };
485
486 thinLTOResolvePrevailingInIndex(Index, isPrevailing, recordNewLinkage,
487 GUIDPreservedSymbols);
488 }
489
490 // Initialize the TargetMachine builder for a given Triple
initTMBuilder(TargetMachineBuilder & TMBuilder,const Triple & TheTriple)491 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
492 const Triple &TheTriple) {
493 // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
494 // FIXME this looks pretty terrible...
495 if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
496 if (TheTriple.getArch() == llvm::Triple::x86_64)
497 TMBuilder.MCpu = "core2";
498 else if (TheTriple.getArch() == llvm::Triple::x86)
499 TMBuilder.MCpu = "yonah";
500 else if (TheTriple.getArch() == llvm::Triple::aarch64 ||
501 TheTriple.getArch() == llvm::Triple::aarch64_32)
502 TMBuilder.MCpu = "cyclone";
503 }
504 TMBuilder.TheTriple = std::move(TheTriple);
505 }
506
507 } // end anonymous namespace
508
addModule(StringRef Identifier,StringRef Data)509 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
510 MemoryBufferRef Buffer(Data, Identifier);
511
512 auto InputOrError = lto::InputFile::create(Buffer);
513 if (!InputOrError)
514 report_fatal_error("ThinLTO cannot create input file: " +
515 toString(InputOrError.takeError()));
516
517 auto TripleStr = (*InputOrError)->getTargetTriple();
518 Triple TheTriple(TripleStr);
519
520 if (Modules.empty())
521 initTMBuilder(TMBuilder, Triple(TheTriple));
522 else if (TMBuilder.TheTriple != TheTriple) {
523 if (!TMBuilder.TheTriple.isCompatibleWith(TheTriple))
524 report_fatal_error("ThinLTO modules with incompatible triples not "
525 "supported");
526 initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
527 }
528
529 Modules.emplace_back(std::move(*InputOrError));
530 }
531
preserveSymbol(StringRef Name)532 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
533 PreservedSymbols.insert(Name);
534 }
535
crossReferenceSymbol(StringRef Name)536 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
537 // FIXME: At the moment, we don't take advantage of this extra information,
538 // we're conservatively considering cross-references as preserved.
539 // CrossReferencedSymbols.insert(Name);
540 PreservedSymbols.insert(Name);
541 }
542
543 // TargetMachine factory
create() const544 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
545 std::string ErrMsg;
546 const Target *TheTarget =
547 TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
548 if (!TheTarget) {
549 report_fatal_error("Can't load target for this Triple: " + ErrMsg);
550 }
551
552 // Use MAttr as the default set of features.
553 SubtargetFeatures Features(MAttr);
554 Features.getDefaultSubtargetFeatures(TheTriple);
555 std::string FeatureStr = Features.getString();
556
557 return std::unique_ptr<TargetMachine>(
558 TheTarget->createTargetMachine(TheTriple.str(), MCpu, FeatureStr, Options,
559 RelocModel, None, CGOptLevel));
560 }
561
562 /**
563 * Produce the combined summary index from all the bitcode files:
564 * "thin-link".
565 */
linkCombinedIndex()566 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
567 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
568 std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
569 uint64_t NextModuleId = 0;
570 for (auto &Mod : Modules) {
571 auto &M = Mod->getSingleBitcodeModule();
572 if (Error Err =
573 M.readSummary(*CombinedIndex, Mod->getName(), NextModuleId++)) {
574 // FIXME diagnose
575 logAllUnhandledErrors(
576 std::move(Err), errs(),
577 "error: can't create module summary index for buffer: ");
578 return nullptr;
579 }
580 }
581 return CombinedIndex;
582 }
583
584 namespace {
585 struct IsExported {
586 const StringMap<FunctionImporter::ExportSetTy> &ExportLists;
587 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
588
IsExported__anon9ad760780d11::IsExported589 IsExported(const StringMap<FunctionImporter::ExportSetTy> &ExportLists,
590 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
591 : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
592
operator ()__anon9ad760780d11::IsExported593 bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
594 const auto &ExportList = ExportLists.find(ModuleIdentifier);
595 return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
596 GUIDPreservedSymbols.count(VI.getGUID());
597 }
598 };
599
600 struct IsPrevailing {
601 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
IsPrevailing__anon9ad760780d11::IsPrevailing602 IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
603 &PrevailingCopy)
604 : PrevailingCopy(PrevailingCopy) {}
605
operator ()__anon9ad760780d11::IsPrevailing606 bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
607 const auto &Prevailing = PrevailingCopy.find(GUID);
608 // Not in map means that there was only one copy, which must be prevailing.
609 if (Prevailing == PrevailingCopy.end())
610 return true;
611 return Prevailing->second == S;
612 };
613 };
614 } // namespace
615
computeDeadSymbolsInIndex(ModuleSummaryIndex & Index,const DenseSet<GlobalValue::GUID> & GUIDPreservedSymbols)616 static void computeDeadSymbolsInIndex(
617 ModuleSummaryIndex &Index,
618 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
619 // We have no symbols resolution available. And can't do any better now in the
620 // case where the prevailing symbol is in a native object. It can be refined
621 // with linker information in the future.
622 auto isPrevailing = [&](GlobalValue::GUID G) {
623 return PrevailingType::Unknown;
624 };
625 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
626 /* ImportEnabled = */ true);
627 }
628
629 /**
630 * Perform promotion and renaming of exported internal functions.
631 * Index is updated to reflect linkage changes from weak resolution.
632 */
promote(Module & TheModule,ModuleSummaryIndex & Index,const lto::InputFile & File)633 void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
634 const lto::InputFile &File) {
635 auto ModuleCount = Index.modulePaths().size();
636 auto ModuleIdentifier = TheModule.getModuleIdentifier();
637
638 // Collect for each module the list of function it defines (GUID -> Summary).
639 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
640 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
641
642 // Convert the preserved symbols set from string to GUID
643 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
644 PreservedSymbols, Triple(TheModule.getTargetTriple()));
645
646 // Add used symbol to the preserved symbols.
647 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
648
649 // Compute "dead" symbols, we don't want to import/export these!
650 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
651
652 // Generate import/export list
653 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
654 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
655 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
656 ExportLists);
657
658 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
659 computePrevailingCopies(Index, PrevailingCopy);
660
661 // Resolve prevailing symbols
662 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
663 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
664 PrevailingCopy);
665
666 thinLTOResolvePrevailingInModule(
667 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
668
669 // Promote the exported values in the index, so that they are promoted
670 // in the module.
671 thinLTOInternalizeAndPromoteInIndex(
672 Index, IsExported(ExportLists, GUIDPreservedSymbols),
673 IsPrevailing(PrevailingCopy));
674
675 promoteModule(TheModule, Index);
676 }
677
678 /**
679 * Perform cross-module importing for the module identified by ModuleIdentifier.
680 */
crossModuleImport(Module & TheModule,ModuleSummaryIndex & Index,const lto::InputFile & File)681 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
682 ModuleSummaryIndex &Index,
683 const lto::InputFile &File) {
684 auto ModuleMap = generateModuleMap(Modules);
685 auto ModuleCount = Index.modulePaths().size();
686
687 // Collect for each module the list of function it defines (GUID -> Summary).
688 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
689 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
690
691 // Convert the preserved symbols set from string to GUID
692 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
693 PreservedSymbols, Triple(TheModule.getTargetTriple()));
694
695 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
696
697 // Compute "dead" symbols, we don't want to import/export these!
698 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
699
700 // Generate import/export list
701 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
702 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
703 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
704 ExportLists);
705 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
706
707 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
708 }
709
710 /**
711 * Compute the list of summaries needed for importing into module.
712 */
gatherImportedSummariesForModule(Module & TheModule,ModuleSummaryIndex & Index,std::map<std::string,GVSummaryMapTy> & ModuleToSummariesForIndex,const lto::InputFile & File)713 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
714 Module &TheModule, ModuleSummaryIndex &Index,
715 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
716 const lto::InputFile &File) {
717 auto ModuleCount = Index.modulePaths().size();
718 auto ModuleIdentifier = TheModule.getModuleIdentifier();
719
720 // Collect for each module the list of function it defines (GUID -> Summary).
721 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
722 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
723
724 // Convert the preserved symbols set from string to GUID
725 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
726 PreservedSymbols, Triple(TheModule.getTargetTriple()));
727
728 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
729
730 // Compute "dead" symbols, we don't want to import/export these!
731 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
732
733 // Generate import/export list
734 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
735 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
736 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
737 ExportLists);
738
739 llvm::gatherImportedSummariesForModule(
740 ModuleIdentifier, ModuleToDefinedGVSummaries,
741 ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
742 }
743
744 /**
745 * Emit the list of files needed for importing into module.
746 */
emitImports(Module & TheModule,StringRef OutputName,ModuleSummaryIndex & Index,const lto::InputFile & File)747 void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
748 ModuleSummaryIndex &Index,
749 const lto::InputFile &File) {
750 auto ModuleCount = Index.modulePaths().size();
751 auto ModuleIdentifier = TheModule.getModuleIdentifier();
752
753 // Collect for each module the list of function it defines (GUID -> Summary).
754 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
755 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
756
757 // Convert the preserved symbols set from string to GUID
758 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
759 PreservedSymbols, Triple(TheModule.getTargetTriple()));
760
761 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
762
763 // Compute "dead" symbols, we don't want to import/export these!
764 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
765
766 // Generate import/export list
767 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
768 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
769 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
770 ExportLists);
771
772 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
773 llvm::gatherImportedSummariesForModule(
774 ModuleIdentifier, ModuleToDefinedGVSummaries,
775 ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
776
777 std::error_code EC;
778 if ((EC = EmitImportsFiles(ModuleIdentifier, OutputName,
779 ModuleToSummariesForIndex)))
780 report_fatal_error(Twine("Failed to open ") + OutputName +
781 " to save imports lists\n");
782 }
783
784 /**
785 * Perform internalization. Runs promote and internalization together.
786 * Index is updated to reflect linkage changes.
787 */
internalize(Module & TheModule,ModuleSummaryIndex & Index,const lto::InputFile & File)788 void ThinLTOCodeGenerator::internalize(Module &TheModule,
789 ModuleSummaryIndex &Index,
790 const lto::InputFile &File) {
791 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
792 auto ModuleCount = Index.modulePaths().size();
793 auto ModuleIdentifier = TheModule.getModuleIdentifier();
794
795 // Convert the preserved symbols set from string to GUID
796 auto GUIDPreservedSymbols =
797 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
798
799 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
800
801 // Collect for each module the list of function it defines (GUID -> Summary).
802 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
803 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
804
805 // Compute "dead" symbols, we don't want to import/export these!
806 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
807
808 // Generate import/export list
809 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
810 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
811 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
812 ExportLists);
813 auto &ExportList = ExportLists[ModuleIdentifier];
814
815 // Be friendly and don't nuke totally the module when the client didn't
816 // supply anything to preserve.
817 if (ExportList.empty() && GUIDPreservedSymbols.empty())
818 return;
819
820 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
821 computePrevailingCopies(Index, PrevailingCopy);
822
823 // Resolve prevailing symbols
824 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
825 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
826 PrevailingCopy);
827
828 // Promote the exported values in the index, so that they are promoted
829 // in the module.
830 thinLTOInternalizeAndPromoteInIndex(
831 Index, IsExported(ExportLists, GUIDPreservedSymbols),
832 IsPrevailing(PrevailingCopy));
833
834 promoteModule(TheModule, Index);
835
836 // Internalization
837 thinLTOResolvePrevailingInModule(
838 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
839
840 thinLTOInternalizeModule(TheModule,
841 ModuleToDefinedGVSummaries[ModuleIdentifier]);
842 }
843
844 /**
845 * Perform post-importing ThinLTO optimizations.
846 */
optimize(Module & TheModule)847 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
848 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
849
850 // Optimize now
851 optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding,
852 nullptr);
853 }
854
855 /// Write out the generated object file, either from CacheEntryPath or from
856 /// OutputBuffer, preferring hard-link when possible.
857 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
858 std::string
writeGeneratedObject(int count,StringRef CacheEntryPath,const MemoryBuffer & OutputBuffer)859 ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
860 const MemoryBuffer &OutputBuffer) {
861 auto ArchName = TMBuilder.TheTriple.getArchName();
862 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
863 llvm::sys::path::append(OutputPath,
864 Twine(count) + "." + ArchName + ".thinlto.o");
865 OutputPath.c_str(); // Ensure the string is null terminated.
866 if (sys::fs::exists(OutputPath))
867 sys::fs::remove(OutputPath);
868
869 // We don't return a memory buffer to the linker, just a list of files.
870 if (!CacheEntryPath.empty()) {
871 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
872 auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
873 if (!Err)
874 return OutputPath.str();
875 // Hard linking failed, try to copy.
876 Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
877 if (!Err)
878 return OutputPath.str();
879 // Copy failed (could be because the CacheEntry was removed from the cache
880 // in the meantime by another process), fall back and try to write down the
881 // buffer to the output.
882 errs() << "error: can't link or copy from cached entry '" << CacheEntryPath
883 << "' to '" << OutputPath << "'\n";
884 }
885 // No cache entry, just write out the buffer.
886 std::error_code Err;
887 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
888 if (Err)
889 report_fatal_error("Can't open output '" + OutputPath + "'\n");
890 OS << OutputBuffer.getBuffer();
891 return OutputPath.str();
892 }
893
894 // Main entry point for the ThinLTO processing
run()895 void ThinLTOCodeGenerator::run() {
896 // Prepare the resulting object vector
897 assert(ProducedBinaries.empty() && "The generator should not be reused");
898 if (SavedObjectsDirectoryPath.empty())
899 ProducedBinaries.resize(Modules.size());
900 else {
901 sys::fs::create_directories(SavedObjectsDirectoryPath);
902 bool IsDir;
903 sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
904 if (!IsDir)
905 report_fatal_error("Unexistent dir: '" + SavedObjectsDirectoryPath + "'");
906 ProducedBinaryFiles.resize(Modules.size());
907 }
908
909 if (CodeGenOnly) {
910 // Perform only parallel codegen and return.
911 ThreadPool Pool;
912 int count = 0;
913 for (auto &Mod : Modules) {
914 Pool.async([&](int count) {
915 LLVMContext Context;
916 Context.setDiscardValueNames(LTODiscardValueNames);
917
918 // Parse module now
919 auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
920 /*IsImporting*/ false);
921
922 // CodeGen
923 auto OutputBuffer = codegenModule(*TheModule, *TMBuilder.create());
924 if (SavedObjectsDirectoryPath.empty())
925 ProducedBinaries[count] = std::move(OutputBuffer);
926 else
927 ProducedBinaryFiles[count] =
928 writeGeneratedObject(count, "", *OutputBuffer);
929 }, count++);
930 }
931
932 return;
933 }
934
935 // Sequential linking phase
936 auto Index = linkCombinedIndex();
937
938 // Save temps: index.
939 if (!SaveTempsDir.empty()) {
940 auto SaveTempPath = SaveTempsDir + "index.bc";
941 std::error_code EC;
942 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
943 if (EC)
944 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
945 " to save optimized bitcode\n");
946 WriteIndexToFile(*Index, OS);
947 }
948
949
950 // Prepare the module map.
951 auto ModuleMap = generateModuleMap(Modules);
952 auto ModuleCount = Modules.size();
953
954 // Collect for each module the list of function it defines (GUID -> Summary).
955 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
956 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
957
958 // Convert the preserved symbols set from string to GUID, this is needed for
959 // computing the caching hash and the internalization.
960 auto GUIDPreservedSymbols =
961 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
962
963 // Add used symbol from inputs to the preserved symbols.
964 for (const auto &M : Modules)
965 addUsedSymbolToPreservedGUID(*M, GUIDPreservedSymbols);
966
967 // Compute "dead" symbols, we don't want to import/export these!
968 computeDeadSymbolsInIndex(*Index, GUIDPreservedSymbols);
969
970 // Synthesize entry counts for functions in the combined index.
971 computeSyntheticCounts(*Index);
972
973 // Perform index-based WPD. This will return immediately if there are
974 // no index entries in the typeIdMetadata map (e.g. if we are instead
975 // performing IR-based WPD in hybrid regular/thin LTO mode).
976 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
977 std::set<GlobalValue::GUID> ExportedGUIDs;
978 runWholeProgramDevirtOnIndex(*Index, ExportedGUIDs, LocalWPDTargetsMap);
979 for (auto GUID : ExportedGUIDs)
980 GUIDPreservedSymbols.insert(GUID);
981
982 // Collect the import/export lists for all modules from the call-graph in the
983 // combined index.
984 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
985 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
986 ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
987 ExportLists);
988
989 // We use a std::map here to be able to have a defined ordering when
990 // producing a hash for the cache entry.
991 // FIXME: we should be able to compute the caching hash for the entry based
992 // on the index, and nuke this map.
993 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
994
995 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
996 computePrevailingCopies(*Index, PrevailingCopy);
997
998 // Resolve prevailing symbols, this has to be computed early because it
999 // impacts the caching.
1000 resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols,
1001 PrevailingCopy);
1002
1003 // Use global summary-based analysis to identify symbols that can be
1004 // internalized (because they aren't exported or preserved as per callback).
1005 // Changes are made in the index, consumed in the ThinLTO backends.
1006 updateIndexWPDForExports(*Index,
1007 IsExported(ExportLists, GUIDPreservedSymbols),
1008 LocalWPDTargetsMap);
1009 thinLTOInternalizeAndPromoteInIndex(
1010 *Index, IsExported(ExportLists, GUIDPreservedSymbols),
1011 IsPrevailing(PrevailingCopy));
1012
1013 // Make sure that every module has an entry in the ExportLists, ImportList,
1014 // GVSummary and ResolvedODR maps to enable threaded access to these maps
1015 // below.
1016 for (auto &Module : Modules) {
1017 auto ModuleIdentifier = Module->getName();
1018 ExportLists[ModuleIdentifier];
1019 ImportLists[ModuleIdentifier];
1020 ResolvedODR[ModuleIdentifier];
1021 ModuleToDefinedGVSummaries[ModuleIdentifier];
1022 }
1023
1024 // Compute the ordering we will process the inputs: the rough heuristic here
1025 // is to sort them per size so that the largest module get schedule as soon as
1026 // possible. This is purely a compile-time optimization.
1027 std::vector<int> ModulesOrdering;
1028 ModulesOrdering.resize(Modules.size());
1029 std::iota(ModulesOrdering.begin(), ModulesOrdering.end(), 0);
1030 llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
1031 auto LSize =
1032 Modules[LeftIndex]->getSingleBitcodeModule().getBuffer().size();
1033 auto RSize =
1034 Modules[RightIndex]->getSingleBitcodeModule().getBuffer().size();
1035 return LSize > RSize;
1036 });
1037
1038 // Parallel optimizer + codegen
1039 {
1040 ThreadPool Pool(ThreadCount);
1041 for (auto IndexCount : ModulesOrdering) {
1042 auto &Mod = Modules[IndexCount];
1043 Pool.async([&](int count) {
1044 auto ModuleIdentifier = Mod->getName();
1045 auto &ExportList = ExportLists[ModuleIdentifier];
1046
1047 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1048
1049 // The module may be cached, this helps handling it.
1050 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1051 ImportLists[ModuleIdentifier], ExportList,
1052 ResolvedODR[ModuleIdentifier],
1053 DefinedGVSummaries, OptLevel, Freestanding,
1054 TMBuilder);
1055 auto CacheEntryPath = CacheEntry.getEntryPath();
1056
1057 {
1058 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1059 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1060 << " '" << CacheEntryPath << "' for buffer "
1061 << count << " " << ModuleIdentifier << "\n");
1062
1063 if (ErrOrBuffer) {
1064 // Cache Hit!
1065 if (SavedObjectsDirectoryPath.empty())
1066 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1067 else
1068 ProducedBinaryFiles[count] = writeGeneratedObject(
1069 count, CacheEntryPath, *ErrOrBuffer.get());
1070 return;
1071 }
1072 }
1073
1074 LLVMContext Context;
1075 Context.setDiscardValueNames(LTODiscardValueNames);
1076 Context.enableDebugTypeODRUniquing();
1077 auto DiagFileOrErr = lto::setupOptimizationRemarks(
1078 Context, RemarksFilename, RemarksPasses, RemarksFormat,
1079 RemarksWithHotness, count);
1080 if (!DiagFileOrErr) {
1081 errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
1082 report_fatal_error("ThinLTO: Can't get an output file for the "
1083 "remarks");
1084 }
1085
1086 // Parse module now
1087 auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1088 /*IsImporting*/ false);
1089
1090 // Save temps: original file.
1091 saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
1092
1093 auto &ImportList = ImportLists[ModuleIdentifier];
1094 // Run the main process now, and generates a binary
1095 auto OutputBuffer = ProcessThinLTOModule(
1096 *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
1097 ExportList, GUIDPreservedSymbols,
1098 ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1099 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count);
1100
1101 // Commit to the cache (if enabled)
1102 CacheEntry.write(*OutputBuffer);
1103
1104 if (SavedObjectsDirectoryPath.empty()) {
1105 // We need to generated a memory buffer for the linker.
1106 if (!CacheEntryPath.empty()) {
1107 // When cache is enabled, reload from the cache if possible.
1108 // Releasing the buffer from the heap and reloading it from the
1109 // cache file with mmap helps us to lower memory pressure.
1110 // The freed memory can be used for the next input file.
1111 // The final binary link will read from the VFS cache (hopefully!)
1112 // or from disk (if the memory pressure was too high).
1113 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1114 if (auto EC = ReloadedBufferOrErr.getError()) {
1115 // On error, keep the preexisting buffer and print a diagnostic.
1116 errs() << "error: can't reload cached file '" << CacheEntryPath
1117 << "': " << EC.message() << "\n";
1118 } else {
1119 OutputBuffer = std::move(*ReloadedBufferOrErr);
1120 }
1121 }
1122 ProducedBinaries[count] = std::move(OutputBuffer);
1123 return;
1124 }
1125 ProducedBinaryFiles[count] = writeGeneratedObject(
1126 count, CacheEntryPath, *OutputBuffer);
1127 }, IndexCount);
1128 }
1129 }
1130
1131 pruneCache(CacheOptions.Path, CacheOptions.Policy);
1132
1133 // If statistics were requested, print them out now.
1134 if (llvm::AreStatisticsEnabled())
1135 llvm::PrintStatistics();
1136 reportAndResetTimings();
1137 }
1138