1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Thin Link Time Optimization library. This library is
10 // intended to be used by linker to optimize code at link time.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
15 #include "llvm/Support/CommandLine.h"
16
17 #include "llvm/ADT/Statistic.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
20 #include "llvm/Analysis/ProfileSummaryInfo.h"
21 #include "llvm/Analysis/TargetLibraryInfo.h"
22 #include "llvm/Analysis/TargetTransformInfo.h"
23 #include "llvm/Bitcode/BitcodeReader.h"
24 #include "llvm/Bitcode/BitcodeWriter.h"
25 #include "llvm/Bitcode/BitcodeWriterPass.h"
26 #include "llvm/Config/llvm-config.h"
27 #include "llvm/IR/DebugInfo.h"
28 #include "llvm/IR/DiagnosticPrinter.h"
29 #include "llvm/IR/LLVMContext.h"
30 #include "llvm/IR/LLVMRemarkStreamer.h"
31 #include "llvm/IR/LegacyPassManager.h"
32 #include "llvm/IR/Mangler.h"
33 #include "llvm/IR/PassTimingInfo.h"
34 #include "llvm/IR/Verifier.h"
35 #include "llvm/IRReader/IRReader.h"
36 #include "llvm/LTO/LTO.h"
37 #include "llvm/LTO/SummaryBasedOptimizations.h"
38 #include "llvm/MC/SubtargetFeature.h"
39 #include "llvm/Object/IRObjectFile.h"
40 #include "llvm/Support/CachePruning.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/Error.h"
43 #include "llvm/Support/FileUtilities.h"
44 #include "llvm/Support/Path.h"
45 #include "llvm/Support/SHA1.h"
46 #include "llvm/Support/SmallVectorMemoryBuffer.h"
47 #include "llvm/Support/TargetRegistry.h"
48 #include "llvm/Support/ThreadPool.h"
49 #include "llvm/Support/Threading.h"
50 #include "llvm/Support/ToolOutputFile.h"
51 #include "llvm/Target/TargetMachine.h"
52 #include "llvm/Transforms/IPO.h"
53 #include "llvm/Transforms/IPO/FunctionImport.h"
54 #include "llvm/Transforms/IPO/Internalize.h"
55 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
56 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
57 #include "llvm/Transforms/ObjCARC.h"
58 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
59
60 #include <numeric>
61
62 #if !defined(_MSC_VER) && !defined(__MINGW32__)
63 #include <unistd.h>
64 #else
65 #include <io.h>
66 #endif
67
68 using namespace llvm;
69
70 #define DEBUG_TYPE "thinlto"
71
72 namespace llvm {
73 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
74 extern cl::opt<bool> LTODiscardValueNames;
75 extern cl::opt<std::string> RemarksFilename;
76 extern cl::opt<std::string> RemarksPasses;
77 extern cl::opt<bool> RemarksWithHotness;
78 extern cl::opt<std::string> RemarksFormat;
79 }
80
81 namespace {
82
83 // Default to using all available threads in the system, but using only one
84 // thred per core, as indicated by the usage of
85 // heavyweight_hardware_concurrency() below.
86 static cl::opt<int> ThreadCount("threads", cl::init(0));
87
88 // Simple helper to save temporary files for debug.
saveTempBitcode(const Module & TheModule,StringRef TempDir,unsigned count,StringRef Suffix)89 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
90 unsigned count, StringRef Suffix) {
91 if (TempDir.empty())
92 return;
93 // User asked to save temps, let dump the bitcode file after import.
94 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
95 std::error_code EC;
96 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
97 if (EC)
98 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
99 " to save optimized bitcode\n");
100 WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
101 }
102
103 static const GlobalValueSummary *
getFirstDefinitionForLinker(const GlobalValueSummaryList & GVSummaryList)104 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
105 // If there is any strong definition anywhere, get it.
106 auto StrongDefForLinker = llvm::find_if(
107 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
108 auto Linkage = Summary->linkage();
109 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
110 !GlobalValue::isWeakForLinker(Linkage);
111 });
112 if (StrongDefForLinker != GVSummaryList.end())
113 return StrongDefForLinker->get();
114 // Get the first *linker visible* definition for this global in the summary
115 // list.
116 auto FirstDefForLinker = llvm::find_if(
117 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
118 auto Linkage = Summary->linkage();
119 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
120 });
121 // Extern templates can be emitted as available_externally.
122 if (FirstDefForLinker == GVSummaryList.end())
123 return nullptr;
124 return FirstDefForLinker->get();
125 }
126
127 // Populate map of GUID to the prevailing copy for any multiply defined
128 // symbols. Currently assume first copy is prevailing, or any strong
129 // definition. Can be refined with Linker information in the future.
computePrevailingCopies(const ModuleSummaryIndex & Index,DenseMap<GlobalValue::GUID,const GlobalValueSummary * > & PrevailingCopy)130 static void computePrevailingCopies(
131 const ModuleSummaryIndex &Index,
132 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
133 auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
134 return GVSummaryList.size() > 1;
135 };
136
137 for (auto &I : Index) {
138 if (HasMultipleCopies(I.second.SummaryList))
139 PrevailingCopy[I.first] =
140 getFirstDefinitionForLinker(I.second.SummaryList);
141 }
142 }
143
144 static StringMap<lto::InputFile *>
generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> & Modules)145 generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
146 StringMap<lto::InputFile *> ModuleMap;
147 for (auto &M : Modules) {
148 assert(ModuleMap.find(M->getName()) == ModuleMap.end() &&
149 "Expect unique Buffer Identifier");
150 ModuleMap[M->getName()] = M.get();
151 }
152 return ModuleMap;
153 }
154
promoteModule(Module & TheModule,const ModuleSummaryIndex & Index,bool ClearDSOLocalOnDeclarations)155 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
156 bool ClearDSOLocalOnDeclarations) {
157 if (renameModuleForThinLTO(TheModule, Index, ClearDSOLocalOnDeclarations))
158 report_fatal_error("renameModuleForThinLTO failed");
159 }
160
161 namespace {
162 class ThinLTODiagnosticInfo : public DiagnosticInfo {
163 const Twine &Msg;
164 public:
ThinLTODiagnosticInfo(const Twine & DiagMsg,DiagnosticSeverity Severity=DS_Error)165 ThinLTODiagnosticInfo(const Twine &DiagMsg,
166 DiagnosticSeverity Severity = DS_Error)
167 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
print(DiagnosticPrinter & DP) const168 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
169 };
170 }
171
172 /// Verify the module and strip broken debug info.
verifyLoadedModule(Module & TheModule)173 static void verifyLoadedModule(Module &TheModule) {
174 bool BrokenDebugInfo = false;
175 if (verifyModule(TheModule, &dbgs(), &BrokenDebugInfo))
176 report_fatal_error("Broken module found, compilation aborted!");
177 if (BrokenDebugInfo) {
178 TheModule.getContext().diagnose(ThinLTODiagnosticInfo(
179 "Invalid debug info found, debug info will be stripped", DS_Warning));
180 StripDebugInfo(TheModule);
181 }
182 }
183
loadModuleFromInput(lto::InputFile * Input,LLVMContext & Context,bool Lazy,bool IsImporting)184 static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
185 LLVMContext &Context,
186 bool Lazy,
187 bool IsImporting) {
188 auto &Mod = Input->getSingleBitcodeModule();
189 SMDiagnostic Err;
190 Expected<std::unique_ptr<Module>> ModuleOrErr =
191 Lazy ? Mod.getLazyModule(Context,
192 /* ShouldLazyLoadMetadata */ true, IsImporting)
193 : Mod.parseModule(Context);
194 if (!ModuleOrErr) {
195 handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
196 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
197 SourceMgr::DK_Error, EIB.message());
198 Err.print("ThinLTO", errs());
199 });
200 report_fatal_error("Can't load module, abort.");
201 }
202 if (!Lazy)
203 verifyLoadedModule(*ModuleOrErr.get());
204 return std::move(*ModuleOrErr);
205 }
206
207 static void
crossImportIntoModule(Module & TheModule,const ModuleSummaryIndex & Index,StringMap<lto::InputFile * > & ModuleMap,const FunctionImporter::ImportMapTy & ImportList,bool ClearDSOLocalOnDeclarations)208 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
209 StringMap<lto::InputFile *> &ModuleMap,
210 const FunctionImporter::ImportMapTy &ImportList,
211 bool ClearDSOLocalOnDeclarations) {
212 auto Loader = [&](StringRef Identifier) {
213 auto &Input = ModuleMap[Identifier];
214 return loadModuleFromInput(Input, TheModule.getContext(),
215 /*Lazy=*/true, /*IsImporting*/ true);
216 };
217
218 FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
219 Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
220 if (!Result) {
221 handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
222 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
223 SourceMgr::DK_Error, EIB.message());
224 Err.print("ThinLTO", errs());
225 });
226 report_fatal_error("importFunctions failed");
227 }
228 // Verify again after cross-importing.
229 verifyLoadedModule(TheModule);
230 }
231
optimizeModule(Module & TheModule,TargetMachine & TM,unsigned OptLevel,bool Freestanding,ModuleSummaryIndex * Index)232 static void optimizeModule(Module &TheModule, TargetMachine &TM,
233 unsigned OptLevel, bool Freestanding,
234 ModuleSummaryIndex *Index) {
235 // Populate the PassManager
236 PassManagerBuilder PMB;
237 PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
238 if (Freestanding)
239 PMB.LibraryInfo->disableAllFunctions();
240 PMB.Inliner = createFunctionInliningPass();
241 // FIXME: should get it from the bitcode?
242 PMB.OptLevel = OptLevel;
243 PMB.LoopVectorize = true;
244 PMB.SLPVectorize = true;
245 // Already did this in verifyLoadedModule().
246 PMB.VerifyInput = false;
247 PMB.VerifyOutput = false;
248 PMB.ImportSummary = Index;
249
250 legacy::PassManager PM;
251
252 // Add the TTI (required to inform the vectorizer about register size for
253 // instance)
254 PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
255
256 // Add optimizations
257 PMB.populateThinLTOPassManager(PM);
258
259 PM.run(TheModule);
260 }
261
262 static void
addUsedSymbolToPreservedGUID(const lto::InputFile & File,DenseSet<GlobalValue::GUID> & PreservedGUID)263 addUsedSymbolToPreservedGUID(const lto::InputFile &File,
264 DenseSet<GlobalValue::GUID> &PreservedGUID) {
265 for (const auto &Sym : File.symbols()) {
266 if (Sym.isUsed())
267 PreservedGUID.insert(GlobalValue::getGUID(Sym.getIRName()));
268 }
269 }
270
271 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
computeGUIDPreservedSymbols(const lto::InputFile & File,const StringSet<> & PreservedSymbols,const Triple & TheTriple,DenseSet<GlobalValue::GUID> & GUIDs)272 static void computeGUIDPreservedSymbols(const lto::InputFile &File,
273 const StringSet<> &PreservedSymbols,
274 const Triple &TheTriple,
275 DenseSet<GlobalValue::GUID> &GUIDs) {
276 // Iterate the symbols in the input file and if the input has preserved symbol
277 // compute the GUID for the symbol.
278 for (const auto &Sym : File.symbols()) {
279 if (PreservedSymbols.count(Sym.getName()) && !Sym.getIRName().empty())
280 GUIDs.insert(GlobalValue::getGUID(GlobalValue::getGlobalIdentifier(
281 Sym.getIRName(), GlobalValue::ExternalLinkage, "")));
282 }
283 }
284
285 static DenseSet<GlobalValue::GUID>
computeGUIDPreservedSymbols(const lto::InputFile & File,const StringSet<> & PreservedSymbols,const Triple & TheTriple)286 computeGUIDPreservedSymbols(const lto::InputFile &File,
287 const StringSet<> &PreservedSymbols,
288 const Triple &TheTriple) {
289 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
290 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
291 GUIDPreservedSymbols);
292 return GUIDPreservedSymbols;
293 }
294
codegenModule(Module & TheModule,TargetMachine & TM)295 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
296 TargetMachine &TM) {
297 SmallVector<char, 128> OutputBuffer;
298
299 // CodeGen
300 {
301 raw_svector_ostream OS(OutputBuffer);
302 legacy::PassManager PM;
303
304 // If the bitcode files contain ARC code and were compiled with optimization,
305 // the ObjCARCContractPass must be run, so do it unconditionally here.
306 PM.add(createObjCARCContractPass());
307
308 // Setup the codegen now.
309 if (TM.addPassesToEmitFile(PM, OS, nullptr, CGFT_ObjectFile,
310 /* DisableVerify */ true))
311 report_fatal_error("Failed to setup codegen");
312
313 // Run codegen now. resulting binary is in OutputBuffer.
314 PM.run(TheModule);
315 }
316 return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
317 }
318
319 /// Manage caching for a single Module.
320 class ModuleCacheEntry {
321 SmallString<128> EntryPath;
322
323 public:
324 // Create a cache entry. This compute a unique hash for the Module considering
325 // the current list of export/import, and offer an interface to query to
326 // access the content in the cache.
ModuleCacheEntry(StringRef CachePath,const ModuleSummaryIndex & Index,StringRef ModuleID,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const std::map<GlobalValue::GUID,GlobalValue::LinkageTypes> & ResolvedODR,const GVSummaryMapTy & DefinedGVSummaries,unsigned OptLevel,bool Freestanding,const TargetMachineBuilder & TMBuilder)327 ModuleCacheEntry(
328 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
329 const FunctionImporter::ImportMapTy &ImportList,
330 const FunctionImporter::ExportSetTy &ExportList,
331 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
332 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
333 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
334 if (CachePath.empty())
335 return;
336
337 if (!Index.modulePaths().count(ModuleID))
338 // The module does not have an entry, it can't have a hash at all
339 return;
340
341 if (all_of(Index.getModuleHash(ModuleID),
342 [](uint32_t V) { return V == 0; }))
343 // No hash entry, no caching!
344 return;
345
346 llvm::lto::Config Conf;
347 Conf.OptLevel = OptLevel;
348 Conf.Options = TMBuilder.Options;
349 Conf.CPU = TMBuilder.MCpu;
350 Conf.MAttrs.push_back(TMBuilder.MAttr);
351 Conf.RelocModel = TMBuilder.RelocModel;
352 Conf.CGOptLevel = TMBuilder.CGOptLevel;
353 Conf.Freestanding = Freestanding;
354 SmallString<40> Key;
355 computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
356 ResolvedODR, DefinedGVSummaries);
357
358 // This choice of file name allows the cache to be pruned (see pruneCache()
359 // in include/llvm/Support/CachePruning.h).
360 sys::path::append(EntryPath, CachePath, "llvmcache-" + Key);
361 }
362
363 // Access the path to this entry in the cache.
getEntryPath()364 StringRef getEntryPath() { return EntryPath; }
365
366 // Try loading the buffer for this cache entry.
tryLoadingBuffer()367 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
368 if (EntryPath.empty())
369 return std::error_code();
370 SmallString<64> ResultPath;
371 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
372 Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
373 if (!FDOrErr)
374 return errorToErrorCode(FDOrErr.takeError());
375 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
376 *FDOrErr, EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
377 sys::fs::closeFile(*FDOrErr);
378 return MBOrErr;
379 }
380
381 // Cache the Produced object file
write(const MemoryBuffer & OutputBuffer)382 void write(const MemoryBuffer &OutputBuffer) {
383 if (EntryPath.empty())
384 return;
385
386 // Write to a temporary to avoid race condition
387 SmallString<128> TempFilename;
388 SmallString<128> CachePath(EntryPath);
389 llvm::sys::path::remove_filename(CachePath);
390 sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
391
392 if (auto Err = handleErrors(
393 llvm::writeFileAtomically(TempFilename, EntryPath,
394 OutputBuffer.getBuffer()),
395 [](const llvm::AtomicFileWriteError &E) {
396 std::string ErrorMsgBuffer;
397 llvm::raw_string_ostream S(ErrorMsgBuffer);
398 E.log(S);
399
400 if (E.Error ==
401 llvm::atomic_write_error::failed_to_create_uniq_file) {
402 errs() << "Error: " << ErrorMsgBuffer << "\n";
403 report_fatal_error("ThinLTO: Can't get a temporary file");
404 }
405 })) {
406 // FIXME
407 consumeError(std::move(Err));
408 }
409 }
410 };
411
412 static std::unique_ptr<MemoryBuffer>
ProcessThinLTOModule(Module & TheModule,ModuleSummaryIndex & Index,StringMap<lto::InputFile * > & ModuleMap,TargetMachine & TM,const FunctionImporter::ImportMapTy & ImportList,const FunctionImporter::ExportSetTy & ExportList,const DenseSet<GlobalValue::GUID> & GUIDPreservedSymbols,const GVSummaryMapTy & DefinedGlobals,const ThinLTOCodeGenerator::CachingOptions & CacheOptions,bool DisableCodeGen,StringRef SaveTempsDir,bool Freestanding,unsigned OptLevel,unsigned count)413 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
414 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
415 const FunctionImporter::ImportMapTy &ImportList,
416 const FunctionImporter::ExportSetTy &ExportList,
417 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
418 const GVSummaryMapTy &DefinedGlobals,
419 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
420 bool DisableCodeGen, StringRef SaveTempsDir,
421 bool Freestanding, unsigned OptLevel, unsigned count) {
422
423 // "Benchmark"-like optimization: single-source case
424 bool SingleModule = (ModuleMap.size() == 1);
425
426 // When linking an ELF shared object, dso_local should be dropped. We
427 // conservatively do this for -fpic.
428 bool ClearDSOLocalOnDeclarations =
429 TM.getTargetTriple().isOSBinFormatELF() &&
430 TM.getRelocationModel() != Reloc::Static &&
431 TheModule.getPIELevel() == PIELevel::Default;
432
433 if (!SingleModule) {
434 promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
435
436 // Apply summary-based prevailing-symbol resolution decisions.
437 thinLTOResolvePrevailingInModule(TheModule, DefinedGlobals);
438
439 // Save temps: after promotion.
440 saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
441 }
442
443 // Be friendly and don't nuke totally the module when the client didn't
444 // supply anything to preserve.
445 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
446 // Apply summary-based internalization decisions.
447 thinLTOInternalizeModule(TheModule, DefinedGlobals);
448 }
449
450 // Save internalized bitcode
451 saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
452
453 if (!SingleModule) {
454 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
455 ClearDSOLocalOnDeclarations);
456
457 // Save temps: after cross-module import.
458 saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
459 }
460
461 optimizeModule(TheModule, TM, OptLevel, Freestanding, &Index);
462
463 saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
464
465 if (DisableCodeGen) {
466 // Configured to stop before CodeGen, serialize the bitcode and return.
467 SmallVector<char, 128> OutputBuffer;
468 {
469 raw_svector_ostream OS(OutputBuffer);
470 ProfileSummaryInfo PSI(TheModule);
471 auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
472 WriteBitcodeToFile(TheModule, OS, true, &Index);
473 }
474 return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
475 }
476
477 return codegenModule(TheModule, TM);
478 }
479
480 /// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
481 /// for caching, and in the \p Index for application during the ThinLTO
482 /// backends. This is needed for correctness for exported symbols (ensure
483 /// at least one copy kept) and a compile-time optimization (to drop duplicate
484 /// copies when possible).
resolvePrevailingInIndex(ModuleSummaryIndex & Index,StringMap<std::map<GlobalValue::GUID,GlobalValue::LinkageTypes>> & ResolvedODR,const DenseSet<GlobalValue::GUID> & GUIDPreservedSymbols,const DenseMap<GlobalValue::GUID,const GlobalValueSummary * > & PrevailingCopy)485 static void resolvePrevailingInIndex(
486 ModuleSummaryIndex &Index,
487 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
488 &ResolvedODR,
489 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
490 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
491 &PrevailingCopy) {
492
493 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
494 const auto &Prevailing = PrevailingCopy.find(GUID);
495 // Not in map means that there was only one copy, which must be prevailing.
496 if (Prevailing == PrevailingCopy.end())
497 return true;
498 return Prevailing->second == S;
499 };
500
501 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
502 GlobalValue::GUID GUID,
503 GlobalValue::LinkageTypes NewLinkage) {
504 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
505 };
506
507 thinLTOResolvePrevailingInIndex(Index, isPrevailing, recordNewLinkage,
508 GUIDPreservedSymbols);
509 }
510
511 // Initialize the TargetMachine builder for a given Triple
initTMBuilder(TargetMachineBuilder & TMBuilder,const Triple & TheTriple)512 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
513 const Triple &TheTriple) {
514 // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
515 // FIXME this looks pretty terrible...
516 if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
517 if (TheTriple.getArch() == llvm::Triple::x86_64)
518 TMBuilder.MCpu = "core2";
519 else if (TheTriple.getArch() == llvm::Triple::x86)
520 TMBuilder.MCpu = "yonah";
521 else if (TheTriple.getArch() == llvm::Triple::aarch64 ||
522 TheTriple.getArch() == llvm::Triple::aarch64_32)
523 TMBuilder.MCpu = "cyclone";
524 }
525 TMBuilder.TheTriple = std::move(TheTriple);
526 }
527
528 } // end anonymous namespace
529
addModule(StringRef Identifier,StringRef Data)530 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
531 MemoryBufferRef Buffer(Data, Identifier);
532
533 auto InputOrError = lto::InputFile::create(Buffer);
534 if (!InputOrError)
535 report_fatal_error("ThinLTO cannot create input file: " +
536 toString(InputOrError.takeError()));
537
538 auto TripleStr = (*InputOrError)->getTargetTriple();
539 Triple TheTriple(TripleStr);
540
541 if (Modules.empty())
542 initTMBuilder(TMBuilder, Triple(TheTriple));
543 else if (TMBuilder.TheTriple != TheTriple) {
544 if (!TMBuilder.TheTriple.isCompatibleWith(TheTriple))
545 report_fatal_error("ThinLTO modules with incompatible triples not "
546 "supported");
547 initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
548 }
549
550 Modules.emplace_back(std::move(*InputOrError));
551 }
552
preserveSymbol(StringRef Name)553 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
554 PreservedSymbols.insert(Name);
555 }
556
crossReferenceSymbol(StringRef Name)557 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
558 // FIXME: At the moment, we don't take advantage of this extra information,
559 // we're conservatively considering cross-references as preserved.
560 // CrossReferencedSymbols.insert(Name);
561 PreservedSymbols.insert(Name);
562 }
563
564 // TargetMachine factory
create() const565 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
566 std::string ErrMsg;
567 const Target *TheTarget =
568 TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
569 if (!TheTarget) {
570 report_fatal_error("Can't load target for this Triple: " + ErrMsg);
571 }
572
573 // Use MAttr as the default set of features.
574 SubtargetFeatures Features(MAttr);
575 Features.getDefaultSubtargetFeatures(TheTriple);
576 std::string FeatureStr = Features.getString();
577
578 return std::unique_ptr<TargetMachine>(
579 TheTarget->createTargetMachine(TheTriple.str(), MCpu, FeatureStr, Options,
580 RelocModel, None, CGOptLevel));
581 }
582
583 /**
584 * Produce the combined summary index from all the bitcode files:
585 * "thin-link".
586 */
linkCombinedIndex()587 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
588 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
589 std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
590 uint64_t NextModuleId = 0;
591 for (auto &Mod : Modules) {
592 auto &M = Mod->getSingleBitcodeModule();
593 if (Error Err =
594 M.readSummary(*CombinedIndex, Mod->getName(), NextModuleId++)) {
595 // FIXME diagnose
596 logAllUnhandledErrors(
597 std::move(Err), errs(),
598 "error: can't create module summary index for buffer: ");
599 return nullptr;
600 }
601 }
602 return CombinedIndex;
603 }
604
605 namespace {
606 struct IsExported {
607 const StringMap<FunctionImporter::ExportSetTy> &ExportLists;
608 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
609
IsExported__anon2058df2b0d11::IsExported610 IsExported(const StringMap<FunctionImporter::ExportSetTy> &ExportLists,
611 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
612 : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
613
operator ()__anon2058df2b0d11::IsExported614 bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
615 const auto &ExportList = ExportLists.find(ModuleIdentifier);
616 return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
617 GUIDPreservedSymbols.count(VI.getGUID());
618 }
619 };
620
621 struct IsPrevailing {
622 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
IsPrevailing__anon2058df2b0d11::IsPrevailing623 IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
624 &PrevailingCopy)
625 : PrevailingCopy(PrevailingCopy) {}
626
operator ()__anon2058df2b0d11::IsPrevailing627 bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
628 const auto &Prevailing = PrevailingCopy.find(GUID);
629 // Not in map means that there was only one copy, which must be prevailing.
630 if (Prevailing == PrevailingCopy.end())
631 return true;
632 return Prevailing->second == S;
633 };
634 };
635 } // namespace
636
computeDeadSymbolsInIndex(ModuleSummaryIndex & Index,const DenseSet<GlobalValue::GUID> & GUIDPreservedSymbols)637 static void computeDeadSymbolsInIndex(
638 ModuleSummaryIndex &Index,
639 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
640 // We have no symbols resolution available. And can't do any better now in the
641 // case where the prevailing symbol is in a native object. It can be refined
642 // with linker information in the future.
643 auto isPrevailing = [&](GlobalValue::GUID G) {
644 return PrevailingType::Unknown;
645 };
646 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
647 /* ImportEnabled = */ true);
648 }
649
650 /**
651 * Perform promotion and renaming of exported internal functions.
652 * Index is updated to reflect linkage changes from weak resolution.
653 */
promote(Module & TheModule,ModuleSummaryIndex & Index,const lto::InputFile & File)654 void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
655 const lto::InputFile &File) {
656 auto ModuleCount = Index.modulePaths().size();
657 auto ModuleIdentifier = TheModule.getModuleIdentifier();
658
659 // Collect for each module the list of function it defines (GUID -> Summary).
660 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
661 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
662
663 // Convert the preserved symbols set from string to GUID
664 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
665 File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
666
667 // Add used symbol to the preserved symbols.
668 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
669
670 // Compute "dead" symbols, we don't want to import/export these!
671 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
672
673 // Generate import/export list
674 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
675 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
676 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
677 ExportLists);
678
679 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
680 computePrevailingCopies(Index, PrevailingCopy);
681
682 // Resolve prevailing symbols
683 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
684 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
685 PrevailingCopy);
686
687 thinLTOResolvePrevailingInModule(
688 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
689
690 // Promote the exported values in the index, so that they are promoted
691 // in the module.
692 thinLTOInternalizeAndPromoteInIndex(
693 Index, IsExported(ExportLists, GUIDPreservedSymbols),
694 IsPrevailing(PrevailingCopy));
695
696 // FIXME Set ClearDSOLocalOnDeclarations.
697 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
698 }
699
700 /**
701 * Perform cross-module importing for the module identified by ModuleIdentifier.
702 */
crossModuleImport(Module & TheModule,ModuleSummaryIndex & Index,const lto::InputFile & File)703 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
704 ModuleSummaryIndex &Index,
705 const lto::InputFile &File) {
706 auto ModuleMap = generateModuleMap(Modules);
707 auto ModuleCount = Index.modulePaths().size();
708
709 // Collect for each module the list of function it defines (GUID -> Summary).
710 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
711 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
712
713 // Convert the preserved symbols set from string to GUID
714 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
715 File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
716
717 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
718
719 // Compute "dead" symbols, we don't want to import/export these!
720 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
721
722 // Generate import/export list
723 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
724 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
725 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
726 ExportLists);
727 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
728
729 // FIXME Set ClearDSOLocalOnDeclarations.
730 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
731 /*ClearDSOLocalOnDeclarations=*/false);
732 }
733
734 /**
735 * Compute the list of summaries needed for importing into module.
736 */
gatherImportedSummariesForModule(Module & TheModule,ModuleSummaryIndex & Index,std::map<std::string,GVSummaryMapTy> & ModuleToSummariesForIndex,const lto::InputFile & File)737 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
738 Module &TheModule, ModuleSummaryIndex &Index,
739 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
740 const lto::InputFile &File) {
741 auto ModuleCount = Index.modulePaths().size();
742 auto ModuleIdentifier = TheModule.getModuleIdentifier();
743
744 // Collect for each module the list of function it defines (GUID -> Summary).
745 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
746 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
747
748 // Convert the preserved symbols set from string to GUID
749 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
750 File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
751
752 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
753
754 // Compute "dead" symbols, we don't want to import/export these!
755 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
756
757 // Generate import/export list
758 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
759 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
760 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
761 ExportLists);
762
763 llvm::gatherImportedSummariesForModule(
764 ModuleIdentifier, ModuleToDefinedGVSummaries,
765 ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
766 }
767
768 /**
769 * Emit the list of files needed for importing into module.
770 */
emitImports(Module & TheModule,StringRef OutputName,ModuleSummaryIndex & Index,const lto::InputFile & File)771 void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
772 ModuleSummaryIndex &Index,
773 const lto::InputFile &File) {
774 auto ModuleCount = Index.modulePaths().size();
775 auto ModuleIdentifier = TheModule.getModuleIdentifier();
776
777 // Collect for each module the list of function it defines (GUID -> Summary).
778 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
779 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
780
781 // Convert the preserved symbols set from string to GUID
782 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
783 File, PreservedSymbols, Triple(TheModule.getTargetTriple()));
784
785 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
786
787 // Compute "dead" symbols, we don't want to import/export these!
788 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
789
790 // Generate import/export list
791 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
792 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
793 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
794 ExportLists);
795
796 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
797 llvm::gatherImportedSummariesForModule(
798 ModuleIdentifier, ModuleToDefinedGVSummaries,
799 ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
800
801 std::error_code EC;
802 if ((EC = EmitImportsFiles(ModuleIdentifier, OutputName,
803 ModuleToSummariesForIndex)))
804 report_fatal_error(Twine("Failed to open ") + OutputName +
805 " to save imports lists\n");
806 }
807
808 /**
809 * Perform internalization. Runs promote and internalization together.
810 * Index is updated to reflect linkage changes.
811 */
internalize(Module & TheModule,ModuleSummaryIndex & Index,const lto::InputFile & File)812 void ThinLTOCodeGenerator::internalize(Module &TheModule,
813 ModuleSummaryIndex &Index,
814 const lto::InputFile &File) {
815 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
816 auto ModuleCount = Index.modulePaths().size();
817 auto ModuleIdentifier = TheModule.getModuleIdentifier();
818
819 // Convert the preserved symbols set from string to GUID
820 auto GUIDPreservedSymbols =
821 computeGUIDPreservedSymbols(File, PreservedSymbols, TMBuilder.TheTriple);
822
823 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
824
825 // Collect for each module the list of function it defines (GUID -> Summary).
826 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
827 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
828
829 // Compute "dead" symbols, we don't want to import/export these!
830 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
831
832 // Generate import/export list
833 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
834 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
835 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
836 ExportLists);
837 auto &ExportList = ExportLists[ModuleIdentifier];
838
839 // Be friendly and don't nuke totally the module when the client didn't
840 // supply anything to preserve.
841 if (ExportList.empty() && GUIDPreservedSymbols.empty())
842 return;
843
844 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
845 computePrevailingCopies(Index, PrevailingCopy);
846
847 // Resolve prevailing symbols
848 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
849 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
850 PrevailingCopy);
851
852 // Promote the exported values in the index, so that they are promoted
853 // in the module.
854 thinLTOInternalizeAndPromoteInIndex(
855 Index, IsExported(ExportLists, GUIDPreservedSymbols),
856 IsPrevailing(PrevailingCopy));
857
858 // FIXME Set ClearDSOLocalOnDeclarations.
859 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
860
861 // Internalization
862 thinLTOResolvePrevailingInModule(
863 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
864
865 thinLTOInternalizeModule(TheModule,
866 ModuleToDefinedGVSummaries[ModuleIdentifier]);
867 }
868
869 /**
870 * Perform post-importing ThinLTO optimizations.
871 */
optimize(Module & TheModule)872 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
873 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
874
875 // Optimize now
876 optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding,
877 nullptr);
878 }
879
880 /// Write out the generated object file, either from CacheEntryPath or from
881 /// OutputBuffer, preferring hard-link when possible.
882 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
883 std::string
writeGeneratedObject(int count,StringRef CacheEntryPath,const MemoryBuffer & OutputBuffer)884 ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
885 const MemoryBuffer &OutputBuffer) {
886 auto ArchName = TMBuilder.TheTriple.getArchName();
887 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
888 llvm::sys::path::append(OutputPath,
889 Twine(count) + "." + ArchName + ".thinlto.o");
890 OutputPath.c_str(); // Ensure the string is null terminated.
891 if (sys::fs::exists(OutputPath))
892 sys::fs::remove(OutputPath);
893
894 // We don't return a memory buffer to the linker, just a list of files.
895 if (!CacheEntryPath.empty()) {
896 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
897 auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
898 if (!Err)
899 return std::string(OutputPath.str());
900 // Hard linking failed, try to copy.
901 Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
902 if (!Err)
903 return std::string(OutputPath.str());
904 // Copy failed (could be because the CacheEntry was removed from the cache
905 // in the meantime by another process), fall back and try to write down the
906 // buffer to the output.
907 errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
908 << "' to '" << OutputPath << "'\n";
909 }
910 // No cache entry, just write out the buffer.
911 std::error_code Err;
912 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
913 if (Err)
914 report_fatal_error("Can't open output '" + OutputPath + "'\n");
915 OS << OutputBuffer.getBuffer();
916 return std::string(OutputPath.str());
917 }
918
919 // Main entry point for the ThinLTO processing
run()920 void ThinLTOCodeGenerator::run() {
921 // Prepare the resulting object vector
922 assert(ProducedBinaries.empty() && "The generator should not be reused");
923 if (SavedObjectsDirectoryPath.empty())
924 ProducedBinaries.resize(Modules.size());
925 else {
926 sys::fs::create_directories(SavedObjectsDirectoryPath);
927 bool IsDir;
928 sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
929 if (!IsDir)
930 report_fatal_error("Unexistent dir: '" + SavedObjectsDirectoryPath + "'");
931 ProducedBinaryFiles.resize(Modules.size());
932 }
933
934 if (CodeGenOnly) {
935 // Perform only parallel codegen and return.
936 ThreadPool Pool;
937 int count = 0;
938 for (auto &Mod : Modules) {
939 Pool.async([&](int count) {
940 LLVMContext Context;
941 Context.setDiscardValueNames(LTODiscardValueNames);
942
943 // Parse module now
944 auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
945 /*IsImporting*/ false);
946
947 // CodeGen
948 auto OutputBuffer = codegenModule(*TheModule, *TMBuilder.create());
949 if (SavedObjectsDirectoryPath.empty())
950 ProducedBinaries[count] = std::move(OutputBuffer);
951 else
952 ProducedBinaryFiles[count] =
953 writeGeneratedObject(count, "", *OutputBuffer);
954 }, count++);
955 }
956
957 return;
958 }
959
960 // Sequential linking phase
961 auto Index = linkCombinedIndex();
962
963 // Save temps: index.
964 if (!SaveTempsDir.empty()) {
965 auto SaveTempPath = SaveTempsDir + "index.bc";
966 std::error_code EC;
967 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
968 if (EC)
969 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
970 " to save optimized bitcode\n");
971 WriteIndexToFile(*Index, OS);
972 }
973
974
975 // Prepare the module map.
976 auto ModuleMap = generateModuleMap(Modules);
977 auto ModuleCount = Modules.size();
978
979 // Collect for each module the list of function it defines (GUID -> Summary).
980 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
981 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
982
983 // Convert the preserved symbols set from string to GUID, this is needed for
984 // computing the caching hash and the internalization.
985 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
986 for (const auto &M : Modules)
987 computeGUIDPreservedSymbols(*M, PreservedSymbols, TMBuilder.TheTriple,
988 GUIDPreservedSymbols);
989
990 // Add used symbol from inputs to the preserved symbols.
991 for (const auto &M : Modules)
992 addUsedSymbolToPreservedGUID(*M, GUIDPreservedSymbols);
993
994 // Compute "dead" symbols, we don't want to import/export these!
995 computeDeadSymbolsInIndex(*Index, GUIDPreservedSymbols);
996
997 // Synthesize entry counts for functions in the combined index.
998 computeSyntheticCounts(*Index);
999
1000 // Currently there is no support for enabling whole program visibility via a
1001 // linker option in the old LTO API, but this call allows it to be specified
1002 // via the internal option. Must be done before WPD below.
1003 updateVCallVisibilityInIndex(*Index,
1004 /* WholeProgramVisibilityEnabledInLTO */ false);
1005
1006 // Perform index-based WPD. This will return immediately if there are
1007 // no index entries in the typeIdMetadata map (e.g. if we are instead
1008 // performing IR-based WPD in hybrid regular/thin LTO mode).
1009 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1010 std::set<GlobalValue::GUID> ExportedGUIDs;
1011 runWholeProgramDevirtOnIndex(*Index, ExportedGUIDs, LocalWPDTargetsMap);
1012 for (auto GUID : ExportedGUIDs)
1013 GUIDPreservedSymbols.insert(GUID);
1014
1015 // Collect the import/export lists for all modules from the call-graph in the
1016 // combined index.
1017 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
1018 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1019 ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
1020 ExportLists);
1021
1022 // We use a std::map here to be able to have a defined ordering when
1023 // producing a hash for the cache entry.
1024 // FIXME: we should be able to compute the caching hash for the entry based
1025 // on the index, and nuke this map.
1026 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1027
1028 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1029 computePrevailingCopies(*Index, PrevailingCopy);
1030
1031 // Resolve prevailing symbols, this has to be computed early because it
1032 // impacts the caching.
1033 resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols,
1034 PrevailingCopy);
1035
1036 // Use global summary-based analysis to identify symbols that can be
1037 // internalized (because they aren't exported or preserved as per callback).
1038 // Changes are made in the index, consumed in the ThinLTO backends.
1039 updateIndexWPDForExports(*Index,
1040 IsExported(ExportLists, GUIDPreservedSymbols),
1041 LocalWPDTargetsMap);
1042 thinLTOInternalizeAndPromoteInIndex(
1043 *Index, IsExported(ExportLists, GUIDPreservedSymbols),
1044 IsPrevailing(PrevailingCopy));
1045
1046 // Make sure that every module has an entry in the ExportLists, ImportList,
1047 // GVSummary and ResolvedODR maps to enable threaded access to these maps
1048 // below.
1049 for (auto &Module : Modules) {
1050 auto ModuleIdentifier = Module->getName();
1051 ExportLists[ModuleIdentifier];
1052 ImportLists[ModuleIdentifier];
1053 ResolvedODR[ModuleIdentifier];
1054 ModuleToDefinedGVSummaries[ModuleIdentifier];
1055 }
1056
1057 std::vector<BitcodeModule *> ModulesVec;
1058 ModulesVec.reserve(Modules.size());
1059 for (auto &Mod : Modules)
1060 ModulesVec.push_back(&Mod->getSingleBitcodeModule());
1061 std::vector<int> ModulesOrdering = lto::generateModulesOrdering(ModulesVec);
1062
1063 // Parallel optimizer + codegen
1064 {
1065 ThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1066 for (auto IndexCount : ModulesOrdering) {
1067 auto &Mod = Modules[IndexCount];
1068 Pool.async([&](int count) {
1069 auto ModuleIdentifier = Mod->getName();
1070 auto &ExportList = ExportLists[ModuleIdentifier];
1071
1072 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1073
1074 // The module may be cached, this helps handling it.
1075 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1076 ImportLists[ModuleIdentifier], ExportList,
1077 ResolvedODR[ModuleIdentifier],
1078 DefinedGVSummaries, OptLevel, Freestanding,
1079 TMBuilder);
1080 auto CacheEntryPath = CacheEntry.getEntryPath();
1081
1082 {
1083 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1084 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1085 << " '" << CacheEntryPath << "' for buffer "
1086 << count << " " << ModuleIdentifier << "\n");
1087
1088 if (ErrOrBuffer) {
1089 // Cache Hit!
1090 if (SavedObjectsDirectoryPath.empty())
1091 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1092 else
1093 ProducedBinaryFiles[count] = writeGeneratedObject(
1094 count, CacheEntryPath, *ErrOrBuffer.get());
1095 return;
1096 }
1097 }
1098
1099 LLVMContext Context;
1100 Context.setDiscardValueNames(LTODiscardValueNames);
1101 Context.enableDebugTypeODRUniquing();
1102 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1103 Context, RemarksFilename, RemarksPasses, RemarksFormat,
1104 RemarksWithHotness, count);
1105 if (!DiagFileOrErr) {
1106 errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
1107 report_fatal_error("ThinLTO: Can't get an output file for the "
1108 "remarks");
1109 }
1110
1111 // Parse module now
1112 auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1113 /*IsImporting*/ false);
1114
1115 // Save temps: original file.
1116 saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
1117
1118 auto &ImportList = ImportLists[ModuleIdentifier];
1119 // Run the main process now, and generates a binary
1120 auto OutputBuffer = ProcessThinLTOModule(
1121 *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
1122 ExportList, GUIDPreservedSymbols,
1123 ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1124 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count);
1125
1126 // Commit to the cache (if enabled)
1127 CacheEntry.write(*OutputBuffer);
1128
1129 if (SavedObjectsDirectoryPath.empty()) {
1130 // We need to generated a memory buffer for the linker.
1131 if (!CacheEntryPath.empty()) {
1132 // When cache is enabled, reload from the cache if possible.
1133 // Releasing the buffer from the heap and reloading it from the
1134 // cache file with mmap helps us to lower memory pressure.
1135 // The freed memory can be used for the next input file.
1136 // The final binary link will read from the VFS cache (hopefully!)
1137 // or from disk (if the memory pressure was too high).
1138 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1139 if (auto EC = ReloadedBufferOrErr.getError()) {
1140 // On error, keep the preexisting buffer and print a diagnostic.
1141 errs() << "remark: can't reload cached file '" << CacheEntryPath
1142 << "': " << EC.message() << "\n";
1143 } else {
1144 OutputBuffer = std::move(*ReloadedBufferOrErr);
1145 }
1146 }
1147 ProducedBinaries[count] = std::move(OutputBuffer);
1148 return;
1149 }
1150 ProducedBinaryFiles[count] = writeGeneratedObject(
1151 count, CacheEntryPath, *OutputBuffer);
1152 }, IndexCount);
1153 }
1154 }
1155
1156 pruneCache(CacheOptions.Path, CacheOptions.Policy);
1157
1158 // If statistics were requested, print them out now.
1159 if (llvm::AreStatisticsEnabled())
1160 llvm::PrintStatistics();
1161 reportAndResetTimings();
1162 }
1163