1 //===--- CrossTranslationUnit.cpp - -----------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 //  This file implements the CrossTranslationUnit interface.
10 //
11 //===----------------------------------------------------------------------===//
12 #include "clang/CrossTU/CrossTranslationUnit.h"
13 #include "clang/AST/ASTImporter.h"
14 #include "clang/AST/Decl.h"
15 #include "clang/AST/ParentMapContext.h"
16 #include "clang/Basic/TargetInfo.h"
17 #include "clang/CrossTU/CrossTUDiagnostic.h"
18 #include "clang/Frontend/ASTUnit.h"
19 #include "clang/Frontend/CompilerInstance.h"
20 #include "clang/Frontend/TextDiagnosticPrinter.h"
21 #include "clang/Index/USRGeneration.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/ADT/Triple.h"
24 #include "llvm/Option/ArgList.h"
25 #include "llvm/Support/ErrorHandling.h"
26 #include "llvm/Support/ManagedStatic.h"
27 #include "llvm/Support/Path.h"
28 #include "llvm/Support/YAMLParser.h"
29 #include "llvm/Support/raw_ostream.h"
30 #include <algorithm>
31 #include <fstream>
32 #include <optional>
33 #include <sstream>
34 #include <tuple>
35 
36 namespace clang {
37 namespace cross_tu {
38 
39 namespace {
40 
41 #define DEBUG_TYPE "CrossTranslationUnit"
42 STATISTIC(NumGetCTUCalled, "The # of getCTUDefinition function called");
43 STATISTIC(
44     NumNotInOtherTU,
45     "The # of getCTUDefinition called but the function is not in any other TU");
46 STATISTIC(NumGetCTUSuccess,
47           "The # of getCTUDefinition successfully returned the "
48           "requested function's body");
49 STATISTIC(NumUnsupportedNodeFound, "The # of imports when the ASTImporter "
50                                    "encountered an unsupported AST Node");
51 STATISTIC(NumNameConflicts, "The # of imports when the ASTImporter "
52                             "encountered an ODR error");
53 STATISTIC(NumTripleMismatch, "The # of triple mismatches");
54 STATISTIC(NumLangMismatch, "The # of language mismatches");
55 STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches");
56 STATISTIC(NumASTLoadThresholdReached,
57           "The # of ASTs not loaded because of threshold");
58 
59 // Same as Triple's equality operator, but we check a field only if that is
60 // known in both instances.
61 bool hasEqualKnownFields(const llvm::Triple &Lhs, const llvm::Triple &Rhs) {
62   using llvm::Triple;
63   if (Lhs.getArch() != Triple::UnknownArch &&
64       Rhs.getArch() != Triple::UnknownArch && Lhs.getArch() != Rhs.getArch())
65     return false;
66   if (Lhs.getSubArch() != Triple::NoSubArch &&
67       Rhs.getSubArch() != Triple::NoSubArch &&
68       Lhs.getSubArch() != Rhs.getSubArch())
69     return false;
70   if (Lhs.getVendor() != Triple::UnknownVendor &&
71       Rhs.getVendor() != Triple::UnknownVendor &&
72       Lhs.getVendor() != Rhs.getVendor())
73     return false;
74   if (!Lhs.isOSUnknown() && !Rhs.isOSUnknown() &&
75       Lhs.getOS() != Rhs.getOS())
76     return false;
77   if (Lhs.getEnvironment() != Triple::UnknownEnvironment &&
78       Rhs.getEnvironment() != Triple::UnknownEnvironment &&
79       Lhs.getEnvironment() != Rhs.getEnvironment())
80     return false;
81   if (Lhs.getObjectFormat() != Triple::UnknownObjectFormat &&
82       Rhs.getObjectFormat() != Triple::UnknownObjectFormat &&
83       Lhs.getObjectFormat() != Rhs.getObjectFormat())
84     return false;
85   return true;
86 }
87 
88 // FIXME: This class is will be removed after the transition to llvm::Error.
89 class IndexErrorCategory : public std::error_category {
90 public:
91   const char *name() const noexcept override { return "clang.index"; }
92 
93   std::string message(int Condition) const override {
94     switch (static_cast<index_error_code>(Condition)) {
95     case index_error_code::success:
96       // There should not be a success error. Jump to unreachable directly.
97       // Add this case to make the compiler stop complaining.
98       break;
99     case index_error_code::unspecified:
100       return "An unknown error has occurred.";
101     case index_error_code::missing_index_file:
102       return "The index file is missing.";
103     case index_error_code::invalid_index_format:
104       return "Invalid index file format.";
105     case index_error_code::multiple_definitions:
106       return "Multiple definitions in the index file.";
107     case index_error_code::missing_definition:
108       return "Missing definition from the index file.";
109     case index_error_code::failed_import:
110       return "Failed to import the definition.";
111     case index_error_code::failed_to_get_external_ast:
112       return "Failed to load external AST source.";
113     case index_error_code::failed_to_generate_usr:
114       return "Failed to generate USR.";
115     case index_error_code::triple_mismatch:
116       return "Triple mismatch";
117     case index_error_code::lang_mismatch:
118       return "Language mismatch";
119     case index_error_code::lang_dialect_mismatch:
120       return "Language dialect mismatch";
121     case index_error_code::load_threshold_reached:
122       return "Load threshold reached";
123     case index_error_code::invocation_list_ambiguous:
124       return "Invocation list file contains multiple references to the same "
125              "source file.";
126     case index_error_code::invocation_list_file_not_found:
127       return "Invocation list file is not found.";
128     case index_error_code::invocation_list_empty:
129       return "Invocation list file is empty.";
130     case index_error_code::invocation_list_wrong_format:
131       return "Invocation list file is in wrong format.";
132     case index_error_code::invocation_list_lookup_unsuccessful:
133       return "Invocation list file does not contain the requested source file.";
134     }
135     llvm_unreachable("Unrecognized index_error_code.");
136   }
137 };
138 
139 static llvm::ManagedStatic<IndexErrorCategory> Category;
140 } // end anonymous namespace
141 
142 char IndexError::ID;
143 
144 void IndexError::log(raw_ostream &OS) const {
145   OS << Category->message(static_cast<int>(Code)) << '\n';
146 }
147 
148 std::error_code IndexError::convertToErrorCode() const {
149   return std::error_code(static_cast<int>(Code), *Category);
150 }
151 
152 /// Parse one line of the input CTU index file.
153 ///
154 /// @param[in]  LineRef     The input CTU index item in format
155 ///                         "<USR-Length>:<USR> <File-Path>".
156 /// @param[out] LookupName  The lookup name in format "<USR-Length>:<USR>".
157 /// @param[out] FilePath    The file path "<File-Path>".
158 static bool parseCrossTUIndexItem(StringRef LineRef, StringRef &LookupName,
159                                   StringRef &FilePath) {
160   // `LineRef` is "<USR-Length>:<USR> <File-Path>" now.
161 
162   size_t USRLength = 0;
163   if (LineRef.consumeInteger(10, USRLength))
164     return false;
165   assert(USRLength && "USRLength should be greater than zero.");
166 
167   if (!LineRef.consume_front(":"))
168     return false;
169 
170   // `LineRef` is now just "<USR> <File-Path>".
171 
172   // Check LookupName length out of bound and incorrect delimiter.
173   if (USRLength >= LineRef.size() || ' ' != LineRef[USRLength])
174     return false;
175 
176   LookupName = LineRef.substr(0, USRLength);
177   FilePath = LineRef.substr(USRLength + 1);
178   return true;
179 }
180 
181 llvm::Expected<llvm::StringMap<std::string>>
182 parseCrossTUIndex(StringRef IndexPath) {
183   std::ifstream ExternalMapFile{std::string(IndexPath)};
184   if (!ExternalMapFile)
185     return llvm::make_error<IndexError>(index_error_code::missing_index_file,
186                                         IndexPath.str());
187 
188   llvm::StringMap<std::string> Result;
189   std::string Line;
190   unsigned LineNo = 1;
191   while (std::getline(ExternalMapFile, Line)) {
192     // Split lookup name and file path
193     StringRef LookupName, FilePathInIndex;
194     if (!parseCrossTUIndexItem(Line, LookupName, FilePathInIndex))
195       return llvm::make_error<IndexError>(
196           index_error_code::invalid_index_format, IndexPath.str(), LineNo);
197 
198     // Store paths with posix-style directory separator.
199     SmallString<32> FilePath(FilePathInIndex);
200     llvm::sys::path::native(FilePath, llvm::sys::path::Style::posix);
201 
202     bool InsertionOccured;
203     std::tie(std::ignore, InsertionOccured) =
204         Result.try_emplace(LookupName, FilePath.begin(), FilePath.end());
205     if (!InsertionOccured)
206       return llvm::make_error<IndexError>(
207           index_error_code::multiple_definitions, IndexPath.str(), LineNo);
208 
209     ++LineNo;
210   }
211   return Result;
212 }
213 
214 std::string
215 createCrossTUIndexString(const llvm::StringMap<std::string> &Index) {
216   std::ostringstream Result;
217   for (const auto &E : Index)
218     Result << E.getKey().size() << ':' << E.getKey().str() << ' '
219            << E.getValue() << '\n';
220   return Result.str();
221 }
222 
223 bool shouldImport(const VarDecl *VD, const ASTContext &ACtx) {
224   CanQualType CT = ACtx.getCanonicalType(VD->getType());
225   return CT.isConstQualified() && VD->getType().isTrivialType(ACtx);
226 }
227 
228 static bool hasBodyOrInit(const FunctionDecl *D, const FunctionDecl *&DefD) {
229   return D->hasBody(DefD);
230 }
231 static bool hasBodyOrInit(const VarDecl *D, const VarDecl *&DefD) {
232   return D->getAnyInitializer(DefD);
233 }
234 template <typename T> static bool hasBodyOrInit(const T *D) {
235   const T *Unused;
236   return hasBodyOrInit(D, Unused);
237 }
238 
239 CrossTranslationUnitContext::CrossTranslationUnitContext(CompilerInstance &CI)
240     : Context(CI.getASTContext()), ASTStorage(CI) {}
241 
242 CrossTranslationUnitContext::~CrossTranslationUnitContext() {}
243 
244 std::optional<std::string>
245 CrossTranslationUnitContext::getLookupName(const NamedDecl *ND) {
246   SmallString<128> DeclUSR;
247   bool Ret = index::generateUSRForDecl(ND, DeclUSR);
248   if (Ret)
249     return {};
250   return std::string(DeclUSR.str());
251 }
252 
253 /// Recursively visits the decls of a DeclContext, and returns one with the
254 /// given USR.
255 template <typename T>
256 const T *
257 CrossTranslationUnitContext::findDefInDeclContext(const DeclContext *DC,
258                                                   StringRef LookupName) {
259   assert(DC && "Declaration Context must not be null");
260   for (const Decl *D : DC->decls()) {
261     const auto *SubDC = dyn_cast<DeclContext>(D);
262     if (SubDC)
263       if (const auto *ND = findDefInDeclContext<T>(SubDC, LookupName))
264         return ND;
265 
266     const auto *ND = dyn_cast<T>(D);
267     const T *ResultDecl;
268     if (!ND || !hasBodyOrInit(ND, ResultDecl))
269       continue;
270     std::optional<std::string> ResultLookupName = getLookupName(ResultDecl);
271     if (!ResultLookupName || *ResultLookupName != LookupName)
272       continue;
273     return ResultDecl;
274   }
275   return nullptr;
276 }
277 
278 template <typename T>
279 llvm::Expected<const T *> CrossTranslationUnitContext::getCrossTUDefinitionImpl(
280     const T *D, StringRef CrossTUDir, StringRef IndexName,
281     bool DisplayCTUProgress) {
282   assert(D && "D is missing, bad call to this function!");
283   assert(!hasBodyOrInit(D) &&
284          "D has a body or init in current translation unit!");
285   ++NumGetCTUCalled;
286   const std::optional<std::string> LookupName = getLookupName(D);
287   if (!LookupName)
288     return llvm::make_error<IndexError>(
289         index_error_code::failed_to_generate_usr);
290   llvm::Expected<ASTUnit *> ASTUnitOrError =
291       loadExternalAST(*LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
292   if (!ASTUnitOrError)
293     return ASTUnitOrError.takeError();
294   ASTUnit *Unit = *ASTUnitOrError;
295   assert(&Unit->getFileManager() ==
296          &Unit->getASTContext().getSourceManager().getFileManager());
297 
298   const llvm::Triple &TripleTo = Context.getTargetInfo().getTriple();
299   const llvm::Triple &TripleFrom =
300       Unit->getASTContext().getTargetInfo().getTriple();
301   // The imported AST had been generated for a different target.
302   // Some parts of the triple in the loaded ASTContext can be unknown while the
303   // very same parts in the target ASTContext are known. Thus we check for the
304   // known parts only.
305   if (!hasEqualKnownFields(TripleTo, TripleFrom)) {
306     // TODO: Pass the SourceLocation of the CallExpression for more precise
307     // diagnostics.
308     ++NumTripleMismatch;
309     return llvm::make_error<IndexError>(index_error_code::triple_mismatch,
310                                         std::string(Unit->getMainFileName()),
311                                         TripleTo.str(), TripleFrom.str());
312   }
313 
314   const auto &LangTo = Context.getLangOpts();
315   const auto &LangFrom = Unit->getASTContext().getLangOpts();
316 
317   // FIXME: Currenty we do not support CTU across C++ and C and across
318   // different dialects of C++.
319   if (LangTo.CPlusPlus != LangFrom.CPlusPlus) {
320     ++NumLangMismatch;
321     return llvm::make_error<IndexError>(index_error_code::lang_mismatch);
322   }
323 
324   // If CPP dialects are different then return with error.
325   //
326   // Consider this STL code:
327   //   template<typename _Alloc>
328   //     struct __alloc_traits
329   //   #if __cplusplus >= 201103L
330   //     : std::allocator_traits<_Alloc>
331   //   #endif
332   //     { // ...
333   //     };
334   // This class template would create ODR errors during merging the two units,
335   // since in one translation unit the class template has a base class, however
336   // in the other unit it has none.
337   if (LangTo.CPlusPlus11 != LangFrom.CPlusPlus11 ||
338       LangTo.CPlusPlus14 != LangFrom.CPlusPlus14 ||
339       LangTo.CPlusPlus17 != LangFrom.CPlusPlus17 ||
340       LangTo.CPlusPlus20 != LangFrom.CPlusPlus20) {
341     ++NumLangDialectMismatch;
342     return llvm::make_error<IndexError>(
343         index_error_code::lang_dialect_mismatch);
344   }
345 
346   TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl();
347   if (const T *ResultDecl = findDefInDeclContext<T>(TU, *LookupName))
348     return importDefinition(ResultDecl, Unit);
349   return llvm::make_error<IndexError>(index_error_code::failed_import);
350 }
351 
352 llvm::Expected<const FunctionDecl *>
353 CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
354                                                   StringRef CrossTUDir,
355                                                   StringRef IndexName,
356                                                   bool DisplayCTUProgress) {
357   return getCrossTUDefinitionImpl(FD, CrossTUDir, IndexName,
358                                   DisplayCTUProgress);
359 }
360 
361 llvm::Expected<const VarDecl *>
362 CrossTranslationUnitContext::getCrossTUDefinition(const VarDecl *VD,
363                                                   StringRef CrossTUDir,
364                                                   StringRef IndexName,
365                                                   bool DisplayCTUProgress) {
366   return getCrossTUDefinitionImpl(VD, CrossTUDir, IndexName,
367                                   DisplayCTUProgress);
368 }
369 
370 void CrossTranslationUnitContext::emitCrossTUDiagnostics(const IndexError &IE) {
371   switch (IE.getCode()) {
372   case index_error_code::missing_index_file:
373     Context.getDiagnostics().Report(diag::err_ctu_error_opening)
374         << IE.getFileName();
375     break;
376   case index_error_code::invalid_index_format:
377     Context.getDiagnostics().Report(diag::err_extdefmap_parsing)
378         << IE.getFileName() << IE.getLineNum();
379     break;
380   case index_error_code::multiple_definitions:
381     Context.getDiagnostics().Report(diag::err_multiple_def_index)
382         << IE.getLineNum();
383     break;
384   case index_error_code::triple_mismatch:
385     Context.getDiagnostics().Report(diag::warn_ctu_incompat_triple)
386         << IE.getFileName() << IE.getTripleToName() << IE.getTripleFromName();
387     break;
388   default:
389     break;
390   }
391 }
392 
393 CrossTranslationUnitContext::ASTUnitStorage::ASTUnitStorage(
394     CompilerInstance &CI)
395     : Loader(CI, CI.getAnalyzerOpts()->CTUDir,
396              CI.getAnalyzerOpts()->CTUInvocationList),
397       LoadGuard(CI.getASTContext().getLangOpts().CPlusPlus
398                     ? CI.getAnalyzerOpts()->CTUImportCppThreshold
399                     : CI.getAnalyzerOpts()->CTUImportThreshold) {}
400 
401 llvm::Expected<ASTUnit *>
402 CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFile(
403     StringRef FileName, bool DisplayCTUProgress) {
404   // Try the cache first.
405   auto ASTCacheEntry = FileASTUnitMap.find(FileName);
406   if (ASTCacheEntry == FileASTUnitMap.end()) {
407 
408     // Do not load if the limit is reached.
409     if (!LoadGuard) {
410       ++NumASTLoadThresholdReached;
411       return llvm::make_error<IndexError>(
412           index_error_code::load_threshold_reached);
413     }
414 
415     auto LoadAttempt = Loader.load(FileName);
416 
417     if (!LoadAttempt)
418       return LoadAttempt.takeError();
419 
420     std::unique_ptr<ASTUnit> LoadedUnit = std::move(LoadAttempt.get());
421 
422     // Need the raw pointer and the unique_ptr as well.
423     ASTUnit *Unit = LoadedUnit.get();
424 
425     // Update the cache.
426     FileASTUnitMap[FileName] = std::move(LoadedUnit);
427 
428     LoadGuard.indicateLoadSuccess();
429 
430     if (DisplayCTUProgress)
431       llvm::errs() << "CTU loaded AST file: " << FileName << "\n";
432 
433     return Unit;
434 
435   } else {
436     // Found in the cache.
437     return ASTCacheEntry->second.get();
438   }
439 }
440 
441 llvm::Expected<ASTUnit *>
442 CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFunction(
443     StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName,
444     bool DisplayCTUProgress) {
445   // Try the cache first.
446   auto ASTCacheEntry = NameASTUnitMap.find(FunctionName);
447   if (ASTCacheEntry == NameASTUnitMap.end()) {
448     // Load the ASTUnit from the pre-dumped AST file specified by ASTFileName.
449 
450     // Ensure that the Index is loaded, as we need to search in it.
451     if (llvm::Error IndexLoadError =
452             ensureCTUIndexLoaded(CrossTUDir, IndexName))
453       return std::move(IndexLoadError);
454 
455     // Check if there is an entry in the index for the function.
456     if (!NameFileMap.count(FunctionName)) {
457       ++NumNotInOtherTU;
458       return llvm::make_error<IndexError>(index_error_code::missing_definition);
459     }
460 
461     // Search in the index for the filename where the definition of FunctionName
462     // resides.
463     if (llvm::Expected<ASTUnit *> FoundForFile =
464             getASTUnitForFile(NameFileMap[FunctionName], DisplayCTUProgress)) {
465 
466       // Update the cache.
467       NameASTUnitMap[FunctionName] = *FoundForFile;
468       return *FoundForFile;
469 
470     } else {
471       return FoundForFile.takeError();
472     }
473   } else {
474     // Found in the cache.
475     return ASTCacheEntry->second;
476   }
477 }
478 
479 llvm::Expected<std::string>
480 CrossTranslationUnitContext::ASTUnitStorage::getFileForFunction(
481     StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName) {
482   if (llvm::Error IndexLoadError = ensureCTUIndexLoaded(CrossTUDir, IndexName))
483     return std::move(IndexLoadError);
484   return NameFileMap[FunctionName];
485 }
486 
487 llvm::Error CrossTranslationUnitContext::ASTUnitStorage::ensureCTUIndexLoaded(
488     StringRef CrossTUDir, StringRef IndexName) {
489   // Dont initialize if the map is filled.
490   if (!NameFileMap.empty())
491     return llvm::Error::success();
492 
493   // Get the absolute path to the index file.
494   SmallString<256> IndexFile = CrossTUDir;
495   if (llvm::sys::path::is_absolute(IndexName))
496     IndexFile = IndexName;
497   else
498     llvm::sys::path::append(IndexFile, IndexName);
499 
500   if (auto IndexMapping = parseCrossTUIndex(IndexFile)) {
501     // Initialize member map.
502     NameFileMap = *IndexMapping;
503     return llvm::Error::success();
504   } else {
505     // Error while parsing CrossTU index file.
506     return IndexMapping.takeError();
507   };
508 }
509 
510 llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
511     StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
512     bool DisplayCTUProgress) {
513   // FIXME: The current implementation only supports loading decls with
514   //        a lookup name from a single translation unit. If multiple
515   //        translation units contains decls with the same lookup name an
516   //        error will be returned.
517 
518   // Try to get the value from the heavily cached storage.
519   llvm::Expected<ASTUnit *> Unit = ASTStorage.getASTUnitForFunction(
520       LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
521 
522   if (!Unit)
523     return Unit.takeError();
524 
525   // Check whether the backing pointer of the Expected is a nullptr.
526   if (!*Unit)
527     return llvm::make_error<IndexError>(
528         index_error_code::failed_to_get_external_ast);
529 
530   return Unit;
531 }
532 
533 CrossTranslationUnitContext::ASTLoader::ASTLoader(
534     CompilerInstance &CI, StringRef CTUDir, StringRef InvocationListFilePath)
535     : CI(CI), CTUDir(CTUDir), InvocationListFilePath(InvocationListFilePath) {}
536 
537 CrossTranslationUnitContext::LoadResultTy
538 CrossTranslationUnitContext::ASTLoader::load(StringRef Identifier) {
539   llvm::SmallString<256> Path;
540   if (llvm::sys::path::is_absolute(Identifier, PathStyle)) {
541     Path = Identifier;
542   } else {
543     Path = CTUDir;
544     llvm::sys::path::append(Path, PathStyle, Identifier);
545   }
546 
547   // The path is stored in the InvocationList member in posix style. To
548   // successfully lookup an entry based on filepath, it must be converted.
549   llvm::sys::path::native(Path, PathStyle);
550 
551   // Normalize by removing relative path components.
552   llvm::sys::path::remove_dots(Path, /*remove_dot_dot*/ true, PathStyle);
553 
554   if (Path.endswith(".ast"))
555     return loadFromDump(Path);
556   else
557     return loadFromSource(Path);
558 }
559 
560 CrossTranslationUnitContext::LoadResultTy
561 CrossTranslationUnitContext::ASTLoader::loadFromDump(StringRef ASTDumpPath) {
562   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
563   TextDiagnosticPrinter *DiagClient =
564       new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
565   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
566   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
567       new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient));
568   return ASTUnit::LoadFromASTFile(
569       std::string(ASTDumpPath.str()),
570       CI.getPCHContainerOperations()->getRawReader(), ASTUnit::LoadEverything,
571       Diags, CI.getFileSystemOpts());
572 }
573 
574 /// Load the AST from a source-file, which is supposed to be located inside the
575 /// YAML formatted invocation list file under the filesystem path specified by
576 /// \p InvocationList. The invocation list should contain absolute paths.
577 /// \p SourceFilePath is the absolute path of the source file that contains the
578 /// function definition the analysis is looking for. The Index is built by the
579 /// \p clang-extdef-mapping tool, which is also supposed to be generating
580 /// absolute paths.
581 ///
582 /// Proper diagnostic emission requires absolute paths, so even if a future
583 /// change introduces the handling of relative paths, this must be taken into
584 /// consideration.
585 CrossTranslationUnitContext::LoadResultTy
586 CrossTranslationUnitContext::ASTLoader::loadFromSource(
587     StringRef SourceFilePath) {
588 
589   if (llvm::Error InitError = lazyInitInvocationList())
590     return std::move(InitError);
591   assert(InvocationList);
592 
593   auto Invocation = InvocationList->find(SourceFilePath);
594   if (Invocation == InvocationList->end())
595     return llvm::make_error<IndexError>(
596         index_error_code::invocation_list_lookup_unsuccessful);
597 
598   const InvocationListTy::mapped_type &InvocationCommand = Invocation->second;
599 
600   SmallVector<const char *, 32> CommandLineArgs(InvocationCommand.size());
601   std::transform(InvocationCommand.begin(), InvocationCommand.end(),
602                  CommandLineArgs.begin(),
603                  [](auto &&CmdPart) { return CmdPart.c_str(); });
604 
605   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts{&CI.getDiagnosticOpts()};
606   auto *DiagClient = new ForwardingDiagnosticConsumer{CI.getDiagnosticClient()};
607   IntrusiveRefCntPtr<DiagnosticIDs> DiagID{
608       CI.getDiagnostics().getDiagnosticIDs()};
609   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
610       new DiagnosticsEngine{DiagID, &*DiagOpts, DiagClient});
611 
612   return std::unique_ptr<ASTUnit>(ASTUnit::LoadFromCommandLine(
613       CommandLineArgs.begin(), (CommandLineArgs.end()),
614       CI.getPCHContainerOperations(), Diags,
615       CI.getHeaderSearchOpts().ResourceDir));
616 }
617 
618 llvm::Expected<InvocationListTy>
619 parseInvocationList(StringRef FileContent, llvm::sys::path::Style PathStyle) {
620   InvocationListTy InvocationList;
621 
622   /// LLVM YAML parser is used to extract information from invocation list file.
623   llvm::SourceMgr SM;
624   llvm::yaml::Stream InvocationFile(FileContent, SM);
625 
626   /// Only the first document is processed.
627   llvm::yaml::document_iterator FirstInvocationFile = InvocationFile.begin();
628 
629   /// There has to be at least one document available.
630   if (FirstInvocationFile == InvocationFile.end())
631     return llvm::make_error<IndexError>(
632         index_error_code::invocation_list_empty);
633 
634   llvm::yaml::Node *DocumentRoot = FirstInvocationFile->getRoot();
635   if (!DocumentRoot)
636     return llvm::make_error<IndexError>(
637         index_error_code::invocation_list_wrong_format);
638 
639   /// According to the format specified the document must be a mapping, where
640   /// the keys are paths to source files, and values are sequences of invocation
641   /// parts.
642   auto *Mappings = dyn_cast<llvm::yaml::MappingNode>(DocumentRoot);
643   if (!Mappings)
644     return llvm::make_error<IndexError>(
645         index_error_code::invocation_list_wrong_format);
646 
647   for (auto &NextMapping : *Mappings) {
648     /// The keys should be strings, which represent a source-file path.
649     auto *Key = dyn_cast<llvm::yaml::ScalarNode>(NextMapping.getKey());
650     if (!Key)
651       return llvm::make_error<IndexError>(
652           index_error_code::invocation_list_wrong_format);
653 
654     SmallString<32> ValueStorage;
655     StringRef SourcePath = Key->getValue(ValueStorage);
656 
657     // Store paths with PathStyle directory separator.
658     SmallString<32> NativeSourcePath(SourcePath);
659     llvm::sys::path::native(NativeSourcePath, PathStyle);
660 
661     StringRef InvocationKey = NativeSourcePath;
662 
663     if (InvocationList.find(InvocationKey) != InvocationList.end())
664       return llvm::make_error<IndexError>(
665           index_error_code::invocation_list_ambiguous);
666 
667     /// The values should be sequences of strings, each representing a part of
668     /// the invocation.
669     auto *Args = dyn_cast<llvm::yaml::SequenceNode>(NextMapping.getValue());
670     if (!Args)
671       return llvm::make_error<IndexError>(
672           index_error_code::invocation_list_wrong_format);
673 
674     for (auto &Arg : *Args) {
675       auto *CmdString = dyn_cast<llvm::yaml::ScalarNode>(&Arg);
676       if (!CmdString)
677         return llvm::make_error<IndexError>(
678             index_error_code::invocation_list_wrong_format);
679       /// Every conversion starts with an empty working storage, as it is not
680       /// clear if this is a requirement of the YAML parser.
681       ValueStorage.clear();
682       InvocationList[InvocationKey].emplace_back(
683           CmdString->getValue(ValueStorage));
684     }
685 
686     if (InvocationList[InvocationKey].empty())
687       return llvm::make_error<IndexError>(
688           index_error_code::invocation_list_wrong_format);
689   }
690 
691   return InvocationList;
692 }
693 
694 llvm::Error CrossTranslationUnitContext::ASTLoader::lazyInitInvocationList() {
695   /// Lazily initialize the invocation list member used for on-demand parsing.
696   if (InvocationList)
697     return llvm::Error::success();
698   if (index_error_code::success != PreviousParsingResult)
699     return llvm::make_error<IndexError>(PreviousParsingResult);
700 
701   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileContent =
702       llvm::MemoryBuffer::getFile(InvocationListFilePath);
703   if (!FileContent) {
704     PreviousParsingResult = index_error_code::invocation_list_file_not_found;
705     return llvm::make_error<IndexError>(PreviousParsingResult);
706   }
707   std::unique_ptr<llvm::MemoryBuffer> ContentBuffer = std::move(*FileContent);
708   assert(ContentBuffer && "If no error was produced after loading, the pointer "
709                           "should not be nullptr.");
710 
711   llvm::Expected<InvocationListTy> ExpectedInvocationList =
712       parseInvocationList(ContentBuffer->getBuffer(), PathStyle);
713 
714   // Handle the error to store the code for next call to this function.
715   if (!ExpectedInvocationList) {
716     llvm::handleAllErrors(
717         ExpectedInvocationList.takeError(),
718         [&](const IndexError &E) { PreviousParsingResult = E.getCode(); });
719     return llvm::make_error<IndexError>(PreviousParsingResult);
720   }
721 
722   InvocationList = *ExpectedInvocationList;
723 
724   return llvm::Error::success();
725 }
726 
727 template <typename T>
728 llvm::Expected<const T *>
729 CrossTranslationUnitContext::importDefinitionImpl(const T *D, ASTUnit *Unit) {
730   assert(hasBodyOrInit(D) && "Decls to be imported should have body or init.");
731 
732   assert(&D->getASTContext() == &Unit->getASTContext() &&
733          "ASTContext of Decl and the unit should match.");
734   ASTImporter &Importer = getOrCreateASTImporter(Unit);
735 
736   auto ToDeclOrError = Importer.Import(D);
737   if (!ToDeclOrError) {
738     handleAllErrors(ToDeclOrError.takeError(), [&](const ASTImportError &IE) {
739       switch (IE.Error) {
740       case ASTImportError::NameConflict:
741         ++NumNameConflicts;
742         break;
743       case ASTImportError::UnsupportedConstruct:
744         ++NumUnsupportedNodeFound;
745         break;
746       case ASTImportError::Unknown:
747         llvm_unreachable("Unknown import error happened.");
748         break;
749       }
750     });
751     return llvm::make_error<IndexError>(index_error_code::failed_import);
752   }
753   auto *ToDecl = cast<T>(*ToDeclOrError);
754   assert(hasBodyOrInit(ToDecl) && "Imported Decl should have body or init.");
755   ++NumGetCTUSuccess;
756 
757   // Parent map is invalidated after changing the AST.
758   ToDecl->getASTContext().getParentMapContext().clear();
759 
760   return ToDecl;
761 }
762 
763 llvm::Expected<const FunctionDecl *>
764 CrossTranslationUnitContext::importDefinition(const FunctionDecl *FD,
765                                               ASTUnit *Unit) {
766   return importDefinitionImpl(FD, Unit);
767 }
768 
769 llvm::Expected<const VarDecl *>
770 CrossTranslationUnitContext::importDefinition(const VarDecl *VD,
771                                               ASTUnit *Unit) {
772   return importDefinitionImpl(VD, Unit);
773 }
774 
775 void CrossTranslationUnitContext::lazyInitImporterSharedSt(
776     TranslationUnitDecl *ToTU) {
777   if (!ImporterSharedSt)
778     ImporterSharedSt = std::make_shared<ASTImporterSharedState>(*ToTU);
779 }
780 
781 ASTImporter &
782 CrossTranslationUnitContext::getOrCreateASTImporter(ASTUnit *Unit) {
783   ASTContext &From = Unit->getASTContext();
784 
785   auto I = ASTUnitImporterMap.find(From.getTranslationUnitDecl());
786   if (I != ASTUnitImporterMap.end())
787     return *I->second;
788   lazyInitImporterSharedSt(Context.getTranslationUnitDecl());
789   ASTImporter *NewImporter = new ASTImporter(
790       Context, Context.getSourceManager().getFileManager(), From,
791       From.getSourceManager().getFileManager(), false, ImporterSharedSt);
792   ASTUnitImporterMap[From.getTranslationUnitDecl()].reset(NewImporter);
793   return *NewImporter;
794 }
795 
796 std::optional<clang::MacroExpansionContext>
797 CrossTranslationUnitContext::getMacroExpansionContextForSourceLocation(
798     const clang::SourceLocation &ToLoc) const {
799   // FIXME: Implement: Record such a context for every imported ASTUnit; lookup.
800   return std::nullopt;
801 }
802 
803 bool CrossTranslationUnitContext::isImportedAsNew(const Decl *ToDecl) const {
804   if (!ImporterSharedSt)
805     return false;
806   return ImporterSharedSt->isNewDecl(const_cast<Decl *>(ToDecl));
807 }
808 
809 bool CrossTranslationUnitContext::hasError(const Decl *ToDecl) const {
810   if (!ImporterSharedSt)
811     return false;
812   return static_cast<bool>(
813       ImporterSharedSt->getImportDeclErrorIfAny(const_cast<Decl *>(ToDecl)));
814 }
815 
816 } // namespace cross_tu
817 } // namespace clang
818