1 //===- Indexing.cpp - Higher level API functions --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CIndexDiagnostic.h"
10 #include "CIndexer.h"
11 #include "CLog.h"
12 #include "CXCursor.h"
13 #include "CXIndexDataConsumer.h"
14 #include "CXSourceLocation.h"
15 #include "CXString.h"
16 #include "CXTranslationUnit.h"
17 #include "clang/AST/ASTConsumer.h"
18 #include "clang/Frontend/ASTUnit.h"
19 #include "clang/Frontend/CompilerInstance.h"
20 #include "clang/Frontend/CompilerInvocation.h"
21 #include "clang/Frontend/FrontendAction.h"
22 #include "clang/Frontend/MultiplexConsumer.h"
23 #include "clang/Frontend/Utils.h"
24 #include "clang/Index/IndexingAction.h"
25 #include "clang/Lex/HeaderSearch.h"
26 #include "clang/Lex/PPCallbacks.h"
27 #include "clang/Lex/PPConditionalDirectiveRecord.h"
28 #include "clang/Lex/Preprocessor.h"
29 #include "clang/Lex/PreprocessorOptions.h"
30 #include "llvm/Support/CrashRecoveryContext.h"
31 #include "llvm/Support/MemoryBuffer.h"
32 #include <cstdio>
33 #include <mutex>
34 #include <utility>
35 
36 using namespace clang;
37 using namespace clang::index;
38 using namespace cxtu;
39 using namespace cxindex;
40 
41 namespace {
42 
43 //===----------------------------------------------------------------------===//
44 // Skip Parsed Bodies
45 //===----------------------------------------------------------------------===//
46 
47 /// A "region" in source code identified by the file/offset of the
48 /// preprocessor conditional directive that it belongs to.
49 /// Multiple, non-consecutive ranges can be parts of the same region.
50 ///
51 /// As an example of different regions separated by preprocessor directives:
52 ///
53 /// \code
54 ///   #1
55 /// #ifdef BLAH
56 ///   #2
57 /// #ifdef CAKE
58 ///   #3
59 /// #endif
60 ///   #2
61 /// #endif
62 ///   #1
63 /// \endcode
64 ///
65 /// There are 3 regions, with non-consecutive parts:
66 ///   #1 is identified as the beginning of the file
67 ///   #2 is identified as the location of "#ifdef BLAH"
68 ///   #3 is identified as the location of "#ifdef CAKE"
69 ///
70 class PPRegion {
71   llvm::sys::fs::UniqueID UniqueID;
72   time_t ModTime;
73   unsigned Offset;
74 public:
75   PPRegion() : UniqueID(0, 0), ModTime(), Offset() {}
76   PPRegion(llvm::sys::fs::UniqueID UniqueID, unsigned offset, time_t modTime)
77       : UniqueID(UniqueID), ModTime(modTime), Offset(offset) {}
78 
79   const llvm::sys::fs::UniqueID &getUniqueID() const { return UniqueID; }
80   unsigned getOffset() const { return Offset; }
81   time_t getModTime() const { return ModTime; }
82 
83   bool isInvalid() const { return *this == PPRegion(); }
84 
85   friend bool operator==(const PPRegion &lhs, const PPRegion &rhs) {
86     return lhs.UniqueID == rhs.UniqueID && lhs.Offset == rhs.Offset &&
87            lhs.ModTime == rhs.ModTime;
88   }
89 };
90 
91 } // end anonymous namespace
92 
93 namespace llvm {
94 
95   template <>
96   struct DenseMapInfo<PPRegion> {
97     static inline PPRegion getEmptyKey() {
98       return PPRegion(llvm::sys::fs::UniqueID(0, 0), unsigned(-1), 0);
99     }
100     static inline PPRegion getTombstoneKey() {
101       return PPRegion(llvm::sys::fs::UniqueID(0, 0), unsigned(-2), 0);
102     }
103 
104     static unsigned getHashValue(const PPRegion &S) {
105       llvm::FoldingSetNodeID ID;
106       const llvm::sys::fs::UniqueID &UniqueID = S.getUniqueID();
107       ID.AddInteger(UniqueID.getFile());
108       ID.AddInteger(UniqueID.getDevice());
109       ID.AddInteger(S.getOffset());
110       ID.AddInteger(S.getModTime());
111       return ID.ComputeHash();
112     }
113 
114     static bool isEqual(const PPRegion &LHS, const PPRegion &RHS) {
115       return LHS == RHS;
116     }
117   };
118 }
119 
120 namespace {
121 
122 /// Keeps track of function bodies that have already been parsed.
123 ///
124 /// Is thread-safe.
125 class ThreadSafeParsedRegions {
126   mutable std::mutex Mutex;
127   llvm::DenseSet<PPRegion> ParsedRegions;
128 
129 public:
130   ~ThreadSafeParsedRegions() = default;
131 
132   llvm::DenseSet<PPRegion> getParsedRegions() const {
133     std::lock_guard<std::mutex> MG(Mutex);
134     return ParsedRegions;
135   }
136 
137   void addParsedRegions(ArrayRef<PPRegion> Regions) {
138     std::lock_guard<std::mutex> MG(Mutex);
139     ParsedRegions.insert(Regions.begin(), Regions.end());
140   }
141 };
142 
143 /// Provides information whether source locations have already been parsed in
144 /// another FrontendAction.
145 ///
146 /// Is NOT thread-safe.
147 class ParsedSrcLocationsTracker {
148   ThreadSafeParsedRegions &ParsedRegionsStorage;
149   PPConditionalDirectiveRecord &PPRec;
150   Preprocessor &PP;
151 
152   /// Snapshot of the shared state at the point when this instance was
153   /// constructed.
154   llvm::DenseSet<PPRegion> ParsedRegionsSnapshot;
155   /// Regions that were queried during this instance lifetime.
156   SmallVector<PPRegion, 32> NewParsedRegions;
157 
158   /// Caching the last queried region.
159   PPRegion LastRegion;
160   bool LastIsParsed;
161 
162 public:
163   /// Creates snapshot of \p ParsedRegionsStorage.
164   ParsedSrcLocationsTracker(ThreadSafeParsedRegions &ParsedRegionsStorage,
165                             PPConditionalDirectiveRecord &ppRec,
166                             Preprocessor &pp)
167       : ParsedRegionsStorage(ParsedRegionsStorage), PPRec(ppRec), PP(pp),
168         ParsedRegionsSnapshot(ParsedRegionsStorage.getParsedRegions()) {}
169 
170   /// \returns true iff \p Loc has already been parsed.
171   ///
172   /// Can provide false-negative in case the location was parsed after this
173   /// instance had been constructed.
174   bool hasAlredyBeenParsed(SourceLocation Loc, FileID FID,
175                            const FileEntry *FE) {
176     assert(FE);
177     PPRegion region = getRegion(Loc, FID, FE);
178     if (region.isInvalid())
179       return false;
180 
181     // Check common case, consecutive functions in the same region.
182     if (LastRegion == region)
183       return LastIsParsed;
184 
185     LastRegion = region;
186     // Source locations can't be revisited during single TU parsing.
187     // That means if we hit the same region again, it's a different location in
188     // the same region and so the "is parsed" value from the snapshot is still
189     // correct.
190     LastIsParsed = ParsedRegionsSnapshot.count(region);
191     if (!LastIsParsed)
192       NewParsedRegions.emplace_back(std::move(region));
193     return LastIsParsed;
194   }
195 
196   /// Updates ParsedRegionsStorage with newly parsed regions.
197   void syncWithStorage() {
198     ParsedRegionsStorage.addParsedRegions(NewParsedRegions);
199   }
200 
201 private:
202   PPRegion getRegion(SourceLocation Loc, FileID FID, const FileEntry *FE) {
203     assert(FE);
204     auto Bail = [this, FE]() {
205       if (isParsedOnceInclude(FE)) {
206         const llvm::sys::fs::UniqueID &ID = FE->getUniqueID();
207         return PPRegion(ID, 0, FE->getModificationTime());
208       }
209       return PPRegion();
210     };
211 
212     SourceLocation RegionLoc = PPRec.findConditionalDirectiveRegionLoc(Loc);
213     assert(RegionLoc.isFileID());
214     if (RegionLoc.isInvalid())
215       return Bail();
216 
217     FileID RegionFID;
218     unsigned RegionOffset;
219     std::tie(RegionFID, RegionOffset) =
220         PPRec.getSourceManager().getDecomposedLoc(RegionLoc);
221 
222     if (RegionFID != FID)
223       return Bail();
224 
225     const llvm::sys::fs::UniqueID &ID = FE->getUniqueID();
226     return PPRegion(ID, RegionOffset, FE->getModificationTime());
227   }
228 
229   bool isParsedOnceInclude(const FileEntry *FE) {
230     return PP.getHeaderSearchInfo().isFileMultipleIncludeGuarded(FE) ||
231            PP.getHeaderSearchInfo().hasFileBeenImported(FE);
232   }
233 };
234 
235 //===----------------------------------------------------------------------===//
236 // IndexPPCallbacks
237 //===----------------------------------------------------------------------===//
238 
239 class IndexPPCallbacks : public PPCallbacks {
240   Preprocessor &PP;
241   CXIndexDataConsumer &DataConsumer;
242   bool IsMainFileEntered;
243 
244 public:
245   IndexPPCallbacks(Preprocessor &PP, CXIndexDataConsumer &dataConsumer)
246     : PP(PP), DataConsumer(dataConsumer), IsMainFileEntered(false) { }
247 
248   void FileChanged(SourceLocation Loc, FileChangeReason Reason,
249                  SrcMgr::CharacteristicKind FileType, FileID PrevFID) override {
250     if (IsMainFileEntered)
251       return;
252 
253     SourceManager &SM = PP.getSourceManager();
254     SourceLocation MainFileLoc = SM.getLocForStartOfFile(SM.getMainFileID());
255 
256     if (Loc == MainFileLoc && Reason == PPCallbacks::EnterFile) {
257       IsMainFileEntered = true;
258       DataConsumer.enteredMainFile(SM.getFileEntryForID(SM.getMainFileID()));
259     }
260   }
261 
262   void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
263                           StringRef FileName, bool IsAngled,
264                           CharSourceRange FilenameRange, const FileEntry *File,
265                           StringRef SearchPath, StringRef RelativePath,
266                           const Module *Imported,
267                           SrcMgr::CharacteristicKind FileType) override {
268     bool isImport = (IncludeTok.is(tok::identifier) &&
269             IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import);
270     DataConsumer.ppIncludedFile(HashLoc, FileName, File, isImport, IsAngled,
271                             Imported);
272   }
273 
274   /// MacroDefined - This hook is called whenever a macro definition is seen.
275   void MacroDefined(const Token &Id, const MacroDirective *MD) override {}
276 
277   /// MacroUndefined - This hook is called whenever a macro #undef is seen.
278   /// MI is released immediately following this callback.
279   void MacroUndefined(const Token &MacroNameTok,
280                       const MacroDefinition &MD,
281                       const MacroDirective *UD) override {}
282 
283   /// MacroExpands - This is called by when a macro invocation is found.
284   void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD,
285                     SourceRange Range, const MacroArgs *Args) override {}
286 
287   /// SourceRangeSkipped - This hook is called when a source range is skipped.
288   /// \param Range The SourceRange that was skipped. The range begins at the
289   /// #if/#else directive and ends after the #endif/#else directive.
290   void SourceRangeSkipped(SourceRange Range, SourceLocation EndifLoc) override {
291   }
292 };
293 
294 //===----------------------------------------------------------------------===//
295 // IndexingConsumer
296 //===----------------------------------------------------------------------===//
297 
298 class IndexingConsumer : public ASTConsumer {
299   CXIndexDataConsumer &DataConsumer;
300 
301 public:
302   IndexingConsumer(CXIndexDataConsumer &dataConsumer,
303                    ParsedSrcLocationsTracker *parsedLocsTracker)
304       : DataConsumer(dataConsumer) {}
305 
306   void Initialize(ASTContext &Context) override {
307     DataConsumer.setASTContext(Context);
308     DataConsumer.startedTranslationUnit();
309   }
310 
311   bool HandleTopLevelDecl(DeclGroupRef DG) override {
312     return !DataConsumer.shouldAbort();
313   }
314 };
315 
316 //===----------------------------------------------------------------------===//
317 // CaptureDiagnosticConsumer
318 //===----------------------------------------------------------------------===//
319 
320 class CaptureDiagnosticConsumer : public DiagnosticConsumer {
321   SmallVector<StoredDiagnostic, 4> Errors;
322 public:
323 
324   void HandleDiagnostic(DiagnosticsEngine::Level level,
325                         const Diagnostic &Info) override {
326     if (level >= DiagnosticsEngine::Error)
327       Errors.push_back(StoredDiagnostic(level, Info));
328   }
329 };
330 
331 //===----------------------------------------------------------------------===//
332 // IndexingFrontendAction
333 //===----------------------------------------------------------------------===//
334 
335 class IndexingFrontendAction : public ASTFrontendAction {
336   std::shared_ptr<CXIndexDataConsumer> DataConsumer;
337   IndexingOptions Opts;
338 
339   ThreadSafeParsedRegions *SKData;
340   std::unique_ptr<ParsedSrcLocationsTracker> ParsedLocsTracker;
341 
342 public:
343   IndexingFrontendAction(std::shared_ptr<CXIndexDataConsumer> dataConsumer,
344                          const IndexingOptions &Opts,
345                          ThreadSafeParsedRegions *skData)
346       : DataConsumer(std::move(dataConsumer)), Opts(Opts), SKData(skData) {}
347 
348   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
349                                                  StringRef InFile) override {
350     PreprocessorOptions &PPOpts = CI.getPreprocessorOpts();
351 
352     if (!PPOpts.ImplicitPCHInclude.empty()) {
353       auto File = CI.getFileManager().getFile(PPOpts.ImplicitPCHInclude);
354       if (File)
355         DataConsumer->importedPCH(*File);
356     }
357 
358     DataConsumer->setASTContext(CI.getASTContext());
359     Preprocessor &PP = CI.getPreprocessor();
360     PP.addPPCallbacks(std::make_unique<IndexPPCallbacks>(PP, *DataConsumer));
361     DataConsumer->setPreprocessor(CI.getPreprocessorPtr());
362 
363     if (SKData) {
364       auto *PPRec = new PPConditionalDirectiveRecord(PP.getSourceManager());
365       PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(PPRec));
366       ParsedLocsTracker =
367           std::make_unique<ParsedSrcLocationsTracker>(*SKData, *PPRec, PP);
368     }
369 
370     std::vector<std::unique_ptr<ASTConsumer>> Consumers;
371     Consumers.push_back(std::make_unique<IndexingConsumer>(
372         *DataConsumer, ParsedLocsTracker.get()));
373     Consumers.push_back(createIndexingASTConsumer(
374         DataConsumer, Opts, CI.getPreprocessorPtr(),
375         [this](const Decl *D) { return this->shouldSkipFunctionBody(D); }));
376     return std::make_unique<MultiplexConsumer>(std::move(Consumers));
377   }
378 
379   bool shouldSkipFunctionBody(const Decl *D) {
380     if (!ParsedLocsTracker) {
381       // Always skip bodies.
382       return true;
383     }
384 
385     const SourceManager &SM = D->getASTContext().getSourceManager();
386     SourceLocation Loc = D->getLocation();
387     if (Loc.isMacroID())
388       return false;
389     if (SM.isInSystemHeader(Loc))
390       return true; // always skip bodies from system headers.
391 
392     FileID FID;
393     unsigned Offset;
394     std::tie(FID, Offset) = SM.getDecomposedLoc(Loc);
395     // Don't skip bodies from main files; this may be revisited.
396     if (SM.getMainFileID() == FID)
397       return false;
398     const FileEntry *FE = SM.getFileEntryForID(FID);
399     if (!FE)
400       return false;
401 
402     return ParsedLocsTracker->hasAlredyBeenParsed(Loc, FID, FE);
403   }
404 
405   TranslationUnitKind getTranslationUnitKind() override {
406     if (DataConsumer->shouldIndexImplicitTemplateInsts())
407       return TU_Complete;
408     else
409       return TU_Prefix;
410   }
411   bool hasCodeCompletionSupport() const override { return false; }
412 
413   void EndSourceFileAction() override {
414     if (ParsedLocsTracker)
415       ParsedLocsTracker->syncWithStorage();
416   }
417 };
418 
419 //===----------------------------------------------------------------------===//
420 // clang_indexSourceFileUnit Implementation
421 //===----------------------------------------------------------------------===//
422 
423 static IndexingOptions getIndexingOptionsFromCXOptions(unsigned index_options) {
424   IndexingOptions IdxOpts;
425   if (index_options & CXIndexOpt_IndexFunctionLocalSymbols)
426     IdxOpts.IndexFunctionLocals = true;
427   if (index_options & CXIndexOpt_IndexImplicitTemplateInstantiations)
428     IdxOpts.IndexImplicitInstantiation = true;
429   return IdxOpts;
430 }
431 
432 struct IndexSessionData {
433   CXIndex CIdx;
434   std::unique_ptr<ThreadSafeParsedRegions> SkipBodyData =
435       std::make_unique<ThreadSafeParsedRegions>();
436 
437   explicit IndexSessionData(CXIndex cIdx) : CIdx(cIdx) {}
438 };
439 
440 } // anonymous namespace
441 
442 static CXErrorCode clang_indexSourceFile_Impl(
443     CXIndexAction cxIdxAction, CXClientData client_data,
444     IndexerCallbacks *client_index_callbacks, unsigned index_callbacks_size,
445     unsigned index_options, const char *source_filename,
446     const char *const *command_line_args, int num_command_line_args,
447     ArrayRef<CXUnsavedFile> unsaved_files, CXTranslationUnit *out_TU,
448     unsigned TU_options) {
449   if (out_TU)
450     *out_TU = nullptr;
451   bool requestedToGetTU = (out_TU != nullptr);
452 
453   if (!cxIdxAction) {
454     return CXError_InvalidArguments;
455   }
456   if (!client_index_callbacks || index_callbacks_size == 0) {
457     return CXError_InvalidArguments;
458   }
459 
460   IndexerCallbacks CB;
461   memset(&CB, 0, sizeof(CB));
462   unsigned ClientCBSize = index_callbacks_size < sizeof(CB)
463                                   ? index_callbacks_size : sizeof(CB);
464   memcpy(&CB, client_index_callbacks, ClientCBSize);
465 
466   IndexSessionData *IdxSession = static_cast<IndexSessionData *>(cxIdxAction);
467   CIndexer *CXXIdx = static_cast<CIndexer *>(IdxSession->CIdx);
468 
469   if (CXXIdx->isOptEnabled(CXGlobalOpt_ThreadBackgroundPriorityForIndexing))
470     setThreadBackgroundPriority();
471 
472   CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::All;
473   if (TU_options & CXTranslationUnit_IgnoreNonErrorsFromIncludedFiles)
474     CaptureDiagnostics = CaptureDiagsKind::AllWithoutNonErrorsFromIncludes;
475   if (Logger::isLoggingEnabled())
476     CaptureDiagnostics = CaptureDiagsKind::None;
477 
478   CaptureDiagnosticConsumer *CaptureDiag = nullptr;
479   if (CaptureDiagnostics != CaptureDiagsKind::None)
480     CaptureDiag = new CaptureDiagnosticConsumer();
481 
482   // Configure the diagnostics.
483   IntrusiveRefCntPtr<DiagnosticsEngine>
484     Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions,
485                                               CaptureDiag,
486                                               /*ShouldOwnClient=*/true));
487 
488   // Recover resources if we crash before exiting this function.
489   llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
490     llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine> >
491     DiagCleanup(Diags.get());
492 
493   std::unique_ptr<std::vector<const char *>> Args(
494       new std::vector<const char *>());
495 
496   // Recover resources if we crash before exiting this method.
497   llvm::CrashRecoveryContextCleanupRegistrar<std::vector<const char*> >
498     ArgsCleanup(Args.get());
499 
500   Args->insert(Args->end(), command_line_args,
501                command_line_args + num_command_line_args);
502 
503   // The 'source_filename' argument is optional.  If the caller does not
504   // specify it then it is assumed that the source file is specified
505   // in the actual argument list.
506   // Put the source file after command_line_args otherwise if '-x' flag is
507   // present it will be unused.
508   if (source_filename)
509     Args->push_back(source_filename);
510 
511   std::shared_ptr<CompilerInvocation> CInvok =
512       createInvocationFromCommandLine(*Args, Diags);
513 
514   if (!CInvok)
515     return CXError_Failure;
516 
517   // Recover resources if we crash before exiting this function.
518   llvm::CrashRecoveryContextCleanupRegistrar<
519       std::shared_ptr<CompilerInvocation>,
520       llvm::CrashRecoveryContextDestructorCleanup<
521           std::shared_ptr<CompilerInvocation>>>
522       CInvokCleanup(&CInvok);
523 
524   if (CInvok->getFrontendOpts().Inputs.empty())
525     return CXError_Failure;
526 
527   typedef SmallVector<std::unique_ptr<llvm::MemoryBuffer>, 8> MemBufferOwner;
528   std::unique_ptr<MemBufferOwner> BufOwner(new MemBufferOwner);
529 
530   // Recover resources if we crash before exiting this method.
531   llvm::CrashRecoveryContextCleanupRegistrar<MemBufferOwner> BufOwnerCleanup(
532       BufOwner.get());
533 
534   for (auto &UF : unsaved_files) {
535     std::unique_ptr<llvm::MemoryBuffer> MB =
536         llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename);
537     CInvok->getPreprocessorOpts().addRemappedFile(UF.Filename, MB.get());
538     BufOwner->push_back(std::move(MB));
539   }
540 
541   // Since libclang is primarily used by batch tools dealing with
542   // (often very broken) source code, where spell-checking can have a
543   // significant negative impact on performance (particularly when
544   // precompiled headers are involved), we disable it.
545   CInvok->getLangOpts()->SpellChecking = false;
546 
547   if (index_options & CXIndexOpt_SuppressWarnings)
548     CInvok->getDiagnosticOpts().IgnoreWarnings = true;
549 
550   // Make sure to use the raw module format.
551   CInvok->getHeaderSearchOpts().ModuleFormat = std::string(
552       CXXIdx->getPCHContainerOperations()->getRawReader().getFormat());
553 
554   auto Unit = ASTUnit::create(CInvok, Diags, CaptureDiagnostics,
555                               /*UserFilesAreVolatile=*/true);
556   if (!Unit)
557     return CXError_InvalidArguments;
558 
559   auto *UPtr = Unit.get();
560   std::unique_ptr<CXTUOwner> CXTU(
561       new CXTUOwner(MakeCXTranslationUnit(CXXIdx, std::move(Unit))));
562 
563   // Recover resources if we crash before exiting this method.
564   llvm::CrashRecoveryContextCleanupRegistrar<CXTUOwner>
565     CXTUCleanup(CXTU.get());
566 
567   // Enable the skip-parsed-bodies optimization only for C++; this may be
568   // revisited.
569   bool SkipBodies = (index_options & CXIndexOpt_SkipParsedBodiesInSession) &&
570       CInvok->getLangOpts()->CPlusPlus;
571   if (SkipBodies)
572     CInvok->getFrontendOpts().SkipFunctionBodies = true;
573 
574   auto DataConsumer =
575     std::make_shared<CXIndexDataConsumer>(client_data, CB, index_options,
576                                           CXTU->getTU());
577   auto IndexAction = std::make_unique<IndexingFrontendAction>(
578       DataConsumer, getIndexingOptionsFromCXOptions(index_options),
579       SkipBodies ? IdxSession->SkipBodyData.get() : nullptr);
580 
581   // Recover resources if we crash before exiting this method.
582   llvm::CrashRecoveryContextCleanupRegistrar<FrontendAction>
583     IndexActionCleanup(IndexAction.get());
584 
585   bool Persistent = requestedToGetTU;
586   bool OnlyLocalDecls = false;
587   bool PrecompilePreamble = false;
588   bool CreatePreambleOnFirstParse = false;
589   bool CacheCodeCompletionResults = false;
590   PreprocessorOptions &PPOpts = CInvok->getPreprocessorOpts();
591   PPOpts.AllowPCHWithCompilerErrors = true;
592 
593   if (requestedToGetTU) {
594     OnlyLocalDecls = CXXIdx->getOnlyLocalDecls();
595     PrecompilePreamble = TU_options & CXTranslationUnit_PrecompiledPreamble;
596     CreatePreambleOnFirstParse =
597         TU_options & CXTranslationUnit_CreatePreambleOnFirstParse;
598     // FIXME: Add a flag for modules.
599     CacheCodeCompletionResults
600       = TU_options & CXTranslationUnit_CacheCompletionResults;
601   }
602 
603   if (TU_options & CXTranslationUnit_DetailedPreprocessingRecord) {
604     PPOpts.DetailedRecord = true;
605   }
606 
607   if (!requestedToGetTU && !CInvok->getLangOpts()->Modules)
608     PPOpts.DetailedRecord = false;
609 
610   // Unless the user specified that they want the preamble on the first parse
611   // set it up to be created on the first reparse. This makes the first parse
612   // faster, trading for a slower (first) reparse.
613   unsigned PrecompilePreambleAfterNParses =
614       !PrecompilePreamble ? 0 : 2 - CreatePreambleOnFirstParse;
615   DiagnosticErrorTrap DiagTrap(*Diags);
616   bool Success = ASTUnit::LoadFromCompilerInvocationAction(
617       std::move(CInvok), CXXIdx->getPCHContainerOperations(), Diags,
618       IndexAction.get(), UPtr, Persistent, CXXIdx->getClangResourcesPath(),
619       OnlyLocalDecls, CaptureDiagnostics, PrecompilePreambleAfterNParses,
620       CacheCodeCompletionResults,
621       /*IncludeBriefCommentsInCodeCompletion=*/false,
622       /*UserFilesAreVolatile=*/true);
623   if (DiagTrap.hasErrorOccurred() && CXXIdx->getDisplayDiagnostics())
624     printDiagsToStderr(UPtr);
625 
626   if (isASTReadError(UPtr))
627     return CXError_ASTReadError;
628 
629   if (!Success)
630     return CXError_Failure;
631 
632   if (out_TU)
633     *out_TU = CXTU->takeTU();
634 
635   return CXError_Success;
636 }
637 
638 //===----------------------------------------------------------------------===//
639 // clang_indexTranslationUnit Implementation
640 //===----------------------------------------------------------------------===//
641 
642 static void indexPreprocessingRecord(ASTUnit &Unit, CXIndexDataConsumer &IdxCtx) {
643   Preprocessor &PP = Unit.getPreprocessor();
644   if (!PP.getPreprocessingRecord())
645     return;
646 
647   // FIXME: Only deserialize inclusion directives.
648 
649   bool isModuleFile = Unit.isModuleFile();
650   for (PreprocessedEntity *PPE : Unit.getLocalPreprocessingEntities()) {
651     if (InclusionDirective *ID = dyn_cast<InclusionDirective>(PPE)) {
652       SourceLocation Loc = ID->getSourceRange().getBegin();
653       // Modules have synthetic main files as input, give an invalid location
654       // if the location points to such a file.
655       if (isModuleFile && Unit.isInMainFileID(Loc))
656         Loc = SourceLocation();
657       IdxCtx.ppIncludedFile(Loc, ID->getFileName(),
658                             ID->getFile(),
659                             ID->getKind() == InclusionDirective::Import,
660                             !ID->wasInQuotes(), ID->importedModule());
661     }
662   }
663 }
664 
665 static CXErrorCode clang_indexTranslationUnit_Impl(
666     CXIndexAction idxAction, CXClientData client_data,
667     IndexerCallbacks *client_index_callbacks, unsigned index_callbacks_size,
668     unsigned index_options, CXTranslationUnit TU) {
669   // Check arguments.
670   if (isNotUsableTU(TU)) {
671     LOG_BAD_TU(TU);
672     return CXError_InvalidArguments;
673   }
674   if (!client_index_callbacks || index_callbacks_size == 0) {
675     return CXError_InvalidArguments;
676   }
677 
678   CIndexer *CXXIdx = TU->CIdx;
679   if (CXXIdx->isOptEnabled(CXGlobalOpt_ThreadBackgroundPriorityForIndexing))
680     setThreadBackgroundPriority();
681 
682   IndexerCallbacks CB;
683   memset(&CB, 0, sizeof(CB));
684   unsigned ClientCBSize = index_callbacks_size < sizeof(CB)
685                                   ? index_callbacks_size : sizeof(CB);
686   memcpy(&CB, client_index_callbacks, ClientCBSize);
687 
688   CXIndexDataConsumer DataConsumer(client_data, CB, index_options, TU);
689 
690   ASTUnit *Unit = cxtu::getASTUnit(TU);
691   if (!Unit)
692     return CXError_Failure;
693 
694   ASTUnit::ConcurrencyCheck Check(*Unit);
695 
696   if (const FileEntry *PCHFile = Unit->getPCHFile())
697     DataConsumer.importedPCH(PCHFile);
698 
699   FileManager &FileMgr = Unit->getFileManager();
700 
701   if (Unit->getOriginalSourceFileName().empty())
702     DataConsumer.enteredMainFile(nullptr);
703   else if (auto MainFile = FileMgr.getFile(Unit->getOriginalSourceFileName()))
704     DataConsumer.enteredMainFile(*MainFile);
705   else
706     DataConsumer.enteredMainFile(nullptr);
707 
708   DataConsumer.setASTContext(Unit->getASTContext());
709   DataConsumer.startedTranslationUnit();
710 
711   indexPreprocessingRecord(*Unit, DataConsumer);
712   indexASTUnit(*Unit, DataConsumer, getIndexingOptionsFromCXOptions(index_options));
713   DataConsumer.indexDiagnostics();
714 
715   return CXError_Success;
716 }
717 
718 //===----------------------------------------------------------------------===//
719 // libclang public APIs.
720 //===----------------------------------------------------------------------===//
721 
722 int clang_index_isEntityObjCContainerKind(CXIdxEntityKind K) {
723   return CXIdxEntity_ObjCClass <= K && K <= CXIdxEntity_ObjCCategory;
724 }
725 
726 const CXIdxObjCContainerDeclInfo *
727 clang_index_getObjCContainerDeclInfo(const CXIdxDeclInfo *DInfo) {
728   if (!DInfo)
729     return nullptr;
730 
731   const DeclInfo *DI = static_cast<const DeclInfo *>(DInfo);
732   if (const ObjCContainerDeclInfo *
733         ContInfo = dyn_cast<ObjCContainerDeclInfo>(DI))
734     return &ContInfo->ObjCContDeclInfo;
735 
736   return nullptr;
737 }
738 
739 const CXIdxObjCInterfaceDeclInfo *
740 clang_index_getObjCInterfaceDeclInfo(const CXIdxDeclInfo *DInfo) {
741   if (!DInfo)
742     return nullptr;
743 
744   const DeclInfo *DI = static_cast<const DeclInfo *>(DInfo);
745   if (const ObjCInterfaceDeclInfo *
746         InterInfo = dyn_cast<ObjCInterfaceDeclInfo>(DI))
747     return &InterInfo->ObjCInterDeclInfo;
748 
749   return nullptr;
750 }
751 
752 const CXIdxObjCCategoryDeclInfo *
753 clang_index_getObjCCategoryDeclInfo(const CXIdxDeclInfo *DInfo){
754   if (!DInfo)
755     return nullptr;
756 
757   const DeclInfo *DI = static_cast<const DeclInfo *>(DInfo);
758   if (const ObjCCategoryDeclInfo *
759         CatInfo = dyn_cast<ObjCCategoryDeclInfo>(DI))
760     return &CatInfo->ObjCCatDeclInfo;
761 
762   return nullptr;
763 }
764 
765 const CXIdxObjCProtocolRefListInfo *
766 clang_index_getObjCProtocolRefListInfo(const CXIdxDeclInfo *DInfo) {
767   if (!DInfo)
768     return nullptr;
769 
770   const DeclInfo *DI = static_cast<const DeclInfo *>(DInfo);
771 
772   if (const ObjCInterfaceDeclInfo *
773         InterInfo = dyn_cast<ObjCInterfaceDeclInfo>(DI))
774     return InterInfo->ObjCInterDeclInfo.protocols;
775 
776   if (const ObjCProtocolDeclInfo *
777         ProtInfo = dyn_cast<ObjCProtocolDeclInfo>(DI))
778     return &ProtInfo->ObjCProtoRefListInfo;
779 
780   if (const ObjCCategoryDeclInfo *CatInfo = dyn_cast<ObjCCategoryDeclInfo>(DI))
781     return CatInfo->ObjCCatDeclInfo.protocols;
782 
783   return nullptr;
784 }
785 
786 const CXIdxObjCPropertyDeclInfo *
787 clang_index_getObjCPropertyDeclInfo(const CXIdxDeclInfo *DInfo) {
788   if (!DInfo)
789     return nullptr;
790 
791   const DeclInfo *DI = static_cast<const DeclInfo *>(DInfo);
792   if (const ObjCPropertyDeclInfo *PropInfo = dyn_cast<ObjCPropertyDeclInfo>(DI))
793     return &PropInfo->ObjCPropDeclInfo;
794 
795   return nullptr;
796 }
797 
798 const CXIdxIBOutletCollectionAttrInfo *
799 clang_index_getIBOutletCollectionAttrInfo(const CXIdxAttrInfo *AInfo) {
800   if (!AInfo)
801     return nullptr;
802 
803   const AttrInfo *DI = static_cast<const AttrInfo *>(AInfo);
804   if (const IBOutletCollectionInfo *
805         IBInfo = dyn_cast<IBOutletCollectionInfo>(DI))
806     return &IBInfo->IBCollInfo;
807 
808   return nullptr;
809 }
810 
811 const CXIdxCXXClassDeclInfo *
812 clang_index_getCXXClassDeclInfo(const CXIdxDeclInfo *DInfo) {
813   if (!DInfo)
814     return nullptr;
815 
816   const DeclInfo *DI = static_cast<const DeclInfo *>(DInfo);
817   if (const CXXClassDeclInfo *ClassInfo = dyn_cast<CXXClassDeclInfo>(DI))
818     return &ClassInfo->CXXClassInfo;
819 
820   return nullptr;
821 }
822 
823 CXIdxClientContainer
824 clang_index_getClientContainer(const CXIdxContainerInfo *info) {
825   if (!info)
826     return nullptr;
827   const ContainerInfo *Container = static_cast<const ContainerInfo *>(info);
828   return Container->IndexCtx->getClientContainerForDC(Container->DC);
829 }
830 
831 void clang_index_setClientContainer(const CXIdxContainerInfo *info,
832                                     CXIdxClientContainer client) {
833   if (!info)
834     return;
835   const ContainerInfo *Container = static_cast<const ContainerInfo *>(info);
836   Container->IndexCtx->addContainerInMap(Container->DC, client);
837 }
838 
839 CXIdxClientEntity clang_index_getClientEntity(const CXIdxEntityInfo *info) {
840   if (!info)
841     return nullptr;
842   const EntityInfo *Entity = static_cast<const EntityInfo *>(info);
843   return Entity->IndexCtx->getClientEntity(Entity->Dcl);
844 }
845 
846 void clang_index_setClientEntity(const CXIdxEntityInfo *info,
847                                  CXIdxClientEntity client) {
848   if (!info)
849     return;
850   const EntityInfo *Entity = static_cast<const EntityInfo *>(info);
851   Entity->IndexCtx->setClientEntity(Entity->Dcl, client);
852 }
853 
854 CXIndexAction clang_IndexAction_create(CXIndex CIdx) {
855   return new IndexSessionData(CIdx);
856 }
857 
858 void clang_IndexAction_dispose(CXIndexAction idxAction) {
859   if (idxAction)
860     delete static_cast<IndexSessionData *>(idxAction);
861 }
862 
863 int clang_indexSourceFile(CXIndexAction idxAction,
864                           CXClientData client_data,
865                           IndexerCallbacks *index_callbacks,
866                           unsigned index_callbacks_size,
867                           unsigned index_options,
868                           const char *source_filename,
869                           const char * const *command_line_args,
870                           int num_command_line_args,
871                           struct CXUnsavedFile *unsaved_files,
872                           unsigned num_unsaved_files,
873                           CXTranslationUnit *out_TU,
874                           unsigned TU_options) {
875   SmallVector<const char *, 4> Args;
876   Args.push_back("clang");
877   Args.append(command_line_args, command_line_args + num_command_line_args);
878   return clang_indexSourceFileFullArgv(
879       idxAction, client_data, index_callbacks, index_callbacks_size,
880       index_options, source_filename, Args.data(), Args.size(), unsaved_files,
881       num_unsaved_files, out_TU, TU_options);
882 }
883 
884 int clang_indexSourceFileFullArgv(
885     CXIndexAction idxAction, CXClientData client_data,
886     IndexerCallbacks *index_callbacks, unsigned index_callbacks_size,
887     unsigned index_options, const char *source_filename,
888     const char *const *command_line_args, int num_command_line_args,
889     struct CXUnsavedFile *unsaved_files, unsigned num_unsaved_files,
890     CXTranslationUnit *out_TU, unsigned TU_options) {
891   LOG_FUNC_SECTION {
892     *Log << source_filename << ": ";
893     for (int i = 0; i != num_command_line_args; ++i)
894       *Log << command_line_args[i] << " ";
895   }
896 
897   if (num_unsaved_files && !unsaved_files)
898     return CXError_InvalidArguments;
899 
900   CXErrorCode result = CXError_Failure;
901   auto IndexSourceFileImpl = [=, &result]() {
902     result = clang_indexSourceFile_Impl(
903         idxAction, client_data, index_callbacks, index_callbacks_size,
904         index_options, source_filename, command_line_args,
905         num_command_line_args,
906         llvm::makeArrayRef(unsaved_files, num_unsaved_files), out_TU,
907         TU_options);
908   };
909 
910   llvm::CrashRecoveryContext CRC;
911 
912   if (!RunSafely(CRC, IndexSourceFileImpl)) {
913     fprintf(stderr, "libclang: crash detected during indexing source file: {\n");
914     fprintf(stderr, "  'source_filename' : '%s'\n", source_filename);
915     fprintf(stderr, "  'command_line_args' : [");
916     for (int i = 0; i != num_command_line_args; ++i) {
917       if (i)
918         fprintf(stderr, ", ");
919       fprintf(stderr, "'%s'", command_line_args[i]);
920     }
921     fprintf(stderr, "],\n");
922     fprintf(stderr, "  'unsaved_files' : [");
923     for (unsigned i = 0; i != num_unsaved_files; ++i) {
924       if (i)
925         fprintf(stderr, ", ");
926       fprintf(stderr, "('%s', '...', %ld)", unsaved_files[i].Filename,
927               unsaved_files[i].Length);
928     }
929     fprintf(stderr, "],\n");
930     fprintf(stderr, "  'options' : %d,\n", TU_options);
931     fprintf(stderr, "}\n");
932 
933     return 1;
934   } else if (getenv("LIBCLANG_RESOURCE_USAGE")) {
935     if (out_TU)
936       PrintLibclangResourceUsage(*out_TU);
937   }
938 
939   return result;
940 }
941 
942 int clang_indexTranslationUnit(CXIndexAction idxAction,
943                                CXClientData client_data,
944                                IndexerCallbacks *index_callbacks,
945                                unsigned index_callbacks_size,
946                                unsigned index_options,
947                                CXTranslationUnit TU) {
948   LOG_FUNC_SECTION {
949     *Log << TU;
950   }
951 
952   CXErrorCode result;
953   auto IndexTranslationUnitImpl = [=, &result]() {
954     result = clang_indexTranslationUnit_Impl(
955         idxAction, client_data, index_callbacks, index_callbacks_size,
956         index_options, TU);
957   };
958 
959   llvm::CrashRecoveryContext CRC;
960 
961   if (!RunSafely(CRC, IndexTranslationUnitImpl)) {
962     fprintf(stderr, "libclang: crash detected during indexing TU\n");
963 
964     return 1;
965   }
966 
967   return result;
968 }
969 
970 void clang_indexLoc_getFileLocation(CXIdxLoc location,
971                                     CXIdxClientFile *indexFile,
972                                     CXFile *file,
973                                     unsigned *line,
974                                     unsigned *column,
975                                     unsigned *offset) {
976   if (indexFile) *indexFile = nullptr;
977   if (file)   *file = nullptr;
978   if (line)   *line = 0;
979   if (column) *column = 0;
980   if (offset) *offset = 0;
981 
982   SourceLocation Loc = SourceLocation::getFromRawEncoding(location.int_data);
983   if (!location.ptr_data[0] || Loc.isInvalid())
984     return;
985 
986   CXIndexDataConsumer &DataConsumer =
987       *static_cast<CXIndexDataConsumer*>(location.ptr_data[0]);
988   DataConsumer.translateLoc(Loc, indexFile, file, line, column, offset);
989 }
990 
991 CXSourceLocation clang_indexLoc_getCXSourceLocation(CXIdxLoc location) {
992   SourceLocation Loc = SourceLocation::getFromRawEncoding(location.int_data);
993   if (!location.ptr_data[0] || Loc.isInvalid())
994     return clang_getNullLocation();
995 
996   CXIndexDataConsumer &DataConsumer =
997       *static_cast<CXIndexDataConsumer*>(location.ptr_data[0]);
998   return cxloc::translateSourceLocation(DataConsumer.getASTContext(), Loc);
999 }
1000