1 //===-- Background.cpp - Build an index in a background thread ------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "index/Background.h"
11 #include "ClangdUnit.h"
12 #include "Compiler.h"
13 #include "Logger.h"
14 #include "SourceCode.h"
15 #include "Threading.h"
16 #include "Trace.h"
17 #include "URI.h"
18 #include "index/IndexAction.h"
19 #include "index/MemIndex.h"
20 #include "index/Serialization.h"
21 #include "index/SymbolCollector.h"
22 #include "clang/Basic/SourceLocation.h"
23 #include "clang/Basic/SourceManager.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/ScopeExit.h"
26 #include "llvm/ADT/StringMap.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/Support/SHA1.h"
29
30 #include <chrono>
31 #include <memory>
32 #include <numeric>
33 #include <queue>
34 #include <random>
35 #include <string>
36 #include <thread>
37
38 namespace clang {
39 namespace clangd {
40 namespace {
41 // Resolves URI to file paths with cache.
42 class URIToFileCache {
43 public:
URIToFileCache(llvm::StringRef HintPath)44 URIToFileCache(llvm::StringRef HintPath) : HintPath(HintPath) {}
45
resolve(llvm::StringRef FileURI)46 llvm::StringRef resolve(llvm::StringRef FileURI) {
47 auto I = URIToPathCache.try_emplace(FileURI);
48 if (I.second) {
49 auto U = URI::parse(FileURI);
50 if (!U) {
51 elog("Failed to parse URI {0}: {1}", FileURI, U.takeError());
52 assert(false && "Failed to parse URI");
53 return "";
54 }
55 auto Path = URI::resolve(*U, HintPath);
56 if (!Path) {
57 elog("Failed to resolve URI {0}: {1}", FileURI, Path.takeError());
58 assert(false && "Failed to resolve URI");
59 return "";
60 }
61 I.first->second = *Path;
62 }
63 return I.first->second;
64 }
65
66 private:
67 std::string HintPath;
68 llvm::StringMap<std::string> URIToPathCache;
69 };
70
71 // We keep only the node "U" and its edges. Any node other than "U" will be
72 // empty in the resultant graph.
getSubGraph(const URI & U,const IncludeGraph & FullGraph)73 IncludeGraph getSubGraph(const URI &U, const IncludeGraph &FullGraph) {
74 IncludeGraph IG;
75
76 std::string FileURI = U.toString();
77 auto Entry = IG.try_emplace(FileURI).first;
78 auto &Node = Entry->getValue();
79 Node = FullGraph.lookup(Entry->getKey());
80 Node.URI = Entry->getKey();
81
82 // URIs inside nodes must point into the keys of the same IncludeGraph.
83 for (auto &Include : Node.DirectIncludes) {
84 auto I = IG.try_emplace(Include).first;
85 I->getValue().URI = I->getKey();
86 Include = I->getKey();
87 }
88
89 return IG;
90 }
91
92 // Creates a filter to not collect index results from files with unchanged
93 // digests.
94 // \p FileDigests contains file digests for the current indexed files.
95 decltype(SymbolCollector::Options::FileFilter)
createFileFilter(const llvm::StringMap<FileDigest> & FileDigests)96 createFileFilter(const llvm::StringMap<FileDigest> &FileDigests) {
97 return [&FileDigests](const SourceManager &SM, FileID FID) {
98 const auto *F = SM.getFileEntryForID(FID);
99 if (!F)
100 return false; // Skip invalid files.
101 auto AbsPath = getCanonicalPath(F, SM);
102 if (!AbsPath)
103 return false; // Skip files without absolute path.
104 auto Digest = digestFile(SM, FID);
105 if (!Digest)
106 return false;
107 auto D = FileDigests.find(*AbsPath);
108 if (D != FileDigests.end() && D->second == Digest)
109 return false; // Skip files that haven't changed.
110 return true;
111 };
112 }
113
114 // We cannot use vfs->makeAbsolute because Cmd.FileName is either absolute or
115 // relative to Cmd.Directory, which might not be the same as current working
116 // directory.
getAbsolutePath(const tooling::CompileCommand & Cmd)117 llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) {
118 llvm::SmallString<128> AbsolutePath;
119 if (llvm::sys::path::is_absolute(Cmd.Filename)) {
120 AbsolutePath = Cmd.Filename;
121 } else {
122 AbsolutePath = Cmd.Directory;
123 llvm::sys::path::append(AbsolutePath, Cmd.Filename);
124 }
125 return AbsolutePath;
126 }
127 } // namespace
128
BackgroundIndex(Context BackgroundContext,const FileSystemProvider & FSProvider,const GlobalCompilationDatabase & CDB,BackgroundIndexStorage::Factory IndexStorageFactory,size_t BuildIndexPeriodMs,size_t ThreadPoolSize)129 BackgroundIndex::BackgroundIndex(
130 Context BackgroundContext, const FileSystemProvider &FSProvider,
131 const GlobalCompilationDatabase &CDB,
132 BackgroundIndexStorage::Factory IndexStorageFactory,
133 size_t BuildIndexPeriodMs, size_t ThreadPoolSize)
134 : SwapIndex(llvm::make_unique<MemIndex>()), FSProvider(FSProvider),
135 CDB(CDB), BackgroundContext(std::move(BackgroundContext)),
136 BuildIndexPeriodMs(BuildIndexPeriodMs),
137 SymbolsUpdatedSinceLastIndex(false),
138 IndexStorageFactory(std::move(IndexStorageFactory)),
139 CommandsChanged(
140 CDB.watch([&](const std::vector<std::string> &ChangedFiles) {
141 enqueue(ChangedFiles);
142 })) {
143 assert(ThreadPoolSize > 0 && "Thread pool size can't be zero.");
144 assert(this->IndexStorageFactory && "Storage factory can not be null!");
145 while (ThreadPoolSize--)
__anon0b2f3d740402null146 ThreadPool.emplace_back([this] { run(); });
147 if (BuildIndexPeriodMs > 0) {
148 log("BackgroundIndex: build symbol index periodically every {0} ms.",
149 BuildIndexPeriodMs);
__anon0b2f3d740502null150 ThreadPool.emplace_back([this] { buildIndex(); });
151 }
152 }
153
~BackgroundIndex()154 BackgroundIndex::~BackgroundIndex() {
155 stop();
156 for (auto &Thread : ThreadPool)
157 Thread.join();
158 }
159
stop()160 void BackgroundIndex::stop() {
161 {
162 std::lock_guard<std::mutex> QueueLock(QueueMu);
163 std::lock_guard<std::mutex> IndexLock(IndexMu);
164 ShouldStop = true;
165 }
166 QueueCV.notify_all();
167 IndexCV.notify_all();
168 }
169
run()170 void BackgroundIndex::run() {
171 WithContext Background(BackgroundContext.clone());
172 while (true) {
173 llvm::Optional<Task> Task;
174 ThreadPriority Priority;
175 {
176 std::unique_lock<std::mutex> Lock(QueueMu);
177 QueueCV.wait(Lock, [&] { return ShouldStop || !Queue.empty(); });
178 if (ShouldStop) {
179 Queue.clear();
180 QueueCV.notify_all();
181 return;
182 }
183 ++NumActiveTasks;
184 std::tie(Task, Priority) = std::move(Queue.front());
185 Queue.pop_front();
186 }
187
188 if (Priority != ThreadPriority::Normal)
189 setCurrentThreadPriority(Priority);
190 (*Task)();
191 if (Priority != ThreadPriority::Normal)
192 setCurrentThreadPriority(ThreadPriority::Normal);
193
194 {
195 std::unique_lock<std::mutex> Lock(QueueMu);
196 assert(NumActiveTasks > 0 && "before decrementing");
197 --NumActiveTasks;
198 }
199 QueueCV.notify_all();
200 }
201 }
202
blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds)203 bool BackgroundIndex::blockUntilIdleForTest(
204 llvm::Optional<double> TimeoutSeconds) {
205 std::unique_lock<std::mutex> Lock(QueueMu);
206 return wait(Lock, QueueCV, timeoutSeconds(TimeoutSeconds),
207 [&] { return Queue.empty() && NumActiveTasks == 0; });
208 }
209
enqueue(const std::vector<std::string> & ChangedFiles)210 void BackgroundIndex::enqueue(const std::vector<std::string> &ChangedFiles) {
211 enqueueTask(
212 [this, ChangedFiles] {
213 trace::Span Tracer("BackgroundIndexEnqueue");
214 // We're doing this asynchronously, because we'll read shards here too.
215 log("Enqueueing {0} commands for indexing", ChangedFiles.size());
216 SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size()));
217
218 auto NeedsReIndexing = loadShards(std::move(ChangedFiles));
219 // Run indexing for files that need to be updated.
220 std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
221 std::mt19937(std::random_device{}()));
222 for (auto &Elem : NeedsReIndexing)
223 enqueue(std::move(Elem.first), Elem.second);
224 },
225 ThreadPriority::Normal);
226 }
227
enqueue(tooling::CompileCommand Cmd,BackgroundIndexStorage * Storage)228 void BackgroundIndex::enqueue(tooling::CompileCommand Cmd,
229 BackgroundIndexStorage *Storage) {
230 enqueueTask(Bind(
231 [this, Storage](tooling::CompileCommand Cmd) {
232 // We can't use llvm::StringRef here since we are going to
233 // move from Cmd during the call below.
234 const std::string FileName = Cmd.Filename;
235 if (auto Error = index(std::move(Cmd), Storage))
236 elog("Indexing {0} failed: {1}", FileName,
237 std::move(Error));
238 },
239 std::move(Cmd)),
240 ThreadPriority::Low);
241 }
242
enqueueTask(Task T,ThreadPriority Priority)243 void BackgroundIndex::enqueueTask(Task T, ThreadPriority Priority) {
244 {
245 std::lock_guard<std::mutex> Lock(QueueMu);
246 auto I = Queue.end();
247 // We first store the tasks with Normal priority in the front of the queue.
248 // Then we store low priority tasks. Normal priority tasks are pretty rare,
249 // they should not grow beyond single-digit numbers, so it is OK to do
250 // linear search and insert after that.
251 if (Priority == ThreadPriority::Normal) {
252 I = llvm::find_if(Queue, [](const std::pair<Task, ThreadPriority> &Elem) {
253 return Elem.second == ThreadPriority::Low;
254 });
255 }
256 Queue.insert(I, {std::move(T), Priority});
257 }
258 QueueCV.notify_all();
259 }
260
261 /// Given index results from a TU, only update symbols coming from files that
262 /// are different or missing from than \p DigestsSnapshot. Also stores new index
263 /// information on IndexStorage.
update(llvm::StringRef MainFile,IndexFileIn Index,const llvm::StringMap<FileDigest> & DigestsSnapshot,BackgroundIndexStorage * IndexStorage)264 void BackgroundIndex::update(llvm::StringRef MainFile, IndexFileIn Index,
265 const llvm::StringMap<FileDigest> &DigestsSnapshot,
266 BackgroundIndexStorage *IndexStorage) {
267 // Partition symbols/references into files.
268 struct File {
269 llvm::DenseSet<const Symbol *> Symbols;
270 llvm::DenseSet<const Ref *> Refs;
271 FileDigest Digest;
272 };
273 llvm::StringMap<File> Files;
274 URIToFileCache URICache(MainFile);
275 for (const auto &IndexIt : *Index.Sources) {
276 const auto &IGN = IndexIt.getValue();
277 const auto AbsPath = URICache.resolve(IGN.URI);
278 const auto DigestIt = DigestsSnapshot.find(AbsPath);
279 // File has different contents.
280 if (DigestIt == DigestsSnapshot.end() || DigestIt->getValue() != IGN.Digest)
281 Files.try_emplace(AbsPath).first->getValue().Digest = IGN.Digest;
282 }
283 for (const auto &Sym : *Index.Symbols) {
284 if (Sym.CanonicalDeclaration) {
285 auto DeclPath = URICache.resolve(Sym.CanonicalDeclaration.FileURI);
286 const auto FileIt = Files.find(DeclPath);
287 if (FileIt != Files.end())
288 FileIt->second.Symbols.insert(&Sym);
289 }
290 // For symbols with different declaration and definition locations, we store
291 // the full symbol in both the header file and the implementation file, so
292 // that merging can tell the preferred symbols (from canonical headers) from
293 // other symbols (e.g. forward declarations).
294 if (Sym.Definition &&
295 Sym.Definition.FileURI != Sym.CanonicalDeclaration.FileURI) {
296 auto DefPath = URICache.resolve(Sym.Definition.FileURI);
297 const auto FileIt = Files.find(DefPath);
298 if (FileIt != Files.end())
299 FileIt->second.Symbols.insert(&Sym);
300 }
301 }
302 llvm::DenseMap<const Ref *, SymbolID> RefToIDs;
303 for (const auto &SymRefs : *Index.Refs) {
304 for (const auto &R : SymRefs.second) {
305 auto Path = URICache.resolve(R.Location.FileURI);
306 const auto FileIt = Files.find(Path);
307 if (FileIt != Files.end()) {
308 auto &F = FileIt->getValue();
309 RefToIDs[&R] = SymRefs.first;
310 F.Refs.insert(&R);
311 }
312 }
313 }
314
315 // Build and store new slabs for each updated file.
316 for (const auto &FileIt : Files) {
317 llvm::StringRef Path = FileIt.getKey();
318 SymbolSlab::Builder Syms;
319 RefSlab::Builder Refs;
320 for (const auto *S : FileIt.second.Symbols)
321 Syms.insert(*S);
322 for (const auto *R : FileIt.second.Refs)
323 Refs.insert(RefToIDs[R], *R);
324 auto SS = llvm::make_unique<SymbolSlab>(std::move(Syms).build());
325 auto RS = llvm::make_unique<RefSlab>(std::move(Refs).build());
326 auto IG = llvm::make_unique<IncludeGraph>(
327 getSubGraph(URI::create(Path), Index.Sources.getValue()));
328 // We need to store shards before updating the index, since the latter
329 // consumes slabs.
330 if (IndexStorage) {
331 IndexFileOut Shard;
332 Shard.Symbols = SS.get();
333 Shard.Refs = RS.get();
334 Shard.Sources = IG.get();
335
336 if (auto Error = IndexStorage->storeShard(Path, Shard))
337 elog("Failed to write background-index shard for file {0}: {1}", Path,
338 std::move(Error));
339 }
340 {
341 std::lock_guard<std::mutex> Lock(DigestsMu);
342 auto Hash = FileIt.second.Digest;
343 // Skip if file is already up to date.
344 auto DigestIt = IndexedFileDigests.try_emplace(Path);
345 if (!DigestIt.second && DigestIt.first->second == Hash)
346 continue;
347 DigestIt.first->second = Hash;
348 // This can override a newer version that is added in another thread, if
349 // this thread sees the older version but finishes later. This should be
350 // rare in practice.
351 IndexedSymbols.update(Path, std::move(SS), std::move(RS));
352 }
353 }
354 }
355
buildIndex()356 void BackgroundIndex::buildIndex() {
357 assert(BuildIndexPeriodMs > 0);
358 while (true) {
359 {
360 std::unique_lock<std::mutex> Lock(IndexMu);
361 if (ShouldStop) // Avoid waiting if stopped.
362 break;
363 // Wait until this is notified to stop or `BuildIndexPeriodMs` has past.
364 IndexCV.wait_for(Lock, std::chrono::milliseconds(BuildIndexPeriodMs));
365 if (ShouldStop) // Avoid rebuilding index if stopped.
366 break;
367 }
368 if (!SymbolsUpdatedSinceLastIndex.exchange(false))
369 continue;
370 // There can be symbol update right after the flag is reset above and before
371 // index is rebuilt below. The new index would contain the updated symbols
372 // but the flag would still be true. This is fine as we would simply run an
373 // extra index build.
374 reset(
375 IndexedSymbols.buildIndex(IndexType::Heavy, DuplicateHandling::Merge));
376 log("BackgroundIndex: rebuilt symbol index.");
377 }
378 }
379
index(tooling::CompileCommand Cmd,BackgroundIndexStorage * IndexStorage)380 llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd,
381 BackgroundIndexStorage *IndexStorage) {
382 trace::Span Tracer("BackgroundIndex");
383 SPAN_ATTACH(Tracer, "file", Cmd.Filename);
384 auto AbsolutePath = getAbsolutePath(Cmd);
385
386 auto FS = FSProvider.getFileSystem();
387 auto Buf = FS->getBufferForFile(AbsolutePath);
388 if (!Buf)
389 return llvm::errorCodeToError(Buf.getError());
390 auto Hash = digest(Buf->get()->getBuffer());
391
392 // Take a snapshot of the digests to avoid locking for each file in the TU.
393 llvm::StringMap<FileDigest> DigestsSnapshot;
394 {
395 std::lock_guard<std::mutex> Lock(DigestsMu);
396 DigestsSnapshot = IndexedFileDigests;
397 }
398
399 log("Indexing {0} (digest:={1})", Cmd.Filename, llvm::toHex(Hash));
400 ParseInputs Inputs;
401 Inputs.FS = std::move(FS);
402 Inputs.FS->setCurrentWorkingDirectory(Cmd.Directory);
403 Inputs.CompileCommand = std::move(Cmd);
404 auto CI = buildCompilerInvocation(Inputs);
405 if (!CI)
406 return llvm::createStringError(llvm::inconvertibleErrorCode(),
407 "Couldn't build compiler invocation");
408 IgnoreDiagnostics IgnoreDiags;
409 auto Clang = prepareCompilerInstance(
410 std::move(CI), /*Preamble=*/nullptr, std::move(*Buf),
411 std::make_shared<PCHContainerOperations>(), Inputs.FS, IgnoreDiags);
412 if (!Clang)
413 return llvm::createStringError(llvm::inconvertibleErrorCode(),
414 "Couldn't build compiler instance");
415
416 SymbolCollector::Options IndexOpts;
417 IndexOpts.FileFilter = createFileFilter(DigestsSnapshot);
418 IndexFileIn Index;
419 auto Action = createStaticIndexingAction(
420 IndexOpts, [&](SymbolSlab S) { Index.Symbols = std::move(S); },
421 [&](RefSlab R) { Index.Refs = std::move(R); },
422 [&](IncludeGraph IG) { Index.Sources = std::move(IG); });
423
424 // We're going to run clang here, and it could potentially crash.
425 // We could use CrashRecoveryContext to try to make indexing crashes nonfatal,
426 // but the leaky "recovery" is pretty scary too in a long-running process.
427 // If crashes are a real problem, maybe we should fork a child process.
428
429 const FrontendInputFile &Input = Clang->getFrontendOpts().Inputs.front();
430 if (!Action->BeginSourceFile(*Clang, Input))
431 return llvm::createStringError(llvm::inconvertibleErrorCode(),
432 "BeginSourceFile() failed");
433 if (!Action->Execute())
434 return llvm::createStringError(llvm::inconvertibleErrorCode(),
435 "Execute() failed");
436 Action->EndSourceFile();
437 if (Clang->hasDiagnostics() &&
438 Clang->getDiagnostics().hasUncompilableErrorOccurred()) {
439 return llvm::createStringError(
440 llvm::inconvertibleErrorCode(),
441 "IndexingAction failed: has uncompilable errors");
442 }
443
444 assert(Index.Symbols && Index.Refs && Index.Sources &&
445 "Symbols, Refs and Sources must be set.");
446
447 log("Indexed {0} ({1} symbols, {2} refs, {3} files)",
448 Inputs.CompileCommand.Filename, Index.Symbols->size(),
449 Index.Refs->numRefs(), Index.Sources->size());
450 SPAN_ATTACH(Tracer, "symbols", int(Index.Symbols->size()));
451 SPAN_ATTACH(Tracer, "refs", int(Index.Refs->numRefs()));
452 SPAN_ATTACH(Tracer, "sources", int(Index.Sources->size()));
453
454 update(AbsolutePath, std::move(Index), DigestsSnapshot, IndexStorage);
455
456 if (BuildIndexPeriodMs > 0)
457 SymbolsUpdatedSinceLastIndex = true;
458 else
459 reset(
460 IndexedSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge));
461
462 return llvm::Error::success();
463 }
464
465 std::vector<BackgroundIndex::Source>
loadShard(const tooling::CompileCommand & Cmd,BackgroundIndexStorage * IndexStorage,llvm::StringSet<> & LoadedShards)466 BackgroundIndex::loadShard(const tooling::CompileCommand &Cmd,
467 BackgroundIndexStorage *IndexStorage,
468 llvm::StringSet<> &LoadedShards) {
469 struct ShardInfo {
470 std::string AbsolutePath;
471 std::unique_ptr<IndexFileIn> Shard;
472 FileDigest Digest;
473 };
474 std::vector<ShardInfo> IntermediateSymbols;
475 // Make sure we don't have duplicate elements in the queue. Keys are absolute
476 // paths.
477 llvm::StringSet<> InQueue;
478 auto FS = FSProvider.getFileSystem();
479 // Dependencies of this TU, paired with the information about whether they
480 // need to be re-indexed or not.
481 std::vector<Source> Dependencies;
482 std::queue<Source> ToVisit;
483 std::string AbsolutePath = getAbsolutePath(Cmd).str();
484 // Up until we load the shard related to a dependency it needs to be
485 // re-indexed.
486 ToVisit.emplace(AbsolutePath, true);
487 InQueue.insert(AbsolutePath);
488 // Goes over each dependency.
489 while (!ToVisit.empty()) {
490 Dependencies.push_back(std::move(ToVisit.front()));
491 // Dependencies is not modified during the rest of the loop, so it is safe
492 // to keep the reference.
493 auto &CurDependency = Dependencies.back();
494 ToVisit.pop();
495 // If we have already seen this shard before(either loaded or failed) don't
496 // re-try again. Since the information in the shard won't change from one TU
497 // to another.
498 if (!LoadedShards.try_emplace(CurDependency.Path).second) {
499 // If the dependency needs to be re-indexed, first occurence would already
500 // have detected that, so we don't need to issue it again.
501 CurDependency.NeedsReIndexing = false;
502 continue;
503 }
504
505 auto Shard = IndexStorage->loadShard(CurDependency.Path);
506 if (!Shard || !Shard->Sources) {
507 // File will be returned as requiring re-indexing to caller.
508 vlog("Failed to load shard: {0}", CurDependency.Path);
509 continue;
510 }
511 // These are the edges in the include graph for current dependency.
512 for (const auto &I : *Shard->Sources) {
513 auto U = URI::parse(I.getKey());
514 if (!U)
515 continue;
516 auto AbsolutePath = URI::resolve(*U, CurDependency.Path);
517 if (!AbsolutePath)
518 continue;
519 // Add file as dependency if haven't seen before.
520 if (InQueue.try_emplace(*AbsolutePath).second)
521 ToVisit.emplace(*AbsolutePath, true);
522 // The node contains symbol information only for current file, the rest is
523 // just edges.
524 if (*AbsolutePath != CurDependency.Path)
525 continue;
526
527 // We found source file info for current dependency.
528 assert(I.getValue().Digest != FileDigest{{0}} && "Digest is empty?");
529 ShardInfo SI;
530 SI.AbsolutePath = CurDependency.Path;
531 SI.Shard = std::move(Shard);
532 SI.Digest = I.getValue().Digest;
533 IntermediateSymbols.push_back(std::move(SI));
534 // Check if the source needs re-indexing.
535 // Get the digest, skip it if file doesn't exist.
536 auto Buf = FS->getBufferForFile(CurDependency.Path);
537 if (!Buf) {
538 elog("Couldn't get buffer for file: {0}: {1}", CurDependency.Path,
539 Buf.getError().message());
540 continue;
541 }
542 // If digests match then dependency doesn't need re-indexing.
543 CurDependency.NeedsReIndexing =
544 digest(Buf->get()->getBuffer()) != I.getValue().Digest;
545 }
546 }
547 // Load shard information into background-index.
548 {
549 std::lock_guard<std::mutex> Lock(DigestsMu);
550 // This can override a newer version that is added in another thread,
551 // if this thread sees the older version but finishes later. This
552 // should be rare in practice.
553 for (const ShardInfo &SI : IntermediateSymbols) {
554 auto SS =
555 SI.Shard->Symbols
556 ? llvm::make_unique<SymbolSlab>(std::move(*SI.Shard->Symbols))
557 : nullptr;
558 auto RS = SI.Shard->Refs
559 ? llvm::make_unique<RefSlab>(std::move(*SI.Shard->Refs))
560 : nullptr;
561 IndexedFileDigests[SI.AbsolutePath] = SI.Digest;
562 IndexedSymbols.update(SI.AbsolutePath, std::move(SS), std::move(RS));
563 }
564 }
565
566 return Dependencies;
567 }
568
569 // Goes over each changed file and loads them from index. Returns the list of
570 // TUs that had out-of-date/no shards.
571 std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
loadShards(std::vector<std::string> ChangedFiles)572 BackgroundIndex::loadShards(std::vector<std::string> ChangedFiles) {
573 std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
574 NeedsReIndexing;
575 // Keeps track of the files that will be reindexed, to make sure we won't
576 // re-index same dependencies more than once. Keys are AbsolutePaths.
577 llvm::StringSet<> FilesToIndex;
578 // Keeps track of the loaded shards to make sure we don't perform redundant
579 // disk IO. Keys are absolute paths.
580 llvm::StringSet<> LoadedShards;
581 for (const auto &File : ChangedFiles) {
582 ProjectInfo PI;
583 auto Cmd = CDB.getCompileCommand(File, &PI);
584 if (!Cmd)
585 continue;
586 BackgroundIndexStorage *IndexStorage = IndexStorageFactory(PI.SourceRoot);
587 auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards);
588 for (const auto &Dependency : Dependencies) {
589 if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path))
590 continue;
591 // FIXME: Currently, we simply schedule indexing on a TU whenever any of
592 // its dependencies needs re-indexing. We might do it smarter by figuring
593 // out a minimal set of TUs that will cover all the stale dependencies.
594 vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.",
595 Cmd->Filename, Dependency.Path);
596 NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage});
597 // Mark all of this TU's dependencies as to-be-indexed so that we won't
598 // try to re-index those.
599 for (const auto &Dependency : Dependencies)
600 FilesToIndex.insert(Dependency.Path);
601 break;
602 }
603 }
604 vlog("Loaded all shards");
605 reset(IndexedSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge));
606
607 return NeedsReIndexing;
608 }
609
610 } // namespace clangd
611 } // namespace clang
612