1 //===-- TUSchedulerTests.cpp ------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "Annotations.h"
10 #include "ClangdServer.h"
11 #include "Diagnostics.h"
12 #include "GlobalCompilationDatabase.h"
13 #include "Matchers.h"
14 #include "ParsedAST.h"
15 #include "Preamble.h"
16 #include "TUScheduler.h"
17 #include "TestFS.h"
18 #include "TestIndex.h"
19 #include "support/Cancellation.h"
20 #include "support/Context.h"
21 #include "support/Path.h"
22 #include "support/TestTracer.h"
23 #include "support/Threading.h"
24 #include "support/ThreadsafeFS.h"
25 #include "clang/Basic/DiagnosticDriver.h"
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/FunctionExtras.h"
28 #include "llvm/ADT/STLExtras.h"
29 #include "llvm/ADT/ScopeExit.h"
30 #include "llvm/ADT/StringExtras.h"
31 #include "llvm/ADT/StringMap.h"
32 #include "llvm/ADT/StringRef.h"
33 #include "gmock/gmock.h"
34 #include "gtest/gtest.h"
35 #include <algorithm>
36 #include <atomic>
37 #include <chrono>
38 #include <cstdint>
39 #include <memory>
40 #include <string>
41 #include <utility>
42
43 namespace clang {
44 namespace clangd {
45 namespace {
46
47 using ::testing::AllOf;
48 using ::testing::AnyOf;
49 using ::testing::Contains;
50 using ::testing::Each;
51 using ::testing::ElementsAre;
52 using ::testing::Eq;
53 using ::testing::Field;
54 using ::testing::IsEmpty;
55 using ::testing::Not;
56 using ::testing::Pair;
57 using ::testing::Pointee;
58 using ::testing::SizeIs;
59 using ::testing::UnorderedElementsAre;
60
61 MATCHER_P2(TUState, PreambleActivity, ASTActivity, "") {
62 if (arg.PreambleActivity != PreambleActivity) {
63 *result_listener << "preamblestate is "
64 << static_cast<uint8_t>(arg.PreambleActivity);
65 return false;
66 }
67 if (arg.ASTActivity.K != ASTActivity) {
68 *result_listener << "aststate is " << arg.ASTActivity.K;
69 return false;
70 }
71 return true;
72 }
73
74 // Simple ContextProvider to verify the provider is invoked & contexts are used.
75 static Key<std::string> BoundPath;
bindPath(PathRef F)76 Context bindPath(PathRef F) {
77 return Context::current().derive(BoundPath, F.str());
78 }
boundPath()79 llvm::StringRef boundPath() {
80 const std::string *V = Context::current().get(BoundPath);
81 return V ? *V : llvm::StringRef("");
82 }
83
optsForTest()84 TUScheduler::Options optsForTest() {
85 TUScheduler::Options Opts(ClangdServer::optsForTest());
86 Opts.ContextProvider = bindPath;
87 return Opts;
88 }
89
90 class TUSchedulerTests : public ::testing::Test {
91 protected:
getInputs(PathRef File,std::string Contents)92 ParseInputs getInputs(PathRef File, std::string Contents) {
93 ParseInputs Inputs;
94 Inputs.CompileCommand = *CDB.getCompileCommand(File);
95 Inputs.TFS = &FS;
96 Inputs.Contents = std::move(Contents);
97 Inputs.Opts = ParseOptions();
98 return Inputs;
99 }
100
updateWithCallback(TUScheduler & S,PathRef File,llvm::StringRef Contents,WantDiagnostics WD,llvm::unique_function<void ()> CB)101 void updateWithCallback(TUScheduler &S, PathRef File,
102 llvm::StringRef Contents, WantDiagnostics WD,
103 llvm::unique_function<void()> CB) {
104 updateWithCallback(S, File, getInputs(File, std::string(Contents)), WD,
105 std::move(CB));
106 }
107
updateWithCallback(TUScheduler & S,PathRef File,ParseInputs Inputs,WantDiagnostics WD,llvm::unique_function<void ()> CB)108 void updateWithCallback(TUScheduler &S, PathRef File, ParseInputs Inputs,
109 WantDiagnostics WD,
110 llvm::unique_function<void()> CB) {
111 WithContextValue Ctx(llvm::make_scope_exit(std::move(CB)));
112 S.update(File, Inputs, WD);
113 }
114
115 static Key<llvm::unique_function<void(PathRef File, std::vector<Diag>)>>
116 DiagsCallbackKey;
117
118 /// A diagnostics callback that should be passed to TUScheduler when it's used
119 /// in updateWithDiags.
captureDiags()120 static std::unique_ptr<ParsingCallbacks> captureDiags() {
121 class CaptureDiags : public ParsingCallbacks {
122 public:
123 void onMainAST(PathRef File, ParsedAST &AST, PublishFn Publish) override {
124 reportDiagnostics(File, *AST.getDiagnostics(), Publish);
125 }
126
127 void onFailedAST(PathRef File, llvm::StringRef Version,
128 std::vector<Diag> Diags, PublishFn Publish) override {
129 reportDiagnostics(File, Diags, Publish);
130 }
131
132 private:
133 void reportDiagnostics(PathRef File, llvm::ArrayRef<Diag> Diags,
134 PublishFn Publish) {
135 auto D = Context::current().get(DiagsCallbackKey);
136 if (!D)
137 return;
138 Publish([&]() {
139 const_cast<
140 llvm::unique_function<void(PathRef, std::vector<Diag>)> &> (*D)(
141 File, std::move(Diags));
142 });
143 }
144 };
145 return std::make_unique<CaptureDiags>();
146 }
147
148 /// Schedule an update and call \p CB with the diagnostics it produces, if
149 /// any. The TUScheduler should be created with captureDiags as a
150 /// DiagsCallback for this to work.
updateWithDiags(TUScheduler & S,PathRef File,ParseInputs Inputs,WantDiagnostics WD,llvm::unique_function<void (std::vector<Diag>)> CB)151 void updateWithDiags(TUScheduler &S, PathRef File, ParseInputs Inputs,
152 WantDiagnostics WD,
153 llvm::unique_function<void(std::vector<Diag>)> CB) {
154 Path OrigFile = File.str();
155 WithContextValue Ctx(DiagsCallbackKey,
156 [OrigFile, CB = std::move(CB)](
157 PathRef File, std::vector<Diag> Diags) mutable {
158 assert(File == OrigFile);
159 CB(std::move(Diags));
160 });
161 S.update(File, std::move(Inputs), WD);
162 }
163
updateWithDiags(TUScheduler & S,PathRef File,llvm::StringRef Contents,WantDiagnostics WD,llvm::unique_function<void (std::vector<Diag>)> CB)164 void updateWithDiags(TUScheduler &S, PathRef File, llvm::StringRef Contents,
165 WantDiagnostics WD,
166 llvm::unique_function<void(std::vector<Diag>)> CB) {
167 return updateWithDiags(S, File, getInputs(File, std::string(Contents)), WD,
168 std::move(CB));
169 }
170
171 MockFS FS;
172 MockCompilationDatabase CDB;
173 };
174
175 Key<llvm::unique_function<void(PathRef File, std::vector<Diag>)>>
176 TUSchedulerTests::DiagsCallbackKey;
177
TEST_F(TUSchedulerTests,MissingFiles)178 TEST_F(TUSchedulerTests, MissingFiles) {
179 TUScheduler S(CDB, optsForTest());
180
181 auto Added = testPath("added.cpp");
182 FS.Files[Added] = "x";
183
184 auto Missing = testPath("missing.cpp");
185 FS.Files[Missing] = "";
186
187 S.update(Added, getInputs(Added, "x"), WantDiagnostics::No);
188
189 // Assert each operation for missing file is an error (even if it's
190 // available in VFS).
191 S.runWithAST("", Missing,
192 [&](Expected<InputsAndAST> AST) { EXPECT_ERROR(AST); });
193 S.runWithPreamble(
194 "", Missing, TUScheduler::Stale,
195 [&](Expected<InputsAndPreamble> Preamble) { EXPECT_ERROR(Preamble); });
196 // remove() shouldn't crash on missing files.
197 S.remove(Missing);
198
199 // Assert there aren't any errors for added file.
200 S.runWithAST("", Added,
201 [&](Expected<InputsAndAST> AST) { EXPECT_TRUE(bool(AST)); });
202 S.runWithPreamble("", Added, TUScheduler::Stale,
203 [&](Expected<InputsAndPreamble> Preamble) {
204 EXPECT_TRUE(bool(Preamble));
205 });
206 S.remove(Added);
207
208 // Assert that all operations fail after removing the file.
209 S.runWithAST("", Added,
210 [&](Expected<InputsAndAST> AST) { EXPECT_ERROR(AST); });
211 S.runWithPreamble("", Added, TUScheduler::Stale,
212 [&](Expected<InputsAndPreamble> Preamble) {
213 ASSERT_FALSE(bool(Preamble));
214 llvm::consumeError(Preamble.takeError());
215 });
216 // remove() shouldn't crash on missing files.
217 S.remove(Added);
218 }
219
TEST_F(TUSchedulerTests,WantDiagnostics)220 TEST_F(TUSchedulerTests, WantDiagnostics) {
221 std::atomic<int> CallbackCount(0);
222 {
223 // To avoid a racy test, don't allow tasks to actually run on the worker
224 // thread until we've scheduled them all.
225 Notification Ready;
226 TUScheduler S(CDB, optsForTest(), captureDiags());
227 auto Path = testPath("foo.cpp");
228 updateWithDiags(S, Path, "", WantDiagnostics::Yes,
229 [&](std::vector<Diag>) { Ready.wait(); });
230 updateWithDiags(S, Path, "request diags", WantDiagnostics::Yes,
231 [&](std::vector<Diag>) { ++CallbackCount; });
232 updateWithDiags(S, Path, "auto (clobbered)", WantDiagnostics::Auto,
233 [&](std::vector<Diag>) {
234 ADD_FAILURE()
235 << "auto should have been cancelled by auto";
236 });
237 updateWithDiags(S, Path, "request no diags", WantDiagnostics::No,
238 [&](std::vector<Diag>) {
239 ADD_FAILURE() << "no diags should not be called back";
240 });
241 updateWithDiags(S, Path, "auto (produces)", WantDiagnostics::Auto,
242 [&](std::vector<Diag>) { ++CallbackCount; });
243 Ready.notify();
244
245 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
246 }
247 EXPECT_EQ(2, CallbackCount);
248 }
249
TEST_F(TUSchedulerTests,Debounce)250 TEST_F(TUSchedulerTests, Debounce) {
251 std::atomic<int> CallbackCount(0);
252 {
253 auto Opts = optsForTest();
254 Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::seconds(1));
255 TUScheduler S(CDB, Opts, captureDiags());
256 // FIXME: we could probably use timeouts lower than 1 second here.
257 auto Path = testPath("foo.cpp");
258 updateWithDiags(S, Path, "auto (debounced)", WantDiagnostics::Auto,
259 [&](std::vector<Diag>) {
260 ADD_FAILURE()
261 << "auto should have been debounced and canceled";
262 });
263 std::this_thread::sleep_for(std::chrono::milliseconds(200));
264 updateWithDiags(S, Path, "auto (timed out)", WantDiagnostics::Auto,
265 [&](std::vector<Diag>) { ++CallbackCount; });
266 std::this_thread::sleep_for(std::chrono::seconds(2));
267 updateWithDiags(S, Path, "auto (shut down)", WantDiagnostics::Auto,
268 [&](std::vector<Diag>) { ++CallbackCount; });
269
270 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
271 }
272 EXPECT_EQ(2, CallbackCount);
273 }
274
TEST_F(TUSchedulerTests,Cancellation)275 TEST_F(TUSchedulerTests, Cancellation) {
276 // We have the following update/read sequence
277 // U0
278 // U1(WantDiags=Yes) <-- cancelled
279 // R1 <-- cancelled
280 // U2(WantDiags=Yes) <-- cancelled
281 // R2A <-- cancelled
282 // R2B
283 // U3(WantDiags=Yes)
284 // R3 <-- cancelled
285 std::vector<std::string> DiagsSeen, ReadsSeen, ReadsCanceled;
286 {
287 Notification Proceed; // Ensure we schedule everything.
288 TUScheduler S(CDB, optsForTest(), captureDiags());
289 auto Path = testPath("foo.cpp");
290 // Helper to schedule a named update and return a function to cancel it.
291 auto Update = [&](std::string ID) -> Canceler {
292 auto T = cancelableTask();
293 WithContext C(std::move(T.first));
294 updateWithDiags(
295 S, Path, "//" + ID, WantDiagnostics::Yes,
296 [&, ID](std::vector<Diag> Diags) { DiagsSeen.push_back(ID); });
297 return std::move(T.second);
298 };
299 // Helper to schedule a named read and return a function to cancel it.
300 auto Read = [&](std::string ID) -> Canceler {
301 auto T = cancelableTask();
302 WithContext C(std::move(T.first));
303 S.runWithAST(ID, Path, [&, ID](llvm::Expected<InputsAndAST> E) {
304 if (auto Err = E.takeError()) {
305 if (Err.isA<CancelledError>()) {
306 ReadsCanceled.push_back(ID);
307 consumeError(std::move(Err));
308 } else {
309 ADD_FAILURE() << "Non-cancelled error for " << ID << ": "
310 << llvm::toString(std::move(Err));
311 }
312 } else {
313 ReadsSeen.push_back(ID);
314 }
315 });
316 return std::move(T.second);
317 };
318
319 updateWithCallback(S, Path, "", WantDiagnostics::Yes,
320 [&]() { Proceed.wait(); });
321 // The second parens indicate cancellation, where present.
322 Update("U1")();
323 Read("R1")();
324 Update("U2")();
325 Read("R2A")();
326 Read("R2B");
327 Update("U3");
328 Read("R3")();
329 Proceed.notify();
330
331 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
332 }
333 EXPECT_THAT(DiagsSeen, ElementsAre("U2", "U3"))
334 << "U1 and all dependent reads were cancelled. "
335 "U2 has a dependent read R2A. "
336 "U3 was not cancelled.";
337 EXPECT_THAT(ReadsSeen, ElementsAre("R2B"))
338 << "All reads other than R2B were cancelled";
339 EXPECT_THAT(ReadsCanceled, ElementsAre("R1", "R2A", "R3"))
340 << "All reads other than R2B were cancelled";
341 }
342
TEST_F(TUSchedulerTests,InvalidationNoCrash)343 TEST_F(TUSchedulerTests, InvalidationNoCrash) {
344 auto Path = testPath("foo.cpp");
345 TUScheduler S(CDB, optsForTest(), captureDiags());
346
347 Notification StartedRunning;
348 Notification ScheduledChange;
349 // We expect invalidation logic to not crash by trying to invalidate a running
350 // request.
351 S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);
352 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
353 S.runWithAST(
354 "invalidatable-but-running", Path,
355 [&](llvm::Expected<InputsAndAST> AST) {
356 StartedRunning.notify();
357 ScheduledChange.wait();
358 ASSERT_TRUE(bool(AST));
359 },
360 TUScheduler::InvalidateOnUpdate);
361 StartedRunning.wait();
362 S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);
363 ScheduledChange.notify();
364 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
365 }
366
TEST_F(TUSchedulerTests,Invalidation)367 TEST_F(TUSchedulerTests, Invalidation) {
368 auto Path = testPath("foo.cpp");
369 TUScheduler S(CDB, optsForTest(), captureDiags());
370 std::atomic<int> Builds(0), Actions(0);
371
372 Notification Start;
373 updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
374 ++Builds;
375 Start.wait();
376 });
377 S.runWithAST(
378 "invalidatable", Path,
379 [&](llvm::Expected<InputsAndAST> AST) {
380 ++Actions;
381 EXPECT_FALSE(bool(AST));
382 llvm::Error E = AST.takeError();
383 EXPECT_TRUE(E.isA<CancelledError>());
384 handleAllErrors(std::move(E), [&](const CancelledError &E) {
385 EXPECT_EQ(E.Reason, static_cast<int>(ErrorCode::ContentModified));
386 });
387 },
388 TUScheduler::InvalidateOnUpdate);
389 S.runWithAST(
390 "not-invalidatable", Path,
391 [&](llvm::Expected<InputsAndAST> AST) {
392 ++Actions;
393 EXPECT_TRUE(bool(AST));
394 },
395 TUScheduler::NoInvalidation);
396 updateWithDiags(S, Path, "b", WantDiagnostics::Auto, [&](std::vector<Diag>) {
397 ++Builds;
398 ADD_FAILURE() << "Shouldn't build, all dependents invalidated";
399 });
400 S.runWithAST(
401 "invalidatable", Path,
402 [&](llvm::Expected<InputsAndAST> AST) {
403 ++Actions;
404 EXPECT_FALSE(bool(AST));
405 llvm::Error E = AST.takeError();
406 EXPECT_TRUE(E.isA<CancelledError>());
407 consumeError(std::move(E));
408 },
409 TUScheduler::InvalidateOnUpdate);
410 updateWithDiags(S, Path, "c", WantDiagnostics::Auto,
411 [&](std::vector<Diag>) { ++Builds; });
412 S.runWithAST(
413 "invalidatable", Path,
414 [&](llvm::Expected<InputsAndAST> AST) {
415 ++Actions;
416 EXPECT_TRUE(bool(AST)) << "Shouldn't be invalidated, no update follows";
417 },
418 TUScheduler::InvalidateOnUpdate);
419 Start.notify();
420 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
421
422 EXPECT_EQ(2, Builds.load()) << "Middle build should be skipped";
423 EXPECT_EQ(4, Actions.load()) << "All actions should run (some with error)";
424 }
425
426 // We don't invalidate requests for updates that don't change the file content.
427 // These are mostly "refresh this file" events synthesized inside clangd itself.
428 // (Usually the AST rebuild is elided after verifying that all inputs are
429 // unchanged, but invalidation decisions happen earlier and so independently).
430 // See https://github.com/clangd/clangd/issues/620
TEST_F(TUSchedulerTests,InvalidationUnchanged)431 TEST_F(TUSchedulerTests, InvalidationUnchanged) {
432 auto Path = testPath("foo.cpp");
433 TUScheduler S(CDB, optsForTest(), captureDiags());
434 std::atomic<int> Actions(0);
435
436 Notification Start;
437 updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
438 Start.wait();
439 });
440 S.runWithAST(
441 "invalidatable", Path,
442 [&](llvm::Expected<InputsAndAST> AST) {
443 ++Actions;
444 EXPECT_TRUE(bool(AST))
445 << "Should not invalidate based on an update with same content: "
446 << llvm::toString(AST.takeError());
447 },
448 TUScheduler::InvalidateOnUpdate);
449 updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
450 ADD_FAILURE() << "Shouldn't build, identical to previous";
451 });
452 Start.notify();
453 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
454
455 EXPECT_EQ(1, Actions.load()) << "All actions should run";
456 }
457
TEST_F(TUSchedulerTests,ManyUpdates)458 TEST_F(TUSchedulerTests, ManyUpdates) {
459 const int FilesCount = 3;
460 const int UpdatesPerFile = 10;
461
462 std::mutex Mut;
463 int TotalASTReads = 0;
464 int TotalPreambleReads = 0;
465 int TotalUpdates = 0;
466 llvm::StringMap<int> LatestDiagVersion;
467
468 // Run TUScheduler and collect some stats.
469 {
470 auto Opts = optsForTest();
471 Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::milliseconds(50));
472 TUScheduler S(CDB, Opts, captureDiags());
473
474 std::vector<std::string> Files;
475 for (int I = 0; I < FilesCount; ++I) {
476 std::string Name = "foo" + std::to_string(I) + ".cpp";
477 Files.push_back(testPath(Name));
478 this->FS.Files[Files.back()] = "";
479 }
480
481 StringRef Contents1 = R"cpp(int a;)cpp";
482 StringRef Contents2 = R"cpp(int main() { return 1; })cpp";
483 StringRef Contents3 = R"cpp(int a; int b; int sum() { return a + b; })cpp";
484
485 StringRef AllContents[] = {Contents1, Contents2, Contents3};
486 const int AllContentsSize = 3;
487
488 // Scheduler may run tasks asynchronously, but should propagate the
489 // context. We stash a nonce in the context, and verify it in the task.
490 static Key<int> NonceKey;
491 int Nonce = 0;
492
493 for (int FileI = 0; FileI < FilesCount; ++FileI) {
494 for (int UpdateI = 0; UpdateI < UpdatesPerFile; ++UpdateI) {
495 auto Contents = AllContents[(FileI + UpdateI) % AllContentsSize];
496
497 auto File = Files[FileI];
498 auto Inputs = getInputs(File, Contents.str());
499 {
500 WithContextValue WithNonce(NonceKey, ++Nonce);
501 Inputs.Version = std::to_string(UpdateI);
502 updateWithDiags(
503 S, File, Inputs, WantDiagnostics::Auto,
504 [File, Nonce, Version(Inputs.Version), &Mut, &TotalUpdates,
505 &LatestDiagVersion](std::vector<Diag>) {
506 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
507 EXPECT_EQ(File, boundPath());
508
509 std::lock_guard<std::mutex> Lock(Mut);
510 ++TotalUpdates;
511 EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());
512 // Make sure Diags are for a newer version.
513 auto It = LatestDiagVersion.try_emplace(File, -1);
514 const int PrevVersion = It.first->second;
515 int CurVersion;
516 ASSERT_TRUE(llvm::to_integer(Version, CurVersion, 10));
517 EXPECT_LT(PrevVersion, CurVersion);
518 It.first->getValue() = CurVersion;
519 });
520 }
521 {
522 WithContextValue WithNonce(NonceKey, ++Nonce);
523 S.runWithAST(
524 "CheckAST", File,
525 [File, Inputs, Nonce, &Mut,
526 &TotalASTReads](Expected<InputsAndAST> AST) {
527 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
528 EXPECT_EQ(File, boundPath());
529
530 ASSERT_TRUE((bool)AST);
531 EXPECT_EQ(AST->Inputs.Contents, Inputs.Contents);
532 EXPECT_EQ(AST->Inputs.Version, Inputs.Version);
533 EXPECT_EQ(AST->AST.version(), Inputs.Version);
534
535 std::lock_guard<std::mutex> Lock(Mut);
536 ++TotalASTReads;
537 EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());
538 });
539 }
540
541 {
542 WithContextValue WithNonce(NonceKey, ++Nonce);
543 S.runWithPreamble(
544 "CheckPreamble", File, TUScheduler::Stale,
545 [File, Inputs, Nonce, &Mut,
546 &TotalPreambleReads](Expected<InputsAndPreamble> Preamble) {
547 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
548 EXPECT_EQ(File, boundPath());
549
550 ASSERT_TRUE((bool)Preamble);
551 EXPECT_EQ(Preamble->Contents, Inputs.Contents);
552
553 std::lock_guard<std::mutex> Lock(Mut);
554 ++TotalPreambleReads;
555 EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());
556 });
557 }
558 }
559 }
560 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
561 } // TUScheduler destructor waits for all operations to finish.
562
563 std::lock_guard<std::mutex> Lock(Mut);
564 // Updates might get coalesced in preamble thread and result in dropping
565 // diagnostics for intermediate snapshots.
566 EXPECT_GE(TotalUpdates, FilesCount);
567 EXPECT_LE(TotalUpdates, FilesCount * UpdatesPerFile);
568 // We should receive diags for last update.
569 for (const auto &Entry : LatestDiagVersion)
570 EXPECT_EQ(Entry.second, UpdatesPerFile - 1);
571 EXPECT_EQ(TotalASTReads, FilesCount * UpdatesPerFile);
572 EXPECT_EQ(TotalPreambleReads, FilesCount * UpdatesPerFile);
573 }
574
TEST_F(TUSchedulerTests,EvictedAST)575 TEST_F(TUSchedulerTests, EvictedAST) {
576 std::atomic<int> BuiltASTCounter(0);
577 auto Opts = optsForTest();
578 Opts.AsyncThreadsCount = 1;
579 Opts.RetentionPolicy.MaxRetainedASTs = 2;
580 trace::TestTracer Tracer;
581 TUScheduler S(CDB, Opts);
582
583 llvm::StringLiteral SourceContents = R"cpp(
584 int* a;
585 double* b = a;
586 )cpp";
587 llvm::StringLiteral OtherSourceContents = R"cpp(
588 int* a;
589 double* b = a + 0;
590 )cpp";
591
592 auto Foo = testPath("foo.cpp");
593 auto Bar = testPath("bar.cpp");
594 auto Baz = testPath("baz.cpp");
595
596 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
597 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
598 // Build one file in advance. We will not access it later, so it will be the
599 // one that the cache will evict.
600 updateWithCallback(S, Foo, SourceContents, WantDiagnostics::Yes,
601 [&BuiltASTCounter]() { ++BuiltASTCounter; });
602 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
603 ASSERT_EQ(BuiltASTCounter.load(), 1);
604 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
605 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));
606
607 // Build two more files. Since we can retain only 2 ASTs, these should be
608 // the ones we see in the cache later.
609 updateWithCallback(S, Bar, SourceContents, WantDiagnostics::Yes,
610 [&BuiltASTCounter]() { ++BuiltASTCounter; });
611 updateWithCallback(S, Baz, SourceContents, WantDiagnostics::Yes,
612 [&BuiltASTCounter]() { ++BuiltASTCounter; });
613 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
614 ASSERT_EQ(BuiltASTCounter.load(), 3);
615 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
616 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(2));
617
618 // Check only the last two ASTs are retained.
619 ASSERT_THAT(S.getFilesWithCachedAST(), UnorderedElementsAre(Bar, Baz));
620
621 // Access the old file again.
622 updateWithCallback(S, Foo, OtherSourceContents, WantDiagnostics::Yes,
623 [&BuiltASTCounter]() { ++BuiltASTCounter; });
624 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
625 ASSERT_EQ(BuiltASTCounter.load(), 4);
626 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
627 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));
628
629 // Check the AST for foo.cpp is retained now and one of the others got
630 // evicted.
631 EXPECT_THAT(S.getFilesWithCachedAST(),
632 UnorderedElementsAre(Foo, AnyOf(Bar, Baz)));
633 }
634
635 // We send "empty" changes to TUScheduler when we think some external event
636 // *might* have invalidated current state (e.g. a header was edited).
637 // Verify that this doesn't evict our cache entries.
TEST_F(TUSchedulerTests,NoopChangesDontThrashCache)638 TEST_F(TUSchedulerTests, NoopChangesDontThrashCache) {
639 auto Opts = optsForTest();
640 Opts.RetentionPolicy.MaxRetainedASTs = 1;
641 TUScheduler S(CDB, Opts);
642
643 auto Foo = testPath("foo.cpp");
644 auto FooInputs = getInputs(Foo, "int x=1;");
645 auto Bar = testPath("bar.cpp");
646 auto BarInputs = getInputs(Bar, "int x=2;");
647
648 // After opening Foo then Bar, AST cache contains Bar.
649 S.update(Foo, FooInputs, WantDiagnostics::Auto);
650 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
651 S.update(Bar, BarInputs, WantDiagnostics::Auto);
652 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
653 ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));
654
655 // Any number of no-op updates to Foo don't dislodge Bar from the cache.
656 S.update(Foo, FooInputs, WantDiagnostics::Auto);
657 S.update(Foo, FooInputs, WantDiagnostics::Auto);
658 S.update(Foo, FooInputs, WantDiagnostics::Auto);
659 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
660 ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));
661 // In fact each file has been built only once.
662 ASSERT_EQ(S.fileStats().lookup(Foo).ASTBuilds, 1u);
663 ASSERT_EQ(S.fileStats().lookup(Bar).ASTBuilds, 1u);
664 }
665
TEST_F(TUSchedulerTests,EmptyPreamble)666 TEST_F(TUSchedulerTests, EmptyPreamble) {
667 TUScheduler S(CDB, optsForTest());
668
669 auto Foo = testPath("foo.cpp");
670 auto Header = testPath("foo.h");
671
672 FS.Files[Header] = "void foo()";
673 FS.Timestamps[Header] = time_t(0);
674 auto WithPreamble = R"cpp(
675 #include "foo.h"
676 int main() {}
677 )cpp";
678 auto WithEmptyPreamble = R"cpp(int main() {})cpp";
679 S.update(Foo, getInputs(Foo, WithPreamble), WantDiagnostics::Auto);
680 S.runWithPreamble(
681 "getNonEmptyPreamble", Foo, TUScheduler::Stale,
682 [&](Expected<InputsAndPreamble> Preamble) {
683 // We expect to get a non-empty preamble.
684 EXPECT_GT(
685 cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,
686 0u);
687 });
688 // Wait while the preamble is being built.
689 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
690
691 // Update the file which results in an empty preamble.
692 S.update(Foo, getInputs(Foo, WithEmptyPreamble), WantDiagnostics::Auto);
693 // Wait while the preamble is being built.
694 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
695 S.runWithPreamble(
696 "getEmptyPreamble", Foo, TUScheduler::Stale,
697 [&](Expected<InputsAndPreamble> Preamble) {
698 // We expect to get an empty preamble.
699 EXPECT_EQ(
700 cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,
701 0u);
702 });
703 }
704
TEST_F(TUSchedulerTests,ASTSignalsSmokeTests)705 TEST_F(TUSchedulerTests, ASTSignalsSmokeTests) {
706 TUScheduler S(CDB, optsForTest());
707 auto Foo = testPath("foo.cpp");
708 auto Header = testPath("foo.h");
709
710 FS.Files[Header] = "namespace tar { int foo(); }";
711 const char *Contents = R"cpp(
712 #include "foo.h"
713 namespace ns {
714 int func() {
715 return tar::foo());
716 }
717 } // namespace ns
718 )cpp";
719 // Update the file which results in an empty preamble.
720 S.update(Foo, getInputs(Foo, Contents), WantDiagnostics::Yes);
721 // Wait while the preamble is being built.
722 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
723 Notification TaskRun;
724 S.runWithPreamble(
725 "ASTSignals", Foo, TUScheduler::Stale,
726 [&](Expected<InputsAndPreamble> IP) {
727 ASSERT_FALSE(!IP);
728 std::vector<std::pair<StringRef, int>> NS;
729 for (const auto &P : IP->Signals->RelatedNamespaces)
730 NS.emplace_back(P.getKey(), P.getValue());
731 EXPECT_THAT(NS,
732 UnorderedElementsAre(Pair("ns::", 1), Pair("tar::", 1)));
733
734 std::vector<std::pair<SymbolID, int>> Sym;
735 for (const auto &P : IP->Signals->ReferencedSymbols)
736 Sym.emplace_back(P.getFirst(), P.getSecond());
737 EXPECT_THAT(Sym, UnorderedElementsAre(Pair(ns("tar").ID, 1),
738 Pair(ns("ns").ID, 1),
739 Pair(func("tar::foo").ID, 1),
740 Pair(func("ns::func").ID, 1)));
741 TaskRun.notify();
742 });
743 TaskRun.wait();
744 }
745
TEST_F(TUSchedulerTests,RunWaitsForPreamble)746 TEST_F(TUSchedulerTests, RunWaitsForPreamble) {
747 // Testing strategy: we update the file and schedule a few preamble reads at
748 // the same time. All reads should get the same non-null preamble.
749 TUScheduler S(CDB, optsForTest());
750 auto Foo = testPath("foo.cpp");
751 auto NonEmptyPreamble = R"cpp(
752 #define FOO 1
753 #define BAR 2
754
755 int main() {}
756 )cpp";
757 constexpr int ReadsToSchedule = 10;
758 std::mutex PreamblesMut;
759 std::vector<const void *> Preambles(ReadsToSchedule, nullptr);
760 S.update(Foo, getInputs(Foo, NonEmptyPreamble), WantDiagnostics::Auto);
761 for (int I = 0; I < ReadsToSchedule; ++I) {
762 S.runWithPreamble(
763 "test", Foo, TUScheduler::Stale,
764 [I, &PreamblesMut, &Preambles](Expected<InputsAndPreamble> IP) {
765 std::lock_guard<std::mutex> Lock(PreamblesMut);
766 Preambles[I] = cantFail(std::move(IP)).Preamble;
767 });
768 }
769 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
770 // Check all actions got the same non-null preamble.
771 std::lock_guard<std::mutex> Lock(PreamblesMut);
772 ASSERT_NE(Preambles[0], nullptr);
773 ASSERT_THAT(Preambles, Each(Preambles[0]));
774 }
775
TEST_F(TUSchedulerTests,NoopOnEmptyChanges)776 TEST_F(TUSchedulerTests, NoopOnEmptyChanges) {
777 TUScheduler S(CDB, optsForTest(), captureDiags());
778
779 auto Source = testPath("foo.cpp");
780 auto Header = testPath("foo.h");
781
782 FS.Files[Header] = "int a;";
783 FS.Timestamps[Header] = time_t(0);
784
785 std::string SourceContents = R"cpp(
786 #include "foo.h"
787 int b = a;
788 )cpp";
789
790 // Return value indicates if the updated callback was received.
791 auto DoUpdate = [&](std::string Contents) -> bool {
792 std::atomic<bool> Updated(false);
793 Updated = false;
794 updateWithDiags(S, Source, Contents, WantDiagnostics::Yes,
795 [&Updated](std::vector<Diag>) { Updated = true; });
796 bool UpdateFinished = S.blockUntilIdle(timeoutSeconds(10));
797 if (!UpdateFinished)
798 ADD_FAILURE() << "Updated has not finished in one second. Threading bug?";
799 return Updated;
800 };
801
802 // Test that subsequent updates with the same inputs do not cause rebuilds.
803 ASSERT_TRUE(DoUpdate(SourceContents));
804 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);
805 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 1u);
806 ASSERT_FALSE(DoUpdate(SourceContents));
807 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);
808 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 1u);
809
810 // Update to a header should cause a rebuild, though.
811 FS.Timestamps[Header] = time_t(1);
812 ASSERT_TRUE(DoUpdate(SourceContents));
813 ASSERT_FALSE(DoUpdate(SourceContents));
814 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 2u);
815 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 2u);
816
817 // Update to the contents should cause a rebuild.
818 SourceContents += "\nint c = b;";
819 ASSERT_TRUE(DoUpdate(SourceContents));
820 ASSERT_FALSE(DoUpdate(SourceContents));
821 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 3u);
822 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 2u);
823
824 // Update to the compile commands should also cause a rebuild.
825 CDB.ExtraClangFlags.push_back("-DSOMETHING");
826 ASSERT_TRUE(DoUpdate(SourceContents));
827 ASSERT_FALSE(DoUpdate(SourceContents));
828 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 4u);
829 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 3u);
830 }
831
832 // We rebuild if a completely missing header exists, but not if one is added
833 // on a higher-priority include path entry (for performance).
834 // (Previously we wouldn't automatically rebuild when files were added).
TEST_F(TUSchedulerTests,MissingHeader)835 TEST_F(TUSchedulerTests, MissingHeader) {
836 CDB.ExtraClangFlags.push_back("-I" + testPath("a"));
837 CDB.ExtraClangFlags.push_back("-I" + testPath("b"));
838 // Force both directories to exist so they don't get pruned.
839 FS.Files.try_emplace("a/__unused__");
840 FS.Files.try_emplace("b/__unused__");
841 TUScheduler S(CDB, optsForTest(), captureDiags());
842
843 auto Source = testPath("foo.cpp");
844 auto HeaderA = testPath("a/foo.h");
845 auto HeaderB = testPath("b/foo.h");
846
847 auto SourceContents = R"cpp(
848 #include "foo.h"
849 int c = b;
850 )cpp";
851
852 ParseInputs Inputs = getInputs(Source, SourceContents);
853 std::atomic<size_t> DiagCount(0);
854
855 // Update the source contents, which should trigger an initial build with
856 // the header file missing.
857 updateWithDiags(
858 S, Source, Inputs, WantDiagnostics::Yes,
859 [&DiagCount](std::vector<Diag> Diags) {
860 ++DiagCount;
861 EXPECT_THAT(Diags,
862 ElementsAre(Field(&Diag::Message, "'foo.h' file not found"),
863 Field(&Diag::Message,
864 "use of undeclared identifier 'b'")));
865 });
866 S.blockUntilIdle(timeoutSeconds(10));
867
868 FS.Files[HeaderB] = "int b;";
869 FS.Timestamps[HeaderB] = time_t(1);
870
871 // The addition of the missing header file triggers a rebuild, no errors.
872 updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,
873 [&DiagCount](std::vector<Diag> Diags) {
874 ++DiagCount;
875 EXPECT_THAT(Diags, IsEmpty());
876 });
877
878 // Ensure previous assertions are done before we touch the FS again.
879 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
880 // Add the high-priority header file, which should reintroduce the error.
881 FS.Files[HeaderA] = "int a;";
882 FS.Timestamps[HeaderA] = time_t(1);
883
884 // This isn't detected: we don't stat a/foo.h to validate the preamble.
885 updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,
886 [&DiagCount](std::vector<Diag> Diags) {
887 ++DiagCount;
888 ADD_FAILURE()
889 << "Didn't expect new diagnostics when adding a/foo.h";
890 });
891
892 // Forcing the reload should should cause a rebuild.
893 Inputs.ForceRebuild = true;
894 updateWithDiags(
895 S, Source, Inputs, WantDiagnostics::Yes,
896 [&DiagCount](std::vector<Diag> Diags) {
897 ++DiagCount;
898 ElementsAre(Field(&Diag::Message, "use of undeclared identifier 'b'"));
899 });
900
901 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
902 EXPECT_EQ(DiagCount, 3U);
903 }
904
TEST_F(TUSchedulerTests,NoChangeDiags)905 TEST_F(TUSchedulerTests, NoChangeDiags) {
906 trace::TestTracer Tracer;
907 TUScheduler S(CDB, optsForTest(), captureDiags());
908
909 auto FooCpp = testPath("foo.cpp");
910 const auto *Contents = "int a; int b;";
911
912 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));
913 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(0));
914 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
915 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
916 updateWithDiags(
917 S, FooCpp, Contents, WantDiagnostics::No,
918 [](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
919 S.runWithAST("touchAST", FooCpp, [](Expected<InputsAndAST> IA) {
920 // Make sure the AST was actually built.
921 cantFail(std::move(IA));
922 });
923 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
924 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));
925 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(1));
926
927 // Even though the inputs didn't change and AST can be reused, we need to
928 // report the diagnostics, as they were not reported previously.
929 std::atomic<bool> SeenDiags(false);
930 updateWithDiags(S, FooCpp, Contents, WantDiagnostics::Auto,
931 [&](std::vector<Diag>) { SeenDiags = true; });
932 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
933 ASSERT_TRUE(SeenDiags);
934 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(1));
935 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
936
937 // Subsequent request does not get any diagnostics callback because the same
938 // diags have previously been reported and the inputs didn't change.
939 updateWithDiags(
940 S, FooCpp, Contents, WantDiagnostics::Auto,
941 [&](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
942 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
943 }
944
TEST_F(TUSchedulerTests,Run)945 TEST_F(TUSchedulerTests, Run) {
946 for (bool Sync : {false, true}) {
947 auto Opts = optsForTest();
948 if (Sync)
949 Opts.AsyncThreadsCount = 0;
950 TUScheduler S(CDB, Opts);
951 std::atomic<int> Counter(0);
952 S.run("add 1", /*Path=*/"", [&] { ++Counter; });
953 S.run("add 2", /*Path=*/"", [&] { Counter += 2; });
954 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
955 EXPECT_EQ(Counter.load(), 3);
956
957 Notification TaskRun;
958 Key<int> TestKey;
959 WithContextValue CtxWithKey(TestKey, 10);
960 const char *Path = "somepath";
961 S.run("props context", Path, [&] {
962 EXPECT_EQ(Context::current().getExisting(TestKey), 10);
963 EXPECT_EQ(Path, boundPath());
964 TaskRun.notify();
965 });
966 TaskRun.wait();
967 }
968 }
969
TEST_F(TUSchedulerTests,TUStatus)970 TEST_F(TUSchedulerTests, TUStatus) {
971 class CaptureTUStatus : public ClangdServer::Callbacks {
972 public:
973 void onFileUpdated(PathRef File, const TUStatus &Status) override {
974 auto ASTAction = Status.ASTActivity.K;
975 auto PreambleAction = Status.PreambleActivity;
976 std::lock_guard<std::mutex> Lock(Mutex);
977 // Only push the action if it has changed. Since TUStatus can be published
978 // from either Preamble or AST thread and when one changes the other stays
979 // the same.
980 // Note that this can result in missing some updates when something other
981 // than action kind changes, e.g. when AST is built/reused the action kind
982 // stays as Building.
983 if (ASTActions.empty() || ASTActions.back() != ASTAction)
984 ASTActions.push_back(ASTAction);
985 if (PreambleActions.empty() || PreambleActions.back() != PreambleAction)
986 PreambleActions.push_back(PreambleAction);
987 }
988
989 std::vector<PreambleAction> preambleStatuses() {
990 std::lock_guard<std::mutex> Lock(Mutex);
991 return PreambleActions;
992 }
993
994 std::vector<ASTAction::Kind> astStatuses() {
995 std::lock_guard<std::mutex> Lock(Mutex);
996 return ASTActions;
997 }
998
999 private:
1000 std::mutex Mutex;
1001 std::vector<ASTAction::Kind> ASTActions;
1002 std::vector<PreambleAction> PreambleActions;
1003 } CaptureTUStatus;
1004 MockFS FS;
1005 MockCompilationDatabase CDB;
1006 ClangdServer Server(CDB, FS, ClangdServer::optsForTest(), &CaptureTUStatus);
1007 Annotations Code("int m^ain () {}");
1008
1009 // We schedule the following tasks in the queue:
1010 // [Update] [GoToDefinition]
1011 Server.addDocument(testPath("foo.cpp"), Code.code(), "1",
1012 WantDiagnostics::Auto);
1013 ASSERT_TRUE(Server.blockUntilIdleForTest());
1014 Server.locateSymbolAt(testPath("foo.cpp"), Code.point(),
1015 [](Expected<std::vector<LocatedSymbol>> Result) {
1016 ASSERT_TRUE((bool)Result);
1017 });
1018 ASSERT_TRUE(Server.blockUntilIdleForTest());
1019
1020 EXPECT_THAT(CaptureTUStatus.preambleStatuses(),
1021 ElementsAre(
1022 // PreambleThread starts idle, as the update is first handled
1023 // by ASTWorker.
1024 PreambleAction::Idle,
1025 // Then it starts building first preamble and releases that to
1026 // ASTWorker.
1027 PreambleAction::Building,
1028 // Then goes idle and stays that way as we don't receive any
1029 // more update requests.
1030 PreambleAction::Idle));
1031 EXPECT_THAT(CaptureTUStatus.astStatuses(),
1032 ElementsAre(
1033 // Starts handling the update action and blocks until the
1034 // first preamble is built.
1035 ASTAction::RunningAction,
1036 // Afterwqards it builds an AST for that preamble to publish
1037 // diagnostics.
1038 ASTAction::Building,
1039 // Then goes idle.
1040 ASTAction::Idle,
1041 // Afterwards we start executing go-to-def.
1042 ASTAction::RunningAction,
1043 // Then go idle.
1044 ASTAction::Idle));
1045 }
1046
TEST_F(TUSchedulerTests,CommandLineErrors)1047 TEST_F(TUSchedulerTests, CommandLineErrors) {
1048 // We should see errors from command-line parsing inside the main file.
1049 CDB.ExtraClangFlags = {"-fsome-unknown-flag"};
1050
1051 // (!) 'Ready' must live longer than TUScheduler.
1052 Notification Ready;
1053
1054 TUScheduler S(CDB, optsForTest(), captureDiags());
1055 std::vector<Diag> Diagnostics;
1056 updateWithDiags(S, testPath("foo.cpp"), "void test() {}",
1057 WantDiagnostics::Yes, [&](std::vector<Diag> D) {
1058 Diagnostics = std::move(D);
1059 Ready.notify();
1060 });
1061 Ready.wait();
1062
1063 EXPECT_THAT(
1064 Diagnostics,
1065 ElementsAre(AllOf(
1066 Field(&Diag::ID, Eq(diag::err_drv_unknown_argument)),
1067 Field(&Diag::Name, Eq("drv_unknown_argument")),
1068 Field(&Diag::Message, "unknown argument: '-fsome-unknown-flag'"))));
1069 }
1070
TEST_F(TUSchedulerTests,CommandLineWarnings)1071 TEST_F(TUSchedulerTests, CommandLineWarnings) {
1072 // We should not see warnings from command-line parsing.
1073 CDB.ExtraClangFlags = {"-Wsome-unknown-warning"};
1074
1075 // (!) 'Ready' must live longer than TUScheduler.
1076 Notification Ready;
1077
1078 TUScheduler S(CDB, optsForTest(), captureDiags());
1079 std::vector<Diag> Diagnostics;
1080 updateWithDiags(S, testPath("foo.cpp"), "void test() {}",
1081 WantDiagnostics::Yes, [&](std::vector<Diag> D) {
1082 Diagnostics = std::move(D);
1083 Ready.notify();
1084 });
1085 Ready.wait();
1086
1087 EXPECT_THAT(Diagnostics, IsEmpty());
1088 }
1089
TEST(DebouncePolicy,Compute)1090 TEST(DebouncePolicy, Compute) {
1091 namespace c = std::chrono;
1092 DebouncePolicy::clock::duration History[] = {
1093 c::seconds(0),
1094 c::seconds(5),
1095 c::seconds(10),
1096 c::seconds(20),
1097 };
1098 DebouncePolicy Policy;
1099 Policy.Min = c::seconds(3);
1100 Policy.Max = c::seconds(25);
1101 // Call Policy.compute(History) and return seconds as a float.
1102 auto Compute = [&](llvm::ArrayRef<DebouncePolicy::clock::duration> History) {
1103 return c::duration_cast<c::duration<float, c::seconds::period>>(
1104 Policy.compute(History))
1105 .count();
1106 };
1107 EXPECT_NEAR(10, Compute(History), 0.01) << "(upper) median = 10";
1108 Policy.RebuildRatio = 1.5;
1109 EXPECT_NEAR(15, Compute(History), 0.01) << "median = 10, ratio = 1.5";
1110 Policy.RebuildRatio = 3;
1111 EXPECT_NEAR(25, Compute(History), 0.01) << "constrained by max";
1112 Policy.RebuildRatio = 0;
1113 EXPECT_NEAR(3, Compute(History), 0.01) << "constrained by min";
1114 EXPECT_NEAR(25, Compute({}), 0.01) << "no history -> max";
1115 }
1116
TEST_F(TUSchedulerTests,AsyncPreambleThread)1117 TEST_F(TUSchedulerTests, AsyncPreambleThread) {
1118 // Blocks preamble thread while building preamble with \p BlockVersion until
1119 // \p N is notified.
1120 class BlockPreambleThread : public ParsingCallbacks {
1121 public:
1122 BlockPreambleThread(llvm::StringRef BlockVersion, Notification &N)
1123 : BlockVersion(BlockVersion), N(N) {}
1124 void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
1125 std::shared_ptr<clang::Preprocessor> PP,
1126 const CanonicalIncludes &) override {
1127 if (Version == BlockVersion)
1128 N.wait();
1129 }
1130
1131 private:
1132 llvm::StringRef BlockVersion;
1133 Notification &N;
1134 };
1135
1136 static constexpr llvm::StringLiteral InputsV0 = "v0";
1137 static constexpr llvm::StringLiteral InputsV1 = "v1";
1138 Notification Ready;
1139 TUScheduler S(CDB, optsForTest(),
1140 std::make_unique<BlockPreambleThread>(InputsV1, Ready));
1141
1142 Path File = testPath("foo.cpp");
1143 auto PI = getInputs(File, "");
1144 PI.Version = InputsV0.str();
1145 S.update(File, PI, WantDiagnostics::Auto);
1146 S.blockUntilIdle(timeoutSeconds(10));
1147
1148 // Block preamble builds.
1149 PI.Version = InputsV1.str();
1150 // Issue second update which will block preamble thread.
1151 S.update(File, PI, WantDiagnostics::Auto);
1152
1153 Notification RunASTAction;
1154 // Issue an AST read, which shouldn't be blocked and see latest version of the
1155 // file.
1156 S.runWithAST("test", File, [&](Expected<InputsAndAST> AST) {
1157 ASSERT_TRUE(bool(AST));
1158 // Make sure preamble is built with stale inputs, but AST was built using
1159 // new ones.
1160 EXPECT_THAT(AST->AST.preambleVersion(), InputsV0);
1161 EXPECT_THAT(AST->Inputs.Version, InputsV1.str());
1162 RunASTAction.notify();
1163 });
1164 RunASTAction.wait();
1165 Ready.notify();
1166 }
1167
1168 // If a header file is missing from the CDB (or inferred using heuristics), and
1169 // it's included by another open file, then we parse it using that files flags.
TEST_F(TUSchedulerTests,IncluderCache)1170 TEST_F(TUSchedulerTests, IncluderCache) {
1171 static std::string Main = testPath("main.cpp"), Main2 = testPath("main2.cpp"),
1172 Main3 = testPath("main3.cpp"),
1173 NoCmd = testPath("no_cmd.h"),
1174 Unreliable = testPath("unreliable.h"),
1175 OK = testPath("ok.h"),
1176 NotIncluded = testPath("not_included.h");
1177 class NoHeadersCDB : public GlobalCompilationDatabase {
1178 llvm::Optional<tooling::CompileCommand>
1179 getCompileCommand(PathRef File) const override {
1180 if (File == NoCmd || File == NotIncluded)
1181 return llvm::None;
1182 auto Basic = getFallbackCommand(File);
1183 Basic.Heuristic.clear();
1184 if (File == Unreliable) {
1185 Basic.Heuristic = "not reliable";
1186 } else if (File == Main) {
1187 Basic.CommandLine.push_back("-DMAIN");
1188 } else if (File == Main2) {
1189 Basic.CommandLine.push_back("-DMAIN2");
1190 } else if (File == Main3) {
1191 Basic.CommandLine.push_back("-DMAIN3");
1192 }
1193 return Basic;
1194 }
1195 } CDB;
1196 TUScheduler S(CDB, optsForTest());
1197 auto GetFlags = [&](PathRef Header) {
1198 S.update(Header, getInputs(Header, ";"), WantDiagnostics::Yes);
1199 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1200 tooling::CompileCommand Cmd;
1201 S.runWithPreamble("GetFlags", Header, TUScheduler::StaleOrAbsent,
1202 [&](llvm::Expected<InputsAndPreamble> Inputs) {
1203 ASSERT_FALSE(!Inputs) << Inputs.takeError();
1204 Cmd = std::move(Inputs->Command);
1205 });
1206 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1207 return Cmd.CommandLine;
1208 };
1209
1210 for (const auto &Path : {NoCmd, Unreliable, OK, NotIncluded})
1211 FS.Files[Path] = ";";
1212
1213 // Initially these files have normal commands from the CDB.
1214 EXPECT_THAT(GetFlags(Main), Contains("-DMAIN")) << "sanity check";
1215 EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN"))) << "no includes yet";
1216
1217 // Now make Main include the others, and some should pick up its flags.
1218 const char *AllIncludes = R"cpp(
1219 #include "no_cmd.h"
1220 #include "ok.h"
1221 #include "unreliable.h"
1222 )cpp";
1223 S.update(Main, getInputs(Main, AllIncludes), WantDiagnostics::Yes);
1224 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1225 EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN"))
1226 << "Included from main file, has no own command";
1227 EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))
1228 << "Included from main file, own command is heuristic";
1229 EXPECT_THAT(GetFlags(OK), Not(Contains("-DMAIN")))
1230 << "Included from main file, but own command is used";
1231 EXPECT_THAT(GetFlags(NotIncluded), Not(Contains("-DMAIN")))
1232 << "Not included from main file";
1233
1234 // Open another file - it won't overwrite the associations with Main.
1235 std::string SomeIncludes = R"cpp(
1236 #include "no_cmd.h"
1237 #include "not_included.h"
1238 )cpp";
1239 S.update(Main2, getInputs(Main2, SomeIncludes), WantDiagnostics::Yes);
1240 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1241 EXPECT_THAT(GetFlags(NoCmd),
1242 AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))
1243 << "mainfile association is stable";
1244 EXPECT_THAT(GetFlags(NotIncluded),
1245 AllOf(Contains("-DMAIN2"), Not(Contains("-DMAIN"))))
1246 << "new headers are associated with new mainfile";
1247
1248 // Remove includes from main - this marks the associations as invalid but
1249 // doesn't actually remove them until another preamble claims them.
1250 S.update(Main, getInputs(Main, ""), WantDiagnostics::Yes);
1251 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1252 EXPECT_THAT(GetFlags(NoCmd),
1253 AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))
1254 << "mainfile association not updated yet!";
1255
1256 // Open yet another file - this time it claims the associations.
1257 S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
1258 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1259 EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))
1260 << "association invalidated and then claimed by main3";
1261 EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))
1262 << "association invalidated but not reclaimed";
1263 EXPECT_THAT(GetFlags(NotIncluded), Contains("-DMAIN2"))
1264 << "association still valid";
1265 }
1266
TEST_F(TUSchedulerTests,PreservesLastActiveFile)1267 TEST_F(TUSchedulerTests, PreservesLastActiveFile) {
1268 for (bool Sync : {false, true}) {
1269 auto Opts = optsForTest();
1270 if (Sync)
1271 Opts.AsyncThreadsCount = 0;
1272 TUScheduler S(CDB, Opts);
1273
1274 auto CheckNoFileActionsSeesLastActiveFile =
1275 [&](llvm::StringRef LastActiveFile) {
1276 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1277 std::atomic<int> Counter(0);
1278 // We only check for run and runQuick as runWithAST and
1279 // runWithPreamble is always bound to a file.
1280 S.run("run-UsesLastActiveFile", /*Path=*/"", [&] {
1281 ++Counter;
1282 EXPECT_EQ(LastActiveFile, boundPath());
1283 });
1284 S.runQuick("runQuick-UsesLastActiveFile", /*Path=*/"", [&] {
1285 ++Counter;
1286 EXPECT_EQ(LastActiveFile, boundPath());
1287 });
1288 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1289 EXPECT_EQ(2, Counter.load());
1290 };
1291
1292 // Check that we see no file initially
1293 CheckNoFileActionsSeesLastActiveFile("");
1294
1295 // Now check that every action scheduled with a particular file changes the
1296 // LastActiveFile.
1297 auto Path = testPath("run.cc");
1298 S.run(Path, Path, [] {});
1299 CheckNoFileActionsSeesLastActiveFile(Path);
1300
1301 Path = testPath("runQuick.cc");
1302 S.runQuick(Path, Path, [] {});
1303 CheckNoFileActionsSeesLastActiveFile(Path);
1304
1305 Path = testPath("runWithAST.cc");
1306 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1307 S.runWithAST(Path, Path, [](llvm::Expected<InputsAndAST> Inp) {
1308 EXPECT_TRUE(bool(Inp));
1309 });
1310 CheckNoFileActionsSeesLastActiveFile(Path);
1311
1312 Path = testPath("runWithPreamble.cc");
1313 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1314 S.runWithPreamble(
1315 Path, Path, TUScheduler::Stale,
1316 [](llvm::Expected<InputsAndPreamble> Inp) { EXPECT_TRUE(bool(Inp)); });
1317 CheckNoFileActionsSeesLastActiveFile(Path);
1318
1319 Path = testPath("update.cc");
1320 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1321 CheckNoFileActionsSeesLastActiveFile(Path);
1322
1323 // An update with the same contents should not change LastActiveFile.
1324 auto LastActive = Path;
1325 Path = testPath("runWithAST.cc");
1326 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1327 CheckNoFileActionsSeesLastActiveFile(LastActive);
1328 }
1329 }
1330 } // namespace
1331 } // namespace clangd
1332 } // namespace clang
1333