1 //===-- SourceCodeTests.cpp  ------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 #include "Annotations.h"
9 #include "Protocol.h"
10 #include "SourceCode.h"
11 #include "TestTU.h"
12 #include "support/Context.h"
13 #include "clang/Basic/LangOptions.h"
14 #include "clang/Basic/SourceLocation.h"
15 #include "clang/Basic/TokenKinds.h"
16 #include "clang/Format/Format.h"
17 #include "llvm/Support/Error.h"
18 #include "llvm/Support/raw_os_ostream.h"
19 #include "llvm/Testing/Support/Annotations.h"
20 #include "llvm/Testing/Support/Error.h"
21 #include "gmock/gmock.h"
22 #include "gtest/gtest.h"
23 #include <tuple>
24 
25 namespace clang {
26 namespace clangd {
27 namespace {
28 
29 using llvm::Failed;
30 using llvm::HasValue;
31 
32 MATCHER_P2(Pos, Line, Col, "") {
33   return arg.line == int(Line) && arg.character == int(Col);
34 }
35 
36 MATCHER_P(MacroName, Name, "") { return arg.Name == Name; }
37 
38 /// A helper to make tests easier to read.
position(int Line,int Character)39 Position position(int Line, int Character) {
40   Position Pos;
41   Pos.line = Line;
42   Pos.character = Character;
43   return Pos;
44 }
45 
TEST(SourceCodeTests,lspLength)46 TEST(SourceCodeTests, lspLength) {
47   EXPECT_EQ(lspLength(""), 0UL);
48   EXPECT_EQ(lspLength("ascii"), 5UL);
49   // BMP
50   EXPECT_EQ(lspLength("↓"), 1UL);
51   EXPECT_EQ(lspLength("¥"), 1UL);
52   // astral
53   EXPECT_EQ(lspLength("��"), 2UL);
54 
55   WithContextValue UTF8(kCurrentOffsetEncoding, OffsetEncoding::UTF8);
56   EXPECT_EQ(lspLength(""), 0UL);
57   EXPECT_EQ(lspLength("ascii"), 5UL);
58   // BMP
59   EXPECT_EQ(lspLength("↓"), 3UL);
60   EXPECT_EQ(lspLength("¥"), 2UL);
61   // astral
62   EXPECT_EQ(lspLength("��"), 4UL);
63 
64   WithContextValue UTF32(kCurrentOffsetEncoding, OffsetEncoding::UTF32);
65   EXPECT_EQ(lspLength(""), 0UL);
66   EXPECT_EQ(lspLength("ascii"), 5UL);
67   // BMP
68   EXPECT_EQ(lspLength("↓"), 1UL);
69   EXPECT_EQ(lspLength("¥"), 1UL);
70   // astral
71   EXPECT_EQ(lspLength("��"), 1UL);
72 }
73 
TEST(SourceCodeTests,lspLengthBadUTF8)74 TEST(SourceCodeTests, lspLengthBadUTF8) {
75   // Results are not well-defined if source file isn't valid UTF-8.
76   // However we shouldn't crash or return something totally wild.
77   const char *BadUTF8[] = {"\xa0", "\xff\xff\xff\xff\xff"};
78 
79   for (OffsetEncoding Encoding :
80        {OffsetEncoding::UTF8, OffsetEncoding::UTF16, OffsetEncoding::UTF32}) {
81     WithContextValue UTF32(kCurrentOffsetEncoding, Encoding);
82     for (const char *Bad : BadUTF8) {
83       EXPECT_GE(lspLength(Bad), 0u);
84       EXPECT_LE(lspLength(Bad), strlen(Bad));
85     }
86   }
87 }
88 
89 // The = → �� below are ASCII (1 byte), BMP (3 bytes), and astral (4 bytes).
90 const char File[] = R"(0:0 = 0
91 1:0 → 8
92 2:0 �� 18)";
93 struct Line {
94   unsigned Number;
95   unsigned Offset;
96   unsigned Length;
97 };
98 Line FileLines[] = {Line{0, 0, 7}, Line{1, 8, 9}, Line{2, 18, 11}};
99 
TEST(SourceCodeTests,PositionToOffset)100 TEST(SourceCodeTests, PositionToOffset) {
101   // line out of bounds
102   EXPECT_THAT_EXPECTED(positionToOffset(File, position(-1, 2)), llvm::Failed());
103   // first line
104   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, -1)),
105                        llvm::Failed()); // out of range
106   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 0)),
107                        llvm::HasValue(0)); // first character
108   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 3)),
109                        llvm::HasValue(3)); // middle character
110   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 6)),
111                        llvm::HasValue(6)); // last character
112   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 7)),
113                        llvm::HasValue(7)); // the newline itself
114   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 7), false),
115                        llvm::HasValue(7));
116   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 8)),
117                        llvm::HasValue(7)); // out of range
118   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 8), false),
119                        llvm::Failed()); // out of range
120   // middle line
121   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, -1)),
122                        llvm::Failed()); // out of range
123   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 0)),
124                        llvm::HasValue(8)); // first character
125   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 3)),
126                        llvm::HasValue(11)); // middle character
127   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 3), false),
128                        llvm::HasValue(11));
129   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 6)),
130                        llvm::HasValue(16)); // last character
131   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 7)),
132                        llvm::HasValue(17)); // the newline itself
133   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 8)),
134                        llvm::HasValue(17)); // out of range
135   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 8), false),
136                        llvm::Failed()); // out of range
137   // last line
138   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, -1)),
139                        llvm::Failed()); // out of range
140   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 0)),
141                        llvm::HasValue(18)); // first character
142   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 3)),
143                        llvm::HasValue(21)); // middle character
144   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 5), false),
145                        llvm::Failed()); // middle of surrogate pair
146   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 5)),
147                        llvm::HasValue(26)); // middle of surrogate pair
148   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 6), false),
149                        llvm::HasValue(26)); // end of surrogate pair
150   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 8)),
151                        llvm::HasValue(28)); // last character
152   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 9)),
153                        llvm::HasValue(29)); // EOF
154   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 10), false),
155                        llvm::Failed()); // out of range
156   // line out of bounds
157   EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 0)), llvm::Failed());
158   EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 1)), llvm::Failed());
159 
160   // Codepoints are similar, except near astral characters.
161   WithContextValue UTF32(kCurrentOffsetEncoding, OffsetEncoding::UTF32);
162   // line out of bounds
163   EXPECT_THAT_EXPECTED(positionToOffset(File, position(-1, 2)), llvm::Failed());
164   // first line
165   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, -1)),
166                        llvm::Failed()); // out of range
167   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 0)),
168                        llvm::HasValue(0)); // first character
169   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 3)),
170                        llvm::HasValue(3)); // middle character
171   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 6)),
172                        llvm::HasValue(6)); // last character
173   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 7)),
174                        llvm::HasValue(7)); // the newline itself
175   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 7), false),
176                        llvm::HasValue(7));
177   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 8)),
178                        llvm::HasValue(7)); // out of range
179   EXPECT_THAT_EXPECTED(positionToOffset(File, position(0, 8), false),
180                        llvm::Failed()); // out of range
181   // middle line
182   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, -1)),
183                        llvm::Failed()); // out of range
184   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 0)),
185                        llvm::HasValue(8)); // first character
186   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 3)),
187                        llvm::HasValue(11)); // middle character
188   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 3), false),
189                        llvm::HasValue(11));
190   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 6)),
191                        llvm::HasValue(16)); // last character
192   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 7)),
193                        llvm::HasValue(17)); // the newline itself
194   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 8)),
195                        llvm::HasValue(17)); // out of range
196   EXPECT_THAT_EXPECTED(positionToOffset(File, position(1, 8), false),
197                        llvm::Failed()); // out of range
198   // last line
199   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, -1)),
200                        llvm::Failed()); // out of range
201   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 0)),
202                        llvm::HasValue(18)); // first character
203   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 4)),
204                        llvm::HasValue(22)); // Before astral character.
205   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 5), false),
206                        llvm::HasValue(26)); // after astral character
207   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 7)),
208                        llvm::HasValue(28)); // last character
209   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 8)),
210                        llvm::HasValue(29)); // EOF
211   EXPECT_THAT_EXPECTED(positionToOffset(File, position(2, 9), false),
212                        llvm::Failed()); // out of range
213   // line out of bounds
214   EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 0)), llvm::Failed());
215   EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 1)), llvm::Failed());
216 
217   // Test UTF-8, where transformations are trivial.
218   WithContextValue UTF8(kCurrentOffsetEncoding, OffsetEncoding::UTF8);
219   EXPECT_THAT_EXPECTED(positionToOffset(File, position(-1, 2)), llvm::Failed());
220   EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 0)), llvm::Failed());
221   for (Line L : FileLines) {
222     EXPECT_THAT_EXPECTED(positionToOffset(File, position(L.Number, -1)),
223                          llvm::Failed()); // out of range
224     for (unsigned I = 0; I <= L.Length; ++I)
225       EXPECT_THAT_EXPECTED(positionToOffset(File, position(L.Number, I)),
226                            llvm::HasValue(L.Offset + I));
227     EXPECT_THAT_EXPECTED(
228         positionToOffset(File, position(L.Number, L.Length + 1)),
229         llvm::HasValue(L.Offset + L.Length));
230     EXPECT_THAT_EXPECTED(
231         positionToOffset(File, position(L.Number, L.Length + 1), false),
232         llvm::Failed()); // out of range
233   }
234 }
235 
TEST(SourceCodeTests,OffsetToPosition)236 TEST(SourceCodeTests, OffsetToPosition) {
237   EXPECT_THAT(offsetToPosition(File, 0), Pos(0, 0)) << "start of file";
238   EXPECT_THAT(offsetToPosition(File, 3), Pos(0, 3)) << "in first line";
239   EXPECT_THAT(offsetToPosition(File, 6), Pos(0, 6)) << "end of first line";
240   EXPECT_THAT(offsetToPosition(File, 7), Pos(0, 7)) << "first newline";
241   EXPECT_THAT(offsetToPosition(File, 8), Pos(1, 0)) << "start of second line";
242   EXPECT_THAT(offsetToPosition(File, 12), Pos(1, 4)) << "before BMP char";
243   EXPECT_THAT(offsetToPosition(File, 13), Pos(1, 5)) << "in BMP char";
244   EXPECT_THAT(offsetToPosition(File, 15), Pos(1, 5)) << "after BMP char";
245   EXPECT_THAT(offsetToPosition(File, 16), Pos(1, 6)) << "end of second line";
246   EXPECT_THAT(offsetToPosition(File, 17), Pos(1, 7)) << "second newline";
247   EXPECT_THAT(offsetToPosition(File, 18), Pos(2, 0)) << "start of last line";
248   EXPECT_THAT(offsetToPosition(File, 21), Pos(2, 3)) << "in last line";
249   EXPECT_THAT(offsetToPosition(File, 22), Pos(2, 4)) << "before astral char";
250   EXPECT_THAT(offsetToPosition(File, 24), Pos(2, 6)) << "in astral char";
251   EXPECT_THAT(offsetToPosition(File, 26), Pos(2, 6)) << "after astral char";
252   EXPECT_THAT(offsetToPosition(File, 28), Pos(2, 8)) << "end of last line";
253   EXPECT_THAT(offsetToPosition(File, 29), Pos(2, 9)) << "EOF";
254   EXPECT_THAT(offsetToPosition(File, 30), Pos(2, 9)) << "out of bounds";
255 
256   // Codepoints are similar, except near astral characters.
257   WithContextValue UTF32(kCurrentOffsetEncoding, OffsetEncoding::UTF32);
258   EXPECT_THAT(offsetToPosition(File, 0), Pos(0, 0)) << "start of file";
259   EXPECT_THAT(offsetToPosition(File, 3), Pos(0, 3)) << "in first line";
260   EXPECT_THAT(offsetToPosition(File, 6), Pos(0, 6)) << "end of first line";
261   EXPECT_THAT(offsetToPosition(File, 7), Pos(0, 7)) << "first newline";
262   EXPECT_THAT(offsetToPosition(File, 8), Pos(1, 0)) << "start of second line";
263   EXPECT_THAT(offsetToPosition(File, 12), Pos(1, 4)) << "before BMP char";
264   EXPECT_THAT(offsetToPosition(File, 13), Pos(1, 5)) << "in BMP char";
265   EXPECT_THAT(offsetToPosition(File, 15), Pos(1, 5)) << "after BMP char";
266   EXPECT_THAT(offsetToPosition(File, 16), Pos(1, 6)) << "end of second line";
267   EXPECT_THAT(offsetToPosition(File, 17), Pos(1, 7)) << "second newline";
268   EXPECT_THAT(offsetToPosition(File, 18), Pos(2, 0)) << "start of last line";
269   EXPECT_THAT(offsetToPosition(File, 21), Pos(2, 3)) << "in last line";
270   EXPECT_THAT(offsetToPosition(File, 22), Pos(2, 4)) << "before astral char";
271   EXPECT_THAT(offsetToPosition(File, 24), Pos(2, 5)) << "in astral char";
272   EXPECT_THAT(offsetToPosition(File, 26), Pos(2, 5)) << "after astral char";
273   EXPECT_THAT(offsetToPosition(File, 28), Pos(2, 7)) << "end of last line";
274   EXPECT_THAT(offsetToPosition(File, 29), Pos(2, 8)) << "EOF";
275   EXPECT_THAT(offsetToPosition(File, 30), Pos(2, 8)) << "out of bounds";
276 
277   WithContextValue UTF8(kCurrentOffsetEncoding, OffsetEncoding::UTF8);
278   for (Line L : FileLines) {
279     for (unsigned I = 0; I <= L.Length; ++I)
280       EXPECT_THAT(offsetToPosition(File, L.Offset + I), Pos(L.Number, I));
281   }
282   EXPECT_THAT(offsetToPosition(File, 30), Pos(2, 11)) << "out of bounds";
283 }
284 
TEST(SourceCodeTests,SourceLocationInMainFile)285 TEST(SourceCodeTests, SourceLocationInMainFile) {
286   Annotations Source(R"cpp(
287     ^in^t ^foo
288     ^bar
289     ^baz ^() {}  {} {} {} { }^
290 )cpp");
291 
292   SourceManagerForFile Owner("foo.cpp", Source.code());
293   SourceManager &SM = Owner.get();
294 
295   SourceLocation StartOfFile = SM.getLocForStartOfFile(SM.getMainFileID());
296   EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, position(0, 0)),
297                        HasValue(StartOfFile));
298   // End of file.
299   EXPECT_THAT_EXPECTED(
300       sourceLocationInMainFile(SM, position(4, 0)),
301       HasValue(StartOfFile.getLocWithOffset(Source.code().size())));
302   // Column number is too large.
303   EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, position(0, 1)), Failed());
304   EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, position(0, 100)),
305                        Failed());
306   EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, position(4, 1)), Failed());
307   // Line number is too large.
308   EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, position(5, 0)), Failed());
309   // Check all positions mentioned in the test return valid results.
310   for (auto P : Source.points()) {
311     size_t Offset = llvm::cantFail(positionToOffset(Source.code(), P));
312     EXPECT_THAT_EXPECTED(sourceLocationInMainFile(SM, P),
313                          HasValue(StartOfFile.getLocWithOffset(Offset)));
314   }
315 }
316 
TEST(SourceCodeTests,CollectIdentifiers)317 TEST(SourceCodeTests, CollectIdentifiers) {
318   auto Style = format::getLLVMStyle();
319   auto IDs = collectIdentifiers(R"cpp(
320   #include "a.h"
321   void foo() { int xyz; int abc = xyz; return foo(); }
322   )cpp",
323                                 Style);
324   EXPECT_EQ(IDs.size(), 7u);
325   EXPECT_EQ(IDs["include"], 1u);
326   EXPECT_EQ(IDs["void"], 1u);
327   EXPECT_EQ(IDs["int"], 2u);
328   EXPECT_EQ(IDs["xyz"], 2u);
329   EXPECT_EQ(IDs["abc"], 1u);
330   EXPECT_EQ(IDs["return"], 1u);
331   EXPECT_EQ(IDs["foo"], 2u);
332 }
333 
TEST(SourceCodeTests,CollectWords)334 TEST(SourceCodeTests, CollectWords) {
335   auto Words = collectWords(R"cpp(
336   #define FIZZ_BUZZ
337   // this is a comment
338   std::string getSomeText() { return "magic word"; }
339   )cpp");
340   std::set<StringRef> ActualWords(Words.keys().begin(), Words.keys().end());
341   std::set<StringRef> ExpectedWords = {"define",  "fizz",   "buzz", "this",
342                                        "comment", "string", "some", "text",
343                                        "return",  "magic",  "word"};
344   EXPECT_EQ(ActualWords, ExpectedWords);
345 }
346 
347 class SpelledWordsTest : public ::testing::Test {
348   llvm::Optional<ParsedAST> AST;
349 
tryWord(const char * Text)350   llvm::Optional<SpelledWord> tryWord(const char *Text) {
351     llvm::Annotations A(Text);
352     auto TU = TestTU::withCode(A.code());
353     AST = TU.build();
354     auto SW = SpelledWord::touching(
355         AST->getSourceManager().getComposedLoc(
356             AST->getSourceManager().getMainFileID(), A.point()),
357         AST->getTokens(), AST->getLangOpts());
358     if (A.ranges().size()) {
359       llvm::StringRef Want = A.code().slice(A.range().Begin, A.range().End);
360       EXPECT_EQ(Want, SW->Text) << Text;
361     }
362     return SW;
363   }
364 
365 protected:
word(const char * Text)366   SpelledWord word(const char *Text) {
367     auto Result = tryWord(Text);
368     EXPECT_TRUE(Result) << Text;
369     return Result.getValueOr(SpelledWord());
370   }
371 
noWord(const char * Text)372   void noWord(const char *Text) { EXPECT_FALSE(tryWord(Text)) << Text; }
373 };
374 
TEST_F(SpelledWordsTest,HeuristicBoundaries)375 TEST_F(SpelledWordsTest, HeuristicBoundaries) {
376   word("// [[^foo]] ");
377   word("// [[f^oo]] ");
378   word("// [[foo^]] ");
379   word("// [[foo^]]+bar ");
380   noWord("//^ foo ");
381   noWord("// foo ^");
382 }
383 
TEST_F(SpelledWordsTest,LikelyIdentifier)384 TEST_F(SpelledWordsTest, LikelyIdentifier) {
385   EXPECT_FALSE(word("// ^foo ").LikelyIdentifier);
386   EXPECT_TRUE(word("// [[^foo_bar]] ").LikelyIdentifier);
387   EXPECT_TRUE(word("// [[^fooBar]] ").LikelyIdentifier);
388   EXPECT_FALSE(word("// H^TTP ").LikelyIdentifier);
389   EXPECT_TRUE(word("// \\p [[^foo]] ").LikelyIdentifier);
390   EXPECT_TRUE(word("// @param[in] [[^foo]] ").LikelyIdentifier);
391   EXPECT_TRUE(word("// `[[f^oo]]` ").LikelyIdentifier);
392   EXPECT_TRUE(word("// bar::[[f^oo]] ").LikelyIdentifier);
393   EXPECT_TRUE(word("// [[f^oo]]::bar ").LikelyIdentifier);
394 }
395 
TEST_F(SpelledWordsTest,Comment)396 TEST_F(SpelledWordsTest, Comment) {
397   auto W = word("// [[^foo]]");
398   EXPECT_FALSE(W.PartOfSpelledToken);
399   EXPECT_FALSE(W.SpelledToken);
400   EXPECT_FALSE(W.ExpandedToken);
401 }
402 
TEST_F(SpelledWordsTest,PartOfString)403 TEST_F(SpelledWordsTest, PartOfString) {
404   auto W = word(R"( auto str = "foo [[^bar]] baz"; )");
405   ASSERT_TRUE(W.PartOfSpelledToken);
406   EXPECT_EQ(W.PartOfSpelledToken->kind(), tok::string_literal);
407   EXPECT_FALSE(W.SpelledToken);
408   EXPECT_FALSE(W.ExpandedToken);
409 }
410 
TEST_F(SpelledWordsTest,DisabledSection)411 TEST_F(SpelledWordsTest, DisabledSection) {
412   auto W = word(R"cpp(
413     #if 0
414     foo [[^bar]] baz
415     #endif
416     )cpp");
417   ASSERT_TRUE(W.SpelledToken);
418   EXPECT_EQ(W.SpelledToken->kind(), tok::identifier);
419   EXPECT_EQ(W.SpelledToken, W.PartOfSpelledToken);
420   EXPECT_FALSE(W.ExpandedToken);
421 }
422 
TEST_F(SpelledWordsTest,Macros)423 TEST_F(SpelledWordsTest, Macros) {
424   auto W = word(R"cpp(
425     #define ID(X) X
426     ID(int [[^i]]);
427     )cpp");
428   ASSERT_TRUE(W.SpelledToken);
429   EXPECT_EQ(W.SpelledToken->kind(), tok::identifier);
430   EXPECT_EQ(W.SpelledToken, W.PartOfSpelledToken);
431   ASSERT_TRUE(W.ExpandedToken);
432   EXPECT_EQ(W.ExpandedToken->kind(), tok::identifier);
433 
434   W = word(R"cpp(
435     #define OBJECT Expansion;
436     int [[^OBJECT]];
437     )cpp");
438   EXPECT_TRUE(W.SpelledToken);
439   EXPECT_FALSE(W.ExpandedToken) << "Expanded token is spelled differently";
440 }
441 
TEST(SourceCodeTests,VisibleNamespaces)442 TEST(SourceCodeTests, VisibleNamespaces) {
443   std::vector<std::pair<const char *, std::vector<std::string>>> Cases = {
444       {
445           R"cpp(
446             // Using directive resolved against enclosing namespaces.
447             using namespace foo;
448             namespace ns {
449             using namespace bar;
450           )cpp",
451           {"ns", "", "bar", "foo", "ns::bar"},
452       },
453       {
454           R"cpp(
455             // Don't include namespaces we've closed, ignore namespace aliases.
456             using namespace clang;
457             using std::swap;
458             namespace clang {
459             namespace clangd {}
460             namespace ll = ::llvm;
461             }
462             namespace clang {
463           )cpp",
464           {"clang", ""},
465       },
466       {
467           R"cpp(
468             // Using directives visible even if a namespace is reopened.
469             // Ignore anonymous namespaces.
470             namespace foo{ using namespace bar; }
471             namespace foo{ namespace {
472           )cpp",
473           {"foo", "", "bar", "foo::bar"},
474       },
475       {
476           R"cpp(
477             // Mismatched braces
478             namespace foo{}
479             }}}
480             namespace bar{
481           )cpp",
482           {"bar", ""},
483       },
484       {
485           R"cpp(
486             // Namespaces with multiple chunks.
487             namespace a::b {
488               using namespace c::d;
489               namespace e::f {
490           )cpp",
491           {
492               "a::b::e::f",
493               "",
494               "a",
495               "a::b",
496               "a::b::c::d",
497               "a::b::e",
498               "a::c::d",
499               "c::d",
500           },
501       },
502       {
503           "",
504           {""},
505       },
506       {
507           R"cpp(
508             // Parse until EOF
509             namespace bar{})cpp",
510           {""},
511       },
512   };
513   for (const auto &Case : Cases) {
514     EXPECT_EQ(Case.second,
515               visibleNamespaces(Case.first, format::getFormattingLangOpts(
516                                                 format::getLLVMStyle())))
517         << Case.first;
518   }
519 }
520 
TEST(SourceCodeTests,GetMacros)521 TEST(SourceCodeTests, GetMacros) {
522   Annotations Code(R"cpp(
523      #define MACRO 123
524      int abc = MA^CRO;
525    )cpp");
526   TestTU TU = TestTU::withCode(Code.code());
527   auto AST = TU.build();
528   auto CurLoc = sourceLocationInMainFile(AST.getSourceManager(), Code.point());
529   ASSERT_TRUE(bool(CurLoc));
530   const auto *Id = syntax::spelledIdentifierTouching(*CurLoc, AST.getTokens());
531   ASSERT_TRUE(Id);
532   auto Result = locateMacroAt(*Id, AST.getPreprocessor());
533   ASSERT_TRUE(Result);
534   EXPECT_THAT(*Result, MacroName("MACRO"));
535 }
536 
TEST(SourceCodeTests,WorksAtBeginOfFile)537 TEST(SourceCodeTests, WorksAtBeginOfFile) {
538   Annotations Code("^MACRO");
539   TestTU TU = TestTU::withCode(Code.code());
540   TU.HeaderCode = "#define MACRO int x;";
541   auto AST = TU.build();
542   auto CurLoc = sourceLocationInMainFile(AST.getSourceManager(), Code.point());
543   ASSERT_TRUE(bool(CurLoc));
544   const auto *Id = syntax::spelledIdentifierTouching(*CurLoc, AST.getTokens());
545   ASSERT_TRUE(Id);
546   auto Result = locateMacroAt(*Id, AST.getPreprocessor());
547   ASSERT_TRUE(Result);
548   EXPECT_THAT(*Result, MacroName("MACRO"));
549 }
550 
TEST(SourceCodeTests,IsInsideMainFile)551 TEST(SourceCodeTests, IsInsideMainFile) {
552   TestTU TU;
553   TU.HeaderCode = R"cpp(
554     #define DEFINE_CLASS(X) class X {};
555     #define DEFINE_YY DEFINE_CLASS(YY)
556 
557     class Header1 {};
558     DEFINE_CLASS(Header2)
559     class Header {};
560   )cpp";
561   TU.Code = R"cpp(
562     #define DEFINE_MAIN4 class Main4{};
563     class Main1 {};
564     DEFINE_CLASS(Main2)
565     DEFINE_YY
566     class Main {};
567     DEFINE_MAIN4
568   )cpp";
569   TU.ExtraArgs.push_back("-DHeader=Header3");
570   TU.ExtraArgs.push_back("-DMain=Main3");
571   auto AST = TU.build();
572   const auto &SM = AST.getSourceManager();
573   auto DeclLoc = [&AST](llvm::StringRef Name) {
574     return findDecl(AST, Name).getLocation();
575   };
576   for (const auto *HeaderDecl : {"Header1", "Header2", "Header3"})
577     EXPECT_FALSE(isInsideMainFile(DeclLoc(HeaderDecl), SM)) << HeaderDecl;
578 
579   for (const auto *MainDecl : {"Main1", "Main2", "Main3", "Main4", "YY"})
580     EXPECT_TRUE(isInsideMainFile(DeclLoc(MainDecl), SM)) << MainDecl;
581 
582   // Main4 is *spelled* in the preamble, but in the main-file part of it.
583   EXPECT_TRUE(isInsideMainFile(SM.getSpellingLoc(DeclLoc("Main4")), SM));
584 }
585 
586 // Test for functions toHalfOpenFileRange and getHalfOpenFileRange
TEST(SourceCodeTests,HalfOpenFileRange)587 TEST(SourceCodeTests, HalfOpenFileRange) {
588   // Each marked range should be the file range of the decl with the same name
589   // and each name should be unique.
590   Annotations Test(R"cpp(
591     #define FOO(X, Y) int Y = ++X
592     #define BAR(X) X + 1
593     #define ECHO(X) X
594 
595     #define BUZZ BAZZ(ADD)
596     #define BAZZ(m) m(1)
597     #define ADD(a) int f = a + 1;
598     template<typename T>
599     class P {};
600 
601     int main() {
602       $a[[P<P<P<P<P<int>>>>> a]];
603       $b[[int b = 1]];
604       $c[[FOO(b, c)]];
605       $d[[FOO(BAR(BAR(b)), d)]];
606       // FIXME: We might want to select everything inside the outer ECHO.
607       ECHO(ECHO($e[[int) ECHO(e]]));
608       // Shouldn't crash.
609       $f[[BUZZ]];
610     }
611   )cpp");
612 
613   ParsedAST AST = TestTU::withCode(Test.code()).build();
614   llvm::errs() << Test.code();
615   const SourceManager &SM = AST.getSourceManager();
616   const LangOptions &LangOpts = AST.getLangOpts();
617   // Turn a SourceLocation into a pair of positions
618   auto SourceRangeToRange = [&SM](SourceRange SrcRange) {
619     return Range{sourceLocToPosition(SM, SrcRange.getBegin()),
620                  sourceLocToPosition(SM, SrcRange.getEnd())};
621   };
622   auto CheckRange = [&](llvm::StringRef Name) {
623     const NamedDecl &Decl = findUnqualifiedDecl(AST, Name);
624     auto FileRange = toHalfOpenFileRange(SM, LangOpts, Decl.getSourceRange());
625     SCOPED_TRACE("Checking range: " + Name);
626     ASSERT_NE(FileRange, llvm::None);
627     Range HalfOpenRange = SourceRangeToRange(*FileRange);
628     EXPECT_EQ(HalfOpenRange, Test.ranges(Name)[0]);
629   };
630 
631   CheckRange("a");
632   CheckRange("b");
633   CheckRange("c");
634   CheckRange("d");
635   CheckRange("e");
636   CheckRange("f");
637 }
638 
TEST(SourceCodeTests,HalfOpenFileRangePathologicalPreprocessor)639 TEST(SourceCodeTests, HalfOpenFileRangePathologicalPreprocessor) {
640   const char *Case = R"cpp(
641 #define MACRO while(1)
642     void test() {
643 [[#include "Expand.inc"
644         br^eak]];
645     }
646   )cpp";
647   Annotations Test(Case);
648   auto TU = TestTU::withCode(Test.code());
649   TU.AdditionalFiles["Expand.inc"] = "MACRO\n";
650   auto AST = TU.build();
651 
652   const auto &Func = cast<FunctionDecl>(findDecl(AST, "test"));
653   const auto &Body = cast<CompoundStmt>(Func.getBody());
654   const auto &Loop = cast<WhileStmt>(*Body->child_begin());
655   llvm::Optional<SourceRange> Range = toHalfOpenFileRange(
656       AST.getSourceManager(), AST.getLangOpts(), Loop->getSourceRange());
657   ASSERT_TRUE(Range) << "Failed to get file range";
658   EXPECT_EQ(AST.getSourceManager().getFileOffset(Range->getBegin()),
659             Test.llvm::Annotations::range().Begin);
660   EXPECT_EQ(AST.getSourceManager().getFileOffset(Range->getEnd()),
661             Test.llvm::Annotations::range().End);
662 }
663 
TEST(SourceCodeTests,IncludeHashLoc)664 TEST(SourceCodeTests, IncludeHashLoc) {
665   const char *Case = R"cpp(
666 $foo^#include "foo.inc"
667 #define HEADER "bar.inc"
668   $bar^#  include HEADER
669   )cpp";
670   Annotations Test(Case);
671   auto TU = TestTU::withCode(Test.code());
672   TU.AdditionalFiles["foo.inc"] = "int foo;\n";
673   TU.AdditionalFiles["bar.inc"] = "int bar;\n";
674   auto AST = TU.build();
675   const auto &SM = AST.getSourceManager();
676 
677   FileID Foo = SM.getFileID(findDecl(AST, "foo").getLocation());
678   EXPECT_EQ(SM.getFileOffset(includeHashLoc(Foo, SM)),
679             Test.llvm::Annotations::point("foo"));
680   FileID Bar = SM.getFileID(findDecl(AST, "bar").getLocation());
681   EXPECT_EQ(SM.getFileOffset(includeHashLoc(Bar, SM)),
682             Test.llvm::Annotations::point("bar"));
683 }
684 
TEST(SourceCodeTests,GetEligiblePoints)685 TEST(SourceCodeTests, GetEligiblePoints) {
686   constexpr struct {
687     const char *Code;
688     const char *FullyQualifiedName;
689     const char *EnclosingNamespace;
690   } Cases[] = {
691       {R"cpp(// FIXME: We should also mark positions before and after
692                  //declarations/definitions as eligible.
693               namespace ns1 {
694               namespace a { namespace ns2 {} }
695               namespace ns2 {^
696               void foo();
697               namespace {}
698               void bar() {}
699               namespace ns3 {}
700               class T {};
701               ^}
702               using namespace ns2;
703               })cpp",
704        "ns1::ns2::symbol", "ns1::ns2::"},
705       {R"cpp(
706               namespace ns1 {^
707               namespace a { namespace ns2 {} }
708               namespace b {}
709               namespace ns {}
710               ^})cpp",
711        "ns1::ns2::symbol", "ns1::"},
712       {R"cpp(
713               namespace x {
714               namespace a { namespace ns2 {} }
715               namespace b {}
716               namespace ns {}
717               }^)cpp",
718        "ns1::ns2::symbol", ""},
719       {R"cpp(
720               namespace ns1 {
721               namespace ns2 {^^}
722               namespace b {}
723               namespace ns2 {^^}
724               }
725               namespace ns1 {namespace ns2 {^^}})cpp",
726        "ns1::ns2::symbol", "ns1::ns2::"},
727       {R"cpp(
728               namespace ns1 {^
729               namespace ns {}
730               namespace b {}
731               namespace ns {}
732               ^}
733               namespace ns1 {^namespace ns {}^})cpp",
734        "ns1::ns2::symbol", "ns1::"},
735   };
736   for (auto Case : Cases) {
737     Annotations Test(Case.Code);
738 
739     auto Res = getEligiblePoints(
740         Test.code(), Case.FullyQualifiedName,
741         format::getFormattingLangOpts(format::getLLVMStyle()));
742     EXPECT_THAT(Res.EligiblePoints, testing::ElementsAreArray(Test.points()))
743         << Test.code();
744     EXPECT_EQ(Res.EnclosingNamespace, Case.EnclosingNamespace) << Test.code();
745   }
746 }
747 
TEST(SourceCodeTests,IdentifierRanges)748 TEST(SourceCodeTests, IdentifierRanges) {
749   Annotations Code(R"cpp(
750    class [[Foo]] {};
751    // Foo
752    /* Foo */
753    void f([[Foo]]* foo1) {
754      [[Foo]] foo2;
755      auto S = [[Foo]]();
756 // cross-line identifier is not supported.
757 F\
758 o\
759 o foo2;
760    }
761   )cpp");
762   LangOptions LangOpts;
763   LangOpts.CPlusPlus = true;
764   EXPECT_EQ(Code.ranges(),
765             collectIdentifierRanges("Foo", Code.code(), LangOpts));
766 }
767 
TEST(SourceCodeTests,isHeaderFile)768 TEST(SourceCodeTests, isHeaderFile) {
769   // Without lang options.
770   EXPECT_TRUE(isHeaderFile("foo.h"));
771   EXPECT_TRUE(isHeaderFile("foo.hh"));
772   EXPECT_TRUE(isHeaderFile("foo.hpp"));
773 
774   EXPECT_FALSE(isHeaderFile("foo.cpp"));
775   EXPECT_FALSE(isHeaderFile("foo.c++"));
776   EXPECT_FALSE(isHeaderFile("foo.cxx"));
777   EXPECT_FALSE(isHeaderFile("foo.cc"));
778   EXPECT_FALSE(isHeaderFile("foo.c"));
779   EXPECT_FALSE(isHeaderFile("foo.mm"));
780   EXPECT_FALSE(isHeaderFile("foo.m"));
781 
782   // With lang options
783   LangOptions LangOpts;
784   LangOpts.IsHeaderFile = true;
785   EXPECT_TRUE(isHeaderFile("string", LangOpts));
786   // Emulate cases where there is no "-x header" flag for a .h file, we still
787   // want to treat it as a header.
788   LangOpts.IsHeaderFile = false;
789   EXPECT_TRUE(isHeaderFile("header.h", LangOpts));
790 }
791 
792 } // namespace
793 } // namespace clangd
794 } // namespace clang
795