1 //==- SemanticHighlightingTests.cpp - SemanticHighlighting tests-*- C++ -* -==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Annotations.h"
10 #include "ClangdServer.h"
11 #include "Protocol.h"
12 #include "SemanticHighlighting.h"
13 #include "SourceCode.h"
14 #include "TestFS.h"
15 #include "TestTU.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/Support/Error.h"
20 #include "llvm/Support/ScopedPrinter.h"
21 #include "gmock/gmock.h"
22 #include <algorithm>
23 
24 namespace clang {
25 namespace clangd {
26 namespace {
27 
28 using testing::IsEmpty;
29 using testing::SizeIs;
30 
31 MATCHER_P(LineNumber, L, "") { return arg.Line == L; }
32 MATCHER(EmptyHighlightings, "") { return arg.Tokens.empty(); }
33 
34 std::vector<HighlightingToken>
makeHighlightingTokens(llvm::ArrayRef<Range> Ranges,HighlightingKind Kind)35 makeHighlightingTokens(llvm::ArrayRef<Range> Ranges, HighlightingKind Kind) {
36   std::vector<HighlightingToken> Tokens(Ranges.size());
37   for (int I = 0, End = Ranges.size(); I < End; ++I) {
38     Tokens[I].R = Ranges[I];
39     Tokens[I].Kind = Kind;
40   }
41 
42   return Tokens;
43 }
44 
getExpectedTokens(Annotations & Test)45 std::vector<HighlightingToken> getExpectedTokens(Annotations &Test) {
46   static const std::map<HighlightingKind, std::string> KindToString{
47       {HighlightingKind::Variable, "Variable"},
48       {HighlightingKind::LocalVariable, "LocalVariable"},
49       {HighlightingKind::Parameter, "Parameter"},
50       {HighlightingKind::Function, "Function"},
51       {HighlightingKind::Class, "Class"},
52       {HighlightingKind::Enum, "Enum"},
53       {HighlightingKind::Namespace, "Namespace"},
54       {HighlightingKind::EnumConstant, "EnumConstant"},
55       {HighlightingKind::Field, "Field"},
56       {HighlightingKind::StaticField, "StaticField"},
57       {HighlightingKind::Method, "Method"},
58       {HighlightingKind::StaticMethod, "StaticMethod"},
59       {HighlightingKind::Typedef, "Typedef"},
60       {HighlightingKind::DependentType, "DependentType"},
61       {HighlightingKind::DependentName, "DependentName"},
62       {HighlightingKind::TemplateParameter, "TemplateParameter"},
63       {HighlightingKind::Concept, "Concept"},
64       {HighlightingKind::Primitive, "Primitive"},
65       {HighlightingKind::Macro, "Macro"}};
66   std::vector<HighlightingToken> ExpectedTokens;
67   for (const auto &KindString : KindToString) {
68     std::vector<HighlightingToken> Toks = makeHighlightingTokens(
69         Test.ranges(KindString.second), KindString.first);
70     ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
71   }
72   llvm::sort(ExpectedTokens);
73   return ExpectedTokens;
74 }
75 
76 /// Annotates the input code with provided semantic highlightings. Results look
77 /// something like:
78 ///   class $Class[[X]] {
79 ///     $Primitive[[int]] $Field[[a]] = 0;
80 ///   };
annotate(llvm::StringRef Input,llvm::ArrayRef<HighlightingToken> Tokens)81 std::string annotate(llvm::StringRef Input,
82                      llvm::ArrayRef<HighlightingToken> Tokens) {
83   assert(std::is_sorted(
84       Tokens.begin(), Tokens.end(),
85       [](const HighlightingToken &L, const HighlightingToken &R) {
86         return L.R.start < R.R.start;
87       }));
88 
89   std::string Result;
90   unsigned NextChar = 0;
91   for (auto &T : Tokens) {
92     unsigned StartOffset = llvm::cantFail(positionToOffset(Input, T.R.start));
93     unsigned EndOffset = llvm::cantFail(positionToOffset(Input, T.R.end));
94     assert(StartOffset <= EndOffset);
95     assert(NextChar <= StartOffset);
96 
97     Result += Input.substr(NextChar, StartOffset - NextChar);
98     Result += std::string(
99         llvm::formatv("${0}[[{1}]]", T.Kind,
100                       Input.substr(StartOffset, EndOffset - StartOffset)));
101     NextChar = EndOffset;
102   }
103   Result += Input.substr(NextChar);
104   return Result;
105 }
106 
checkHighlightings(llvm::StringRef Code,std::vector<std::pair<llvm::StringRef,llvm::StringRef>> AdditionalFiles={})107 void checkHighlightings(llvm::StringRef Code,
108                         std::vector<std::pair</*FileName*/ llvm::StringRef,
109                                               /*FileContent*/ llvm::StringRef>>
110                             AdditionalFiles = {}) {
111   Annotations Test(Code);
112   TestTU TU;
113   TU.Code = std::string(Test.code());
114 
115   // FIXME: Auto-completion in a template requires disabling delayed template
116   // parsing.
117   TU.ExtraArgs.push_back("-fno-delayed-template-parsing");
118   TU.ExtraArgs.push_back("-std=c++20");
119 
120   for (auto File : AdditionalFiles)
121     TU.AdditionalFiles.insert({File.first, std::string(File.second)});
122   auto AST = TU.build();
123 
124   EXPECT_EQ(Code, annotate(Test.code(), getSemanticHighlightings(AST)));
125 }
126 
127 // Any annotations in OldCode and NewCode are converted into their corresponding
128 // HighlightingToken. The tokens are diffed against each other. Any lines where
129 // the tokens should diff must be marked with a ^ somewhere on that line in
130 // NewCode. If there are diffs that aren't marked with ^ the test fails. The
131 // test also fails if there are lines marked with ^ that don't differ.
checkDiffedHighlights(llvm::StringRef OldCode,llvm::StringRef NewCode)132 void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
133   Annotations OldTest(OldCode);
134   Annotations NewTest(NewCode);
135   std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
136   std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
137 
138   llvm::DenseMap<int, std::vector<HighlightingToken>> ExpectedLines;
139   for (const Position &Point : NewTest.points()) {
140     ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
141                                // are inserted on these lines later.
142   }
143   std::vector<LineHighlightings> ExpectedLinePairHighlighting;
144   for (const HighlightingToken &Token : NewTokens) {
145     auto It = ExpectedLines.find(Token.R.start.line);
146     if (It != ExpectedLines.end())
147       It->second.push_back(Token);
148   }
149   for (auto &LineTokens : ExpectedLines)
150     ExpectedLinePairHighlighting.push_back(
151         {LineTokens.first, LineTokens.second, /*IsInactive = */ false});
152 
153   std::vector<LineHighlightings> ActualDiffed =
154       diffHighlightings(NewTokens, OldTokens);
155   EXPECT_THAT(ActualDiffed,
156               testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting))
157       << OldCode;
158 }
159 
TEST(SemanticHighlighting,GetsCorrectTokens)160 TEST(SemanticHighlighting, GetsCorrectTokens) {
161   const char *TestCases[] = {
162       R"cpp(
163       struct $Class[[AS]] {
164         double $Field[[SomeMember]];
165       };
166       struct {
167       } $Variable[[S]];
168       void $Function[[foo]](int $Parameter[[A]], $Class[[AS]] $Parameter[[As]]) {
169         $Primitive[[auto]] $LocalVariable[[VeryLongVariableName]] = 12312;
170         $Class[[AS]]     $LocalVariable[[AA]];
171         $Primitive[[auto]] $LocalVariable[[L]] = $LocalVariable[[AA]].$Field[[SomeMember]] + $Parameter[[A]];
172         auto $LocalVariable[[FN]] = [ $LocalVariable[[AA]]](int $Parameter[[A]]) -> void {};
173         $LocalVariable[[FN]](12312);
174       }
175     )cpp",
176       R"cpp(
177       void $Function[[foo]](int);
178       void $Function[[Gah]]();
179       void $Function[[foo]]() {
180         auto $LocalVariable[[Bou]] = $Function[[Gah]];
181       }
182       struct $Class[[A]] {
183         void $Method[[abc]]();
184       };
185     )cpp",
186       R"cpp(
187       namespace $Namespace[[abc]] {
188         template<typename $TemplateParameter[[T]]>
189         struct $Class[[A]] {
190           $TemplateParameter[[T]] $Field[[t]];
191         };
192       }
193       template<typename $TemplateParameter[[T]]>
194       struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
195         typename $TemplateParameter[[T]]::$DependentType[[A]]* $Field[[D]];
196       };
197       $Namespace[[abc]]::$Class[[A]]<int> $Variable[[AA]];
198       typedef $Namespace[[abc]]::$Class[[A]]<int> $Class[[AAA]];
199       struct $Class[[B]] {
200         $Class[[B]]();
201         ~$Class[[B]]();
202         void operator<<($Class[[B]]);
203         $Class[[AAA]] $Field[[AA]];
204       };
205       $Class[[B]]::$Class[[B]]() {}
206       $Class[[B]]::~$Class[[B]]() {}
207       void $Function[[f]] () {
208         $Class[[B]] $LocalVariable[[BB]] = $Class[[B]]();
209         $LocalVariable[[BB]].~$Class[[B]]();
210         $Class[[B]]();
211       }
212     )cpp",
213       R"cpp(
214       enum class $Enum[[E]] {
215         $EnumConstant[[A]],
216         $EnumConstant[[B]],
217       };
218       enum $Enum[[EE]] {
219         $EnumConstant[[Hi]],
220       };
221       struct $Class[[A]] {
222         $Enum[[E]] $Field[[EEE]];
223         $Enum[[EE]] $Field[[EEEE]];
224       };
225       int $Variable[[I]] = $EnumConstant[[Hi]];
226       $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
227     )cpp",
228       R"cpp(
229       namespace $Namespace[[abc]] {
230         namespace {}
231         namespace $Namespace[[bcd]] {
232           struct $Class[[A]] {};
233           namespace $Namespace[[cde]] {
234             struct $Class[[A]] {
235               enum class $Enum[[B]] {
236                 $EnumConstant[[Hi]],
237               };
238             };
239           }
240         }
241       }
242       using namespace $Namespace[[abc]]::$Namespace[[bcd]];
243       namespace $Namespace[[vwz]] =
244             $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
245       $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
246       $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
247             $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
248       ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
249       ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
250     )cpp",
251       R"cpp(
252       struct $Class[[D]] {
253         double $Field[[C]];
254       };
255       struct $Class[[A]] {
256         double $Field[[B]];
257         $Class[[D]] $Field[[E]];
258         static double $StaticField[[S]];
259         static void $StaticMethod[[bar]]() {}
260         void $Method[[foo]]() {
261           $Field[[B]] = 123;
262           this->$Field[[B]] = 156;
263           this->$Method[[foo]]();
264           $Method[[foo]]();
265           $StaticMethod[[bar]]();
266           $StaticField[[S]] = 90.1;
267         }
268       };
269       void $Function[[foo]]() {
270         $Class[[A]] $LocalVariable[[AA]];
271         $LocalVariable[[AA]].$Field[[B]] += 2;
272         $LocalVariable[[AA]].$Method[[foo]]();
273         $LocalVariable[[AA]].$Field[[E]].$Field[[C]];
274         $Class[[A]]::$StaticField[[S]] = 90;
275       }
276     )cpp",
277       R"cpp(
278       struct $Class[[AA]] {
279         int $Field[[A]];
280       };
281       int $Variable[[B]];
282       $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
283     )cpp",
284       R"cpp(
285       namespace $Namespace[[a]] {
286         struct $Class[[A]] {};
287         typedef char $Primitive[[C]];
288       }
289       typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
290       using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
291       enum class $Enum[[E]] {};
292       typedef $Enum[[E]] $Enum[[C]];
293       typedef $Enum[[C]] $Enum[[CC]];
294       using $Enum[[CD]] = $Enum[[CC]];
295       $Enum[[CC]] $Function[[f]]($Class[[B]]);
296       $Enum[[CD]] $Function[[f]]($Class[[BB]]);
297       typedef $Namespace[[a]]::$Primitive[[C]] $Primitive[[PC]];
298       typedef float $Primitive[[F]];
299     )cpp",
300       R"cpp(
301       template<typename $TemplateParameter[[T]], typename = void>
302       class $Class[[A]] {
303         $TemplateParameter[[T]] $Field[[AA]];
304         $TemplateParameter[[T]] $Method[[foo]]();
305       };
306       template<class $TemplateParameter[[TT]]>
307       class $Class[[B]] {
308         $Class[[A]]<$TemplateParameter[[TT]]> $Field[[AA]];
309       };
310       template<class $TemplateParameter[[TT]], class $TemplateParameter[[GG]]>
311       class $Class[[BB]] {};
312       template<class $TemplateParameter[[T]]>
313       class $Class[[BB]]<$TemplateParameter[[T]], int> {};
314       template<class $TemplateParameter[[T]]>
315       class $Class[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
316 
317       template<template<class> class $TemplateParameter[[T]], class $TemplateParameter[[C]]>
318       $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function[[f]]();
319 
320       template<typename>
321       class $Class[[Foo]] {};
322 
323       template<typename $TemplateParameter[[T]]>
324       void $Function[[foo]]($TemplateParameter[[T]] ...);
325     )cpp",
326       R"cpp(
327       template <class $TemplateParameter[[T]]>
328       struct $Class[[Tmpl]] {$TemplateParameter[[T]] $Field[[x]] = 0;};
329       extern template struct $Class[[Tmpl]]<float>;
330       template struct $Class[[Tmpl]]<double>;
331     )cpp",
332       // This test is to guard against highlightings disappearing when using
333       // conversion operators as their behaviour in the clang AST differ from
334       // other CXXMethodDecls.
335       R"cpp(
336       class $Class[[Foo]] {};
337       struct $Class[[Bar]] {
338         explicit operator $Class[[Foo]]*() const;
339         explicit operator int() const;
340         operator $Class[[Foo]]();
341       };
342       void $Function[[f]]() {
343         $Class[[Bar]] $LocalVariable[[B]];
344         $Class[[Foo]] $LocalVariable[[F]] = $LocalVariable[[B]];
345         $Class[[Foo]] *$LocalVariable[[FP]] = ($Class[[Foo]]*)$LocalVariable[[B]];
346         int $LocalVariable[[I]] = (int)$LocalVariable[[B]];
347       }
348     )cpp"
349       R"cpp(
350       struct $Class[[B]] {};
351       struct $Class[[A]] {
352         $Class[[B]] $Field[[BB]];
353         $Class[[A]] &operator=($Class[[A]] &&$Parameter[[O]]);
354       };
355 
356       $Class[[A]] &$Class[[A]]::operator=($Class[[A]] &&$Parameter[[O]]) = default;
357     )cpp",
358       R"cpp(
359       enum $Enum[[En]] {
360         $EnumConstant[[EC]],
361       };
362       class $Class[[Foo]] {};
363       class $Class[[Bar]] {
364       public:
365         $Class[[Foo]] $Field[[Fo]];
366         $Enum[[En]] $Field[[E]];
367         int $Field[[I]];
368         $Class[[Bar]] ($Class[[Foo]] $Parameter[[F]],
369                 $Enum[[En]] $Parameter[[E]])
370         : $Field[[Fo]] ($Parameter[[F]]), $Field[[E]] ($Parameter[[E]]),
371           $Field[[I]] (123) {}
372       };
373       class $Class[[Bar2]] : public $Class[[Bar]] {
374         $Class[[Bar2]]() : $Class[[Bar]]($Class[[Foo]](), $EnumConstant[[EC]]) {}
375       };
376     )cpp",
377       R"cpp(
378       enum $Enum[[E]] {
379         $EnumConstant[[E]],
380       };
381       class $Class[[Foo]] {};
382       $Enum[[auto]] $Variable[[AE]] = $Enum[[E]]::$EnumConstant[[E]];
383       $Class[[auto]] $Variable[[AF]] = $Class[[Foo]]();
384       $Class[[decltype]](auto) $Variable[[AF2]] = $Class[[Foo]]();
385       $Class[[auto]] *$Variable[[AFP]] = &$Variable[[AF]];
386       $Enum[[auto]] &$Variable[[AER]] = $Variable[[AE]];
387       $Primitive[[auto]] $Variable[[Form]] = 10.2 + 2 * 4;
388       $Primitive[[decltype]]($Variable[[Form]]) $Variable[[F]] = 10;
389       auto $Variable[[Fun]] = []()->void{};
390     )cpp",
391       R"cpp(
392       class $Class[[G]] {};
393       template<$Class[[G]] *$TemplateParameter[[U]]>
394       class $Class[[GP]] {};
395       template<$Class[[G]] &$TemplateParameter[[U]]>
396       class $Class[[GR]] {};
397       template<int *$TemplateParameter[[U]]>
398       class $Class[[IP]] {
399         void $Method[[f]]() {
400           *$TemplateParameter[[U]] += 5;
401         }
402       };
403       template<unsigned $TemplateParameter[[U]] = 2>
404       class $Class[[Foo]] {
405         void $Method[[f]]() {
406           for(int $LocalVariable[[I]] = 0;
407             $LocalVariable[[I]] < $TemplateParameter[[U]];) {}
408         }
409       };
410 
411       $Class[[G]] $Variable[[L]];
412       void $Function[[f]]() {
413         $Class[[Foo]]<123> $LocalVariable[[F]];
414         $Class[[GP]]<&$Variable[[L]]> $LocalVariable[[LL]];
415         $Class[[GR]]<$Variable[[L]]> $LocalVariable[[LLL]];
416       }
417     )cpp",
418       R"cpp(
419       template<typename $TemplateParameter[[T]],
420         void ($TemplateParameter[[T]]::*$TemplateParameter[[method]])(int)>
421       struct $Class[[G]] {
422         void $Method[[foo]](
423             $TemplateParameter[[T]] *$Parameter[[O]]) {
424           ($Parameter[[O]]->*$TemplateParameter[[method]])(10);
425         }
426       };
427       struct $Class[[F]] {
428         void $Method[[f]](int);
429       };
430       template<void (*$TemplateParameter[[Func]])()>
431       struct $Class[[A]] {
432         void $Method[[f]]() {
433           (*$TemplateParameter[[Func]])();
434         }
435       };
436 
437       void $Function[[foo]]() {
438         $Class[[F]] $LocalVariable[[FF]];
439         $Class[[G]]<$Class[[F]], &$Class[[F]]::$Method[[f]]> $LocalVariable[[GG]];
440         $LocalVariable[[GG]].$Method[[foo]](&$LocalVariable[[FF]]);
441         $Class[[A]]<$Function[[foo]]> $LocalVariable[[AA]];
442       }
443     )cpp",
444       // Tokens that share a source range but have conflicting Kinds are not
445       // highlighted.
446       R"cpp(
447       #define $Macro[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; }
448       #define $Macro[[DEF_CLASS]](T) class T {};
449       // Preamble ends.
450       $Macro[[DEF_MULTIPLE]](XYZ);
451       $Macro[[DEF_MULTIPLE]](XYZW);
452       $Macro[[DEF_CLASS]]($Class[[A]])
453       #define $Macro[[MACRO_CONCAT]](X, V, T) T foo##X = V
454       #define $Macro[[DEF_VAR]](X, V) int X = V
455       #define $Macro[[DEF_VAR_T]](T, X, V) T X = V
456       #define $Macro[[DEF_VAR_REV]](V, X) DEF_VAR(X, V)
457       #define $Macro[[CPY]](X) X
458       #define $Macro[[DEF_VAR_TYPE]](X, Y) X Y
459       #define $Macro[[SOME_NAME]] variable
460       #define $Macro[[SOME_NAME_SET]] variable2 = 123
461       #define $Macro[[INC_VAR]](X) X += 2
462       void $Function[[foo]]() {
463         $Macro[[DEF_VAR]]($LocalVariable[[X]],  123);
464         $Macro[[DEF_VAR_REV]](908, $LocalVariable[[XY]]);
465         int $Macro[[CPY]]( $LocalVariable[[XX]] );
466         $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable[[AA]]);
467         double $Macro[[SOME_NAME]];
468         int $Macro[[SOME_NAME_SET]];
469         $LocalVariable[[variable]] = 20.1;
470         $Macro[[MACRO_CONCAT]](var, 2, float);
471         $Macro[[DEF_VAR_T]]($Class[[A]], $Macro[[CPY]](
472               $Macro[[CPY]]($LocalVariable[[Nested]])),
473             $Macro[[CPY]]($Class[[A]]()));
474         $Macro[[INC_VAR]]($LocalVariable[[variable]]);
475       }
476       void $Macro[[SOME_NAME]]();
477       $Macro[[DEF_VAR]]($Variable[[MMMMM]], 567);
478       $Macro[[DEF_VAR_REV]](756, $Variable[[AB]]);
479 
480       #define $Macro[[CALL_FN]](F) F();
481       #define $Macro[[DEF_FN]](F) void F ()
482       $Macro[[DEF_FN]]($Function[[g]]) {
483         $Macro[[CALL_FN]]($Function[[foo]]);
484       }
485     )cpp",
486       R"cpp(
487       #define $Macro[[fail]](expr) expr
488       #define $Macro[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); }
489       // Preamble ends.
490       int $Variable[[x]];
491       int $Variable[[y]];
492       int $Function[[f]]();
493       void $Function[[foo]]() {
494         $Macro[[assert]]($Variable[[x]] != $Variable[[y]]);
495         $Macro[[assert]]($Variable[[x]] != $Function[[f]]());
496       }
497     )cpp",
498       // highlighting all macro references
499       R"cpp(
500       #ifndef $Macro[[name]]
501       #define $Macro[[name]]
502       #endif
503 
504       #define $Macro[[test]]
505       #undef $Macro[[test]]
506 $InactiveCode[[]]      #ifdef $Macro[[test]]
507 $InactiveCode[[]]      #endif
508 
509 $InactiveCode[[]]      #if defined($Macro[[test]])
510 $InactiveCode[[]]      #endif
511     )cpp",
512       R"cpp(
513       struct $Class[[S]] {
514         float $Field[[Value]];
515         $Class[[S]] *$Field[[Next]];
516       };
517       $Class[[S]] $Variable[[Global]][2] = {$Class[[S]](), $Class[[S]]()};
518       void $Function[[f]]($Class[[S]] $Parameter[[P]]) {
519         int $LocalVariable[[A]][2] = {1,2};
520         auto [$Variable[[B1]], $Variable[[B2]]] = $LocalVariable[[A]];
521         auto [$Variable[[G1]], $Variable[[G2]]] = $Variable[[Global]];
522         $Class[[auto]] [$Variable[[P1]], $Variable[[P2]]] = $Parameter[[P]];
523         // Highlights references to BindingDecls.
524         $Variable[[B1]]++;
525       }
526     )cpp",
527       R"cpp(
528       template<class $TemplateParameter[[T]]>
529       class $Class[[A]] {
530         using $TemplateParameter[[TemplateParam1]] = $TemplateParameter[[T]];
531         typedef $TemplateParameter[[T]] $TemplateParameter[[TemplateParam2]];
532         using $Primitive[[IntType]] = int;
533 
534         using $Typedef[[Pointer]] = $TemplateParameter[[T]] *;
535         using $Typedef[[LVReference]] = $TemplateParameter[[T]] &;
536         using $Typedef[[RVReference]] = $TemplateParameter[[T]]&&;
537         using $Typedef[[Array]] = $TemplateParameter[[T]]*[3];
538         using $Typedef[[MemberPointer]] = int ($Class[[A]]::*)(int);
539 
540         // Use various previously defined typedefs in a function type.
541         void $Method[[func]](
542           $Typedef[[Pointer]], $Typedef[[LVReference]], $Typedef[[RVReference]],
543           $Typedef[[Array]], $Typedef[[MemberPointer]]);
544       };
545     )cpp",
546       R"cpp(
547       template <class $TemplateParameter[[T]]>
548       void $Function[[phase1]]($TemplateParameter[[T]]);
549       template <class $TemplateParameter[[T]]>
550       void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
551         $Function[[phase1]]($Parameter[[P]]);
552         $DependentName[[phase2]]($Parameter[[P]]);
553       }
554     )cpp",
555       R"cpp(
556       class $Class[[A]] {
557         template <class $TemplateParameter[[T]]>
558         void $Method[[bar]]($TemplateParameter[[T]]);
559       };
560 
561       template <class $TemplateParameter[[U]]>
562       void $Function[[foo]]($TemplateParameter[[U]] $Parameter[[P]]) {
563         $Class[[A]]().$Method[[bar]]($Parameter[[P]]);
564       }
565     )cpp",
566       R"cpp(
567       struct $Class[[A]] {
568         template <class $TemplateParameter[[T]]>
569         static void $StaticMethod[[foo]]($TemplateParameter[[T]]);
570       };
571 
572       template <class $TemplateParameter[[T]]>
573       struct $Class[[B]] {
574         void $Method[[bar]]() {
575           $Class[[A]]::$StaticMethod[[foo]]($TemplateParameter[[T]]());
576         }
577       };
578     )cpp",
579       R"cpp(
580       template <class $TemplateParameter[[T]]>
581       void $Function[[foo]](typename $TemplateParameter[[T]]::$DependentType[[Type]]
582                                             = $TemplateParameter[[T]]::$DependentName[[val]]);
583     )cpp",
584       R"cpp(
585       template <class $TemplateParameter[[T]]>
586       void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
587         $Parameter[[P]].$DependentName[[Field]];
588       }
589     )cpp",
590       R"cpp(
591       template <class $TemplateParameter[[T]]>
592       class $Class[[A]] {
593         int $Method[[foo]]() {
594           return $TemplateParameter[[T]]::$DependentName[[Field]];
595         }
596       };
597     )cpp",
598       // Highlighting the using decl as the underlying using shadow decl.
599       R"cpp(
600       void $Function[[foo]]();
601       using ::$Function[[foo]];
602     )cpp",
603       // Highlighting of template template arguments.
604       R"cpp(
605       template <template <class> class $TemplateParameter[[TT]],
606                 template <class> class ...$TemplateParameter[[TTs]]>
607       struct $Class[[Foo]] {
608         $Class[[Foo]]<$TemplateParameter[[TT]], $TemplateParameter[[TTs]]...>
609           *$Field[[t]];
610       };
611     )cpp",
612       // Inactive code highlighting
613       R"cpp(
614       // Code in the preamble.
615       // Inactive lines get an empty InactiveCode token at the beginning.
616 $InactiveCode[[]]      #ifdef $Macro[[test]]
617 $InactiveCode[[]]      #endif
618 
619       // A declaration to cause the preamble to end.
620       int $Variable[[EndPreamble]];
621 
622       // Code after the preamble.
623       // Code inside inactive blocks does not get regular highlightings
624       // because it's not part of the AST.
625 $InactiveCode[[]]      #ifdef $Macro[[test]]
626 $InactiveCode[[]]      int Inactive2;
627 $InactiveCode[[]]      #endif
628 
629       #ifndef $Macro[[test]]
630       int $Variable[[Active1]];
631       #endif
632 
633 $InactiveCode[[]]      #ifdef $Macro[[test]]
634 $InactiveCode[[]]      int Inactive3;
635 $InactiveCode[[]]      #else
636       int $Variable[[Active2]];
637       #endif
638     )cpp",
639       // Argument to 'sizeof...'
640       R"cpp(
641       template <typename... $TemplateParameter[[Elements]]>
642       struct $Class[[TupleSize]] {
643         static const int $StaticField[[size]] =
644 sizeof...($TemplateParameter[[Elements]]);
645       };
646     )cpp",
647       // More dependent types
648       R"cpp(
649       template <typename $TemplateParameter[[T]]>
650       struct $Class[[Waldo]] {
651         using $Typedef[[Location1]] = typename $TemplateParameter[[T]]
652             ::$DependentType[[Resolver]]::$DependentType[[Location]];
653         using $Typedef[[Location2]] = typename $TemplateParameter[[T]]
654             ::template $DependentType[[Resolver]]<$TemplateParameter[[T]]>
655             ::$DependentType[[Location]];
656         using $Typedef[[Location3]] = typename $TemplateParameter[[T]]
657             ::$DependentType[[Resolver]]
658             ::template $DependentType[[Location]]<$TemplateParameter[[T]]>;
659         static const int $StaticField[[Value]] = $TemplateParameter[[T]]
660             ::$DependentType[[Resolver]]::$DependentName[[Value]];
661       };
662     )cpp",
663       // Dependent name with heuristic target
664       R"cpp(
665       template <typename>
666       struct $Class[[Foo]] {
667         int $Field[[Waldo]];
668         void $Method[[bar]]() {
669           $Class[[Foo]]().$Field[[Waldo]];
670         }
671         template <typename $TemplateParameter[[U]]>
672         void $Method[[bar1]]() {
673           $Class[[Foo]]<$TemplateParameter[[U]]>().$Field[[Waldo]];
674         }
675       };
676     )cpp",
677       // Concepts
678       R"cpp(
679       template <typename $TemplateParameter[[T]]>
680       concept $Concept[[Fooable]] =
681           requires($TemplateParameter[[T]] $Parameter[[F]]) {
682             $Parameter[[F]].$DependentName[[foo]]();
683           };
684       template <typename $TemplateParameter[[T]]>
685           requires $Concept[[Fooable]]<$TemplateParameter[[T]]>
686       void $Function[[bar]]($TemplateParameter[[T]] $Parameter[[F]]) {
687         $Parameter[[F]].$DependentName[[foo]]();
688       }
689     )cpp"};
690   for (const auto &TestCase : TestCases) {
691     checkHighlightings(TestCase);
692   }
693 
694   checkHighlightings(R"cpp(
695     class $Class[[A]] {
696       #include "imp.h"
697     };
698   )cpp",
699                      {{"imp.h", R"cpp(
700     int someMethod();
701     void otherMethod();
702   )cpp"}});
703 
704   // A separate test for macros in headers.
705   checkHighlightings(R"cpp(
706     #include "imp.h"
707     $Macro[[DEFINE_Y]]
708     $Macro[[DXYZ_Y]](A);
709   )cpp",
710                      {{"imp.h", R"cpp(
711     #define DXYZ(X) class X {};
712     #define DXYZ_Y(Y) DXYZ(x##Y)
713     #define DEFINE(X) int X;
714     #define DEFINE_Y DEFINE(Y)
715   )cpp"}});
716 }
717 
TEST(SemanticHighlighting,GeneratesHighlightsWhenFileChange)718 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
719   class HighlightingsCounter : public ClangdServer::Callbacks {
720   public:
721     std::atomic<int> Count = {0};
722 
723     void onHighlightingsReady(
724         PathRef File, llvm::StringRef Version,
725         std::vector<HighlightingToken> Highlightings) override {
726       ++Count;
727     }
728   };
729 
730   auto FooCpp = testPath("foo.cpp");
731   MockFS FS;
732   FS.Files[FooCpp] = "";
733 
734   MockCompilationDatabase MCD;
735   HighlightingsCounter Counter;
736   ClangdServer Server(MCD, FS, ClangdServer::optsForTest(), &Counter);
737   Server.addDocument(FooCpp, "int a;");
738   ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for server";
739   ASSERT_EQ(Counter.Count, 1);
740 }
741 
742 // Ranges are highlighted as variables, unless highlighted as $Function etc.
tokens(llvm::StringRef MarkedText)743 std::vector<HighlightingToken> tokens(llvm::StringRef MarkedText) {
744   Annotations A(MarkedText);
745   std::vector<HighlightingToken> Results;
746   for (const Range& R : A.ranges())
747     Results.push_back({HighlightingKind::Variable, R});
748   for (unsigned I = 0; I < static_cast<unsigned>(HighlightingKind::LastKind); ++I) {
749     HighlightingKind Kind = static_cast<HighlightingKind>(I);
750     for (const Range& R : A.ranges(llvm::to_string(Kind)))
751       Results.push_back({Kind, R});
752   }
753   llvm::sort(Results);
754   return Results;
755 }
756 
TEST(SemanticHighlighting,toSemanticTokens)757 TEST(SemanticHighlighting, toSemanticTokens) {
758   auto Results = toSemanticTokens(tokens(R"(
759  [[blah]]
760 
761     $Function[[big]] [[bang]]
762   )"));
763 
764   ASSERT_THAT(Results, SizeIs(3));
765   EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
766   EXPECT_EQ(Results[0].deltaLine, 1u);
767   EXPECT_EQ(Results[0].deltaStart, 1u);
768   EXPECT_EQ(Results[0].length, 4u);
769 
770   EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function));
771   EXPECT_EQ(Results[1].deltaLine, 2u);
772   EXPECT_EQ(Results[1].deltaStart, 4u);
773   EXPECT_EQ(Results[1].length, 3u);
774 
775   EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable));
776   EXPECT_EQ(Results[2].deltaLine, 0u);
777   EXPECT_EQ(Results[2].deltaStart, 4u);
778   EXPECT_EQ(Results[2].length, 4u);
779 }
780 
TEST(SemanticHighlighting,diffSemanticTokens)781 TEST(SemanticHighlighting, diffSemanticTokens) {
782   auto Before = toSemanticTokens(tokens(R"(
783     [[foo]] [[bar]] [[baz]]
784     [[one]] [[two]] [[three]]
785   )"));
786   EXPECT_THAT(diffTokens(Before, Before), IsEmpty());
787 
788   auto After = toSemanticTokens(tokens(R"(
789     [[foo]] [[hello]] [[world]] [[baz]]
790     [[one]] [[two]] [[three]]
791   )"));
792 
793   // Replace [bar, baz] with [hello, world, baz]
794   auto Diff = diffTokens(Before, After);
795   ASSERT_THAT(Diff, SizeIs(1));
796   EXPECT_EQ(1u, Diff.front().startToken);
797   EXPECT_EQ(2u, Diff.front().deleteTokens);
798   ASSERT_THAT(Diff.front().tokens, SizeIs(3));
799   // hello
800   EXPECT_EQ(0u, Diff.front().tokens[0].deltaLine);
801   EXPECT_EQ(4u, Diff.front().tokens[0].deltaStart);
802   EXPECT_EQ(5u, Diff.front().tokens[0].length);
803   // world
804   EXPECT_EQ(0u, Diff.front().tokens[1].deltaLine);
805   EXPECT_EQ(6u, Diff.front().tokens[1].deltaStart);
806   EXPECT_EQ(5u, Diff.front().tokens[1].length);
807   // baz
808   EXPECT_EQ(0u, Diff.front().tokens[2].deltaLine);
809   EXPECT_EQ(6u, Diff.front().tokens[2].deltaStart);
810   EXPECT_EQ(3u, Diff.front().tokens[2].length);
811 }
812 
TEST(SemanticHighlighting,toTheiaSemanticHighlightingInformation)813 TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
814   auto CreatePosition = [](int Line, int Character) -> Position {
815     Position Pos;
816     Pos.line = Line;
817     Pos.character = Character;
818     return Pos;
819   };
820 
821   std::vector<LineHighlightings> Tokens{
822       {3,
823        {{HighlightingKind::Variable,
824          Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
825         {HighlightingKind::Function,
826          Range{CreatePosition(3, 4), CreatePosition(3, 7)}}},
827        /* IsInactive = */ false},
828       {1,
829        {{HighlightingKind::Variable,
830          Range{CreatePosition(1, 1), CreatePosition(1, 5)}}},
831        /* IsInactive = */ true}};
832   std::vector<TheiaSemanticHighlightingInformation> ActualResults =
833       toTheiaSemanticHighlightingInformation(Tokens);
834   std::vector<TheiaSemanticHighlightingInformation> ExpectedResults = {
835       {3, "AAAACAAEAAAAAAAEAAMAAw=="}, {1, "AAAAAQAEAAA="}};
836   EXPECT_EQ(ActualResults, ExpectedResults);
837 }
838 
TEST(SemanticHighlighting,HighlightingDiffer)839 TEST(SemanticHighlighting, HighlightingDiffer) {
840   struct {
841     llvm::StringRef OldCode;
842     llvm::StringRef NewCode;
843   } TestCases[]{{
844                     R"(
845         $Variable[[A]]
846         $Class[[B]]
847         $Function[[C]]
848       )",
849                     R"(
850         $Variable[[A]]
851         $Class[[D]]
852         $Function[[C]]
853       )"},
854                 {
855                     R"(
856         $Class[[C]]
857         $Field[[F]]
858         $Variable[[V]]
859         $Class[[C]] $Variable[[V]] $Field[[F]]
860       )",
861                     R"(
862         $Class[[C]]
863         $Field[[F]]
864        ^$Function[[F]]
865         $Class[[C]] $Variable[[V]] $Field[[F]]
866       )"},
867                 {
868                     R"(
869 
870         $Class[[A]]
871         $Variable[[A]]
872       )",
873                     R"(
874 
875        ^
876        ^$Class[[A]]
877        ^$Variable[[A]]
878       )"},
879                 {
880                     R"(
881         $Class[[C]]
882         $Field[[F]]
883         $Variable[[V]]
884         $Class[[C]] $Variable[[V]] $Field[[F]]
885       )",
886                     R"(
887         $Class[[C]]
888        ^
889        ^
890         $Class[[C]] $Variable[[V]] $Field[[F]]
891       )"},
892                 {
893                     R"(
894         $Class[[A]]
895         $Variable[[A]]
896         $Variable[[A]]
897       )",
898                     R"(
899         $Class[[A]]
900        ^$Variable[[AA]]
901         $Variable[[A]]
902       )"},
903                 {
904                     R"(
905         $Class[[A]]
906         $Variable[[A]]
907       )",
908                     R"(
909         $Class[[A]]
910         $Variable[[A]]
911        ^$Class[[A]]
912        ^$Variable[[A]]
913       )"},
914                 {
915                     R"(
916         $Variable[[A]]
917         $Variable[[A]]
918         $Variable[[A]]
919       )",
920                     R"(
921        ^$Class[[A]]
922        ^$Class[[A]]
923        ^$Class[[A]]
924       )"}};
925 
926   for (const auto &Test : TestCases)
927     checkDiffedHighlights(Test.OldCode, Test.NewCode);
928 }
929 
TEST(SemanticHighlighting,DiffBeyondTheEndOfFile)930 TEST(SemanticHighlighting, DiffBeyondTheEndOfFile) {
931   llvm::StringRef OldCode =
932       R"(
933         $Class[[A]]
934         $Variable[[A]]
935         $Class[[A]]
936         $Variable[[A]]
937       )";
938   llvm::StringRef NewCode =
939       R"(
940         $Class[[A]] // line 1
941         $Variable[[A]] // line 2
942       )";
943 
944   Annotations OldTest(OldCode);
945   Annotations NewTest(NewCode);
946   std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
947   std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
948 
949   auto ActualDiff = diffHighlightings(NewTokens, OldTokens);
950   EXPECT_THAT(ActualDiff,
951               testing::UnorderedElementsAre(
952                   testing::AllOf(LineNumber(3), EmptyHighlightings()),
953                   testing::AllOf(LineNumber(4), EmptyHighlightings())));
954 }
955 
956 } // namespace
957 } // namespace clangd
958 } // namespace clang
959