1 //==- SemanticHighlightingTests.cpp - SemanticHighlighting tests-*- C++ -* -==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Annotations.h"
10 #include "ClangdServer.h"
11 #include "Protocol.h"
12 #include "SemanticHighlighting.h"
13 #include "SourceCode.h"
14 #include "TestFS.h"
15 #include "TestTU.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/Support/Error.h"
20 #include "llvm/Support/ScopedPrinter.h"
21 #include "gmock/gmock.h"
22 #include <algorithm>
23 
24 namespace clang {
25 namespace clangd {
26 namespace {
27 
28 using testing::IsEmpty;
29 using testing::SizeIs;
30 
31 MATCHER_P(LineNumber, L, "") { return arg.Line == L; }
32 MATCHER(EmptyHighlightings, "") { return arg.Tokens.empty(); }
33 
34 std::vector<HighlightingToken>
makeHighlightingTokens(llvm::ArrayRef<Range> Ranges,HighlightingKind Kind)35 makeHighlightingTokens(llvm::ArrayRef<Range> Ranges, HighlightingKind Kind) {
36   std::vector<HighlightingToken> Tokens(Ranges.size());
37   for (int I = 0, End = Ranges.size(); I < End; ++I) {
38     Tokens[I].R = Ranges[I];
39     Tokens[I].Kind = Kind;
40   }
41 
42   return Tokens;
43 }
44 
getExpectedTokens(Annotations & Test)45 std::vector<HighlightingToken> getExpectedTokens(Annotations &Test) {
46   static const std::map<HighlightingKind, std::string> KindToString{
47       {HighlightingKind::Variable, "Variable"},
48       {HighlightingKind::LocalVariable, "LocalVariable"},
49       {HighlightingKind::Parameter, "Parameter"},
50       {HighlightingKind::Function, "Function"},
51       {HighlightingKind::Class, "Class"},
52       {HighlightingKind::Enum, "Enum"},
53       {HighlightingKind::Namespace, "Namespace"},
54       {HighlightingKind::EnumConstant, "EnumConstant"},
55       {HighlightingKind::Field, "Field"},
56       {HighlightingKind::StaticField, "StaticField"},
57       {HighlightingKind::Method, "Method"},
58       {HighlightingKind::StaticMethod, "StaticMethod"},
59       {HighlightingKind::Typedef, "Typedef"},
60       {HighlightingKind::DependentType, "DependentType"},
61       {HighlightingKind::DependentName, "DependentName"},
62       {HighlightingKind::TemplateParameter, "TemplateParameter"},
63       {HighlightingKind::Concept, "Concept"},
64       {HighlightingKind::Primitive, "Primitive"},
65       {HighlightingKind::Macro, "Macro"}};
66   std::vector<HighlightingToken> ExpectedTokens;
67   for (const auto &KindString : KindToString) {
68     std::vector<HighlightingToken> Toks = makeHighlightingTokens(
69         Test.ranges(KindString.second), KindString.first);
70     ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
71   }
72   llvm::sort(ExpectedTokens);
73   return ExpectedTokens;
74 }
75 
76 /// Annotates the input code with provided semantic highlightings. Results look
77 /// something like:
78 ///   class $Class[[X]] {
79 ///     $Primitive[[int]] $Field[[a]] = 0;
80 ///   };
annotate(llvm::StringRef Input,llvm::ArrayRef<HighlightingToken> Tokens)81 std::string annotate(llvm::StringRef Input,
82                      llvm::ArrayRef<HighlightingToken> Tokens) {
83   assert(std::is_sorted(
84       Tokens.begin(), Tokens.end(),
85       [](const HighlightingToken &L, const HighlightingToken &R) {
86         return L.R.start < R.R.start;
87       }));
88 
89   std::string Result;
90   unsigned NextChar = 0;
91   for (auto &T : Tokens) {
92     unsigned StartOffset = llvm::cantFail(positionToOffset(Input, T.R.start));
93     unsigned EndOffset = llvm::cantFail(positionToOffset(Input, T.R.end));
94     assert(StartOffset <= EndOffset);
95     assert(NextChar <= StartOffset);
96 
97     Result += Input.substr(NextChar, StartOffset - NextChar);
98     Result += std::string(
99         llvm::formatv("${0}[[{1}]]", T.Kind,
100                       Input.substr(StartOffset, EndOffset - StartOffset)));
101     NextChar = EndOffset;
102   }
103   Result += Input.substr(NextChar);
104   return Result;
105 }
106 
checkHighlightings(llvm::StringRef Code,std::vector<std::pair<llvm::StringRef,llvm::StringRef>> AdditionalFiles={})107 void checkHighlightings(llvm::StringRef Code,
108                         std::vector<std::pair</*FileName*/ llvm::StringRef,
109                                               /*FileContent*/ llvm::StringRef>>
110                             AdditionalFiles = {}) {
111   Annotations Test(Code);
112   TestTU TU;
113   TU.Code = std::string(Test.code());
114 
115   // FIXME: Auto-completion in a template requires disabling delayed template
116   // parsing.
117   TU.ExtraArgs.push_back("-fno-delayed-template-parsing");
118   TU.ExtraArgs.push_back("-std=c++20");
119 
120   for (auto File : AdditionalFiles)
121     TU.AdditionalFiles.insert({File.first, std::string(File.second)});
122   auto AST = TU.build();
123 
124   EXPECT_EQ(Code, annotate(Test.code(), getSemanticHighlightings(AST)));
125 }
126 
127 // Any annotations in OldCode and NewCode are converted into their corresponding
128 // HighlightingToken. The tokens are diffed against each other. Any lines where
129 // the tokens should diff must be marked with a ^ somewhere on that line in
130 // NewCode. If there are diffs that aren't marked with ^ the test fails. The
131 // test also fails if there are lines marked with ^ that don't differ.
checkDiffedHighlights(llvm::StringRef OldCode,llvm::StringRef NewCode)132 void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
133   Annotations OldTest(OldCode);
134   Annotations NewTest(NewCode);
135   std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
136   std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
137 
138   llvm::DenseMap<int, std::vector<HighlightingToken>> ExpectedLines;
139   for (const Position &Point : NewTest.points()) {
140     ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
141                                // are inserted on these lines later.
142   }
143   std::vector<LineHighlightings> ExpectedLinePairHighlighting;
144   for (const HighlightingToken &Token : NewTokens) {
145     auto It = ExpectedLines.find(Token.R.start.line);
146     if (It != ExpectedLines.end())
147       It->second.push_back(Token);
148   }
149   for (auto &LineTokens : ExpectedLines)
150     ExpectedLinePairHighlighting.push_back(
151         {LineTokens.first, LineTokens.second, /*IsInactive = */ false});
152 
153   std::vector<LineHighlightings> ActualDiffed =
154       diffHighlightings(NewTokens, OldTokens);
155   EXPECT_THAT(ActualDiffed,
156               testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting))
157       << OldCode;
158 }
159 
TEST(SemanticHighlighting,GetsCorrectTokens)160 TEST(SemanticHighlighting, GetsCorrectTokens) {
161   const char *TestCases[] = {
162       R"cpp(
163       struct $Class[[AS]] {
164         double $Field[[SomeMember]];
165       };
166       struct {
167       } $Variable[[S]];
168       void $Function[[foo]](int $Parameter[[A]], $Class[[AS]] $Parameter[[As]]) {
169         $Primitive[[auto]] $LocalVariable[[VeryLongVariableName]] = 12312;
170         $Class[[AS]]     $LocalVariable[[AA]];
171         $Primitive[[auto]] $LocalVariable[[L]] = $LocalVariable[[AA]].$Field[[SomeMember]] + $Parameter[[A]];
172         auto $LocalVariable[[FN]] = [ $LocalVariable[[AA]]](int $Parameter[[A]]) -> void {};
173         $LocalVariable[[FN]](12312);
174       }
175     )cpp",
176       R"cpp(
177       void $Function[[foo]](int);
178       void $Function[[Gah]]();
179       void $Function[[foo]]() {
180         auto $LocalVariable[[Bou]] = $Function[[Gah]];
181       }
182       struct $Class[[A]] {
183         void $Method[[abc]]();
184       };
185     )cpp",
186       R"cpp(
187       namespace $Namespace[[abc]] {
188         template<typename $TemplateParameter[[T]]>
189         struct $Class[[A]] {
190           $TemplateParameter[[T]] $Field[[t]];
191         };
192       }
193       template<typename $TemplateParameter[[T]]>
194       struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
195         typename $TemplateParameter[[T]]::$DependentType[[A]]* $Field[[D]];
196       };
197       $Namespace[[abc]]::$Class[[A]]<int> $Variable[[AA]];
198       typedef $Namespace[[abc]]::$Class[[A]]<int> $Class[[AAA]];
199       struct $Class[[B]] {
200         $Class[[B]]();
201         ~$Class[[B]]();
202         void operator<<($Class[[B]]);
203         $Class[[AAA]] $Field[[AA]];
204       };
205       $Class[[B]]::$Class[[B]]() {}
206       $Class[[B]]::~$Class[[B]]() {}
207       void $Function[[f]] () {
208         $Class[[B]] $LocalVariable[[BB]] = $Class[[B]]();
209         $LocalVariable[[BB]].~$Class[[B]]();
210         $Class[[B]]();
211       }
212     )cpp",
213       R"cpp(
214       enum class $Enum[[E]] {
215         $EnumConstant[[A]],
216         $EnumConstant[[B]],
217       };
218       enum $Enum[[EE]] {
219         $EnumConstant[[Hi]],
220       };
221       struct $Class[[A]] {
222         $Enum[[E]] $Field[[EEE]];
223         $Enum[[EE]] $Field[[EEEE]];
224       };
225       int $Variable[[I]] = $EnumConstant[[Hi]];
226       $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
227     )cpp",
228       R"cpp(
229       namespace $Namespace[[abc]] {
230         namespace {}
231         namespace $Namespace[[bcd]] {
232           struct $Class[[A]] {};
233           namespace $Namespace[[cde]] {
234             struct $Class[[A]] {
235               enum class $Enum[[B]] {
236                 $EnumConstant[[Hi]],
237               };
238             };
239           }
240         }
241       }
242       using namespace $Namespace[[abc]]::$Namespace[[bcd]];
243       namespace $Namespace[[vwz]] =
244             $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
245       $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
246       $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
247             $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
248       ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
249       ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
250     )cpp",
251       R"cpp(
252       struct $Class[[D]] {
253         double $Field[[C]];
254       };
255       struct $Class[[A]] {
256         double $Field[[B]];
257         $Class[[D]] $Field[[E]];
258         static double $StaticField[[S]];
259         static void $StaticMethod[[bar]]() {}
260         void $Method[[foo]]() {
261           $Field[[B]] = 123;
262           this->$Field[[B]] = 156;
263           this->$Method[[foo]]();
264           $Method[[foo]]();
265           $StaticMethod[[bar]]();
266           $StaticField[[S]] = 90.1;
267         }
268       };
269       void $Function[[foo]]() {
270         $Class[[A]] $LocalVariable[[AA]];
271         $LocalVariable[[AA]].$Field[[B]] += 2;
272         $LocalVariable[[AA]].$Method[[foo]]();
273         $LocalVariable[[AA]].$Field[[E]].$Field[[C]];
274         $Class[[A]]::$StaticField[[S]] = 90;
275       }
276     )cpp",
277       R"cpp(
278       struct $Class[[AA]] {
279         int $Field[[A]];
280       };
281       int $Variable[[B]];
282       $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
283     )cpp",
284       R"cpp(
285       namespace $Namespace[[a]] {
286         struct $Class[[A]] {};
287         typedef char $Primitive[[C]];
288       }
289       typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
290       using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
291       enum class $Enum[[E]] {};
292       typedef $Enum[[E]] $Enum[[C]];
293       typedef $Enum[[C]] $Enum[[CC]];
294       using $Enum[[CD]] = $Enum[[CC]];
295       $Enum[[CC]] $Function[[f]]($Class[[B]]);
296       $Enum[[CD]] $Function[[f]]($Class[[BB]]);
297       typedef $Namespace[[a]]::$Primitive[[C]] $Primitive[[PC]];
298       typedef float $Primitive[[F]];
299     )cpp",
300       R"cpp(
301       template<typename $TemplateParameter[[T]], typename = void>
302       class $Class[[A]] {
303         $TemplateParameter[[T]] $Field[[AA]];
304         $TemplateParameter[[T]] $Method[[foo]]();
305       };
306       template<class $TemplateParameter[[TT]]>
307       class $Class[[B]] {
308         $Class[[A]]<$TemplateParameter[[TT]]> $Field[[AA]];
309       };
310       template<class $TemplateParameter[[TT]], class $TemplateParameter[[GG]]>
311       class $Class[[BB]] {};
312       template<class $TemplateParameter[[T]]>
313       class $Class[[BB]]<$TemplateParameter[[T]], int> {};
314       template<class $TemplateParameter[[T]]>
315       class $Class[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
316 
317       template<template<class> class $TemplateParameter[[T]], class $TemplateParameter[[C]]>
318       $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function[[f]]();
319 
320       template<typename>
321       class $Class[[Foo]] {};
322 
323       template<typename $TemplateParameter[[T]]>
324       void $Function[[foo]]($TemplateParameter[[T]] ...);
325     )cpp",
326       R"cpp(
327       template <class $TemplateParameter[[T]]>
328       struct $Class[[Tmpl]] {$TemplateParameter[[T]] $Field[[x]] = 0;};
329       extern template struct $Class[[Tmpl]]<float>;
330       template struct $Class[[Tmpl]]<double>;
331     )cpp",
332       // This test is to guard against highlightings disappearing when using
333       // conversion operators as their behaviour in the clang AST differ from
334       // other CXXMethodDecls.
335       R"cpp(
336       class $Class[[Foo]] {};
337       struct $Class[[Bar]] {
338         explicit operator $Class[[Foo]]*() const;
339         explicit operator int() const;
340         operator $Class[[Foo]]();
341       };
342       void $Function[[f]]() {
343         $Class[[Bar]] $LocalVariable[[B]];
344         $Class[[Foo]] $LocalVariable[[F]] = $LocalVariable[[B]];
345         $Class[[Foo]] *$LocalVariable[[FP]] = ($Class[[Foo]]*)$LocalVariable[[B]];
346         int $LocalVariable[[I]] = (int)$LocalVariable[[B]];
347       }
348     )cpp",
349       R"cpp(
350       struct $Class[[B]] {};
351       struct $Class[[A]] {
352         $Class[[B]] $Field[[BB]];
353         $Class[[A]] &operator=($Class[[A]] &&$Parameter[[O]]);
354       };
355 
356       $Class[[A]] &$Class[[A]]::operator=($Class[[A]] &&$Parameter[[O]]) = default;
357     )cpp",
358       R"cpp(
359       enum $Enum[[En]] {
360         $EnumConstant[[EC]],
361       };
362       class $Class[[Foo]] {};
363       class $Class[[Bar]] {
364       public:
365         $Class[[Foo]] $Field[[Fo]];
366         $Enum[[En]] $Field[[E]];
367         int $Field[[I]];
368         $Class[[Bar]] ($Class[[Foo]] $Parameter[[F]],
369                 $Enum[[En]] $Parameter[[E]])
370         : $Field[[Fo]] ($Parameter[[F]]), $Field[[E]] ($Parameter[[E]]),
371           $Field[[I]] (123) {}
372       };
373       class $Class[[Bar2]] : public $Class[[Bar]] {
374         $Class[[Bar2]]() : $Class[[Bar]]($Class[[Foo]](), $EnumConstant[[EC]]) {}
375       };
376     )cpp",
377       R"cpp(
378       enum $Enum[[E]] {
379         $EnumConstant[[E]],
380       };
381       class $Class[[Foo]] {};
382       $Enum[[auto]] $Variable[[AE]] = $Enum[[E]]::$EnumConstant[[E]];
383       $Class[[auto]] $Variable[[AF]] = $Class[[Foo]]();
384       $Class[[decltype]](auto) $Variable[[AF2]] = $Class[[Foo]]();
385       $Class[[auto]] *$Variable[[AFP]] = &$Variable[[AF]];
386       $Enum[[auto]] &$Variable[[AER]] = $Variable[[AE]];
387       $Primitive[[auto]] $Variable[[Form]] = 10.2 + 2 * 4;
388       $Primitive[[decltype]]($Variable[[Form]]) $Variable[[F]] = 10;
389       auto $Variable[[Fun]] = []()->void{};
390     )cpp",
391       R"cpp(
392       class $Class[[G]] {};
393       template<$Class[[G]] *$TemplateParameter[[U]]>
394       class $Class[[GP]] {};
395       template<$Class[[G]] &$TemplateParameter[[U]]>
396       class $Class[[GR]] {};
397       template<int *$TemplateParameter[[U]]>
398       class $Class[[IP]] {
399         void $Method[[f]]() {
400           *$TemplateParameter[[U]] += 5;
401         }
402       };
403       template<unsigned $TemplateParameter[[U]] = 2>
404       class $Class[[Foo]] {
405         void $Method[[f]]() {
406           for(int $LocalVariable[[I]] = 0;
407             $LocalVariable[[I]] < $TemplateParameter[[U]];) {}
408         }
409       };
410 
411       $Class[[G]] $Variable[[L]];
412       void $Function[[f]]() {
413         $Class[[Foo]]<123> $LocalVariable[[F]];
414         $Class[[GP]]<&$Variable[[L]]> $LocalVariable[[LL]];
415         $Class[[GR]]<$Variable[[L]]> $LocalVariable[[LLL]];
416       }
417     )cpp",
418       R"cpp(
419       template<typename $TemplateParameter[[T]],
420         void ($TemplateParameter[[T]]::*$TemplateParameter[[method]])(int)>
421       struct $Class[[G]] {
422         void $Method[[foo]](
423             $TemplateParameter[[T]] *$Parameter[[O]]) {
424           ($Parameter[[O]]->*$TemplateParameter[[method]])(10);
425         }
426       };
427       struct $Class[[F]] {
428         void $Method[[f]](int);
429       };
430       template<void (*$TemplateParameter[[Func]])()>
431       struct $Class[[A]] {
432         void $Method[[f]]() {
433           (*$TemplateParameter[[Func]])();
434         }
435       };
436 
437       void $Function[[foo]]() {
438         $Class[[F]] $LocalVariable[[FF]];
439         $Class[[G]]<$Class[[F]], &$Class[[F]]::$Method[[f]]> $LocalVariable[[GG]];
440         $LocalVariable[[GG]].$Method[[foo]](&$LocalVariable[[FF]]);
441         $Class[[A]]<$Function[[foo]]> $LocalVariable[[AA]];
442       }
443     )cpp",
444       // Tokens that share a source range but have conflicting Kinds are not
445       // highlighted.
446       R"cpp(
447       #define $Macro[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; }
448       #define $Macro[[DEF_CLASS]](T) class T {};
449       // Preamble ends.
450       $Macro[[DEF_MULTIPLE]](XYZ);
451       $Macro[[DEF_MULTIPLE]](XYZW);
452       $Macro[[DEF_CLASS]]($Class[[A]])
453       #define $Macro[[MACRO_CONCAT]](X, V, T) T foo##X = V
454       #define $Macro[[DEF_VAR]](X, V) int X = V
455       #define $Macro[[DEF_VAR_T]](T, X, V) T X = V
456       #define $Macro[[DEF_VAR_REV]](V, X) DEF_VAR(X, V)
457       #define $Macro[[CPY]](X) X
458       #define $Macro[[DEF_VAR_TYPE]](X, Y) X Y
459       #define $Macro[[SOME_NAME]] variable
460       #define $Macro[[SOME_NAME_SET]] variable2 = 123
461       #define $Macro[[INC_VAR]](X) X += 2
462       void $Function[[foo]]() {
463         $Macro[[DEF_VAR]]($LocalVariable[[X]],  123);
464         $Macro[[DEF_VAR_REV]](908, $LocalVariable[[XY]]);
465         int $Macro[[CPY]]( $LocalVariable[[XX]] );
466         $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable[[AA]]);
467         double $Macro[[SOME_NAME]];
468         int $Macro[[SOME_NAME_SET]];
469         $LocalVariable[[variable]] = 20.1;
470         $Macro[[MACRO_CONCAT]](var, 2, float);
471         $Macro[[DEF_VAR_T]]($Class[[A]], $Macro[[CPY]](
472               $Macro[[CPY]]($LocalVariable[[Nested]])),
473             $Macro[[CPY]]($Class[[A]]()));
474         $Macro[[INC_VAR]]($LocalVariable[[variable]]);
475       }
476       void $Macro[[SOME_NAME]]();
477       $Macro[[DEF_VAR]]($Variable[[MMMMM]], 567);
478       $Macro[[DEF_VAR_REV]](756, $Variable[[AB]]);
479 
480       #define $Macro[[CALL_FN]](F) F();
481       #define $Macro[[DEF_FN]](F) void F ()
482       $Macro[[DEF_FN]]($Function[[g]]) {
483         $Macro[[CALL_FN]]($Function[[foo]]);
484       }
485     )cpp",
486       R"cpp(
487       #define $Macro[[fail]](expr) expr
488       #define $Macro[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); }
489       // Preamble ends.
490       int $Variable[[x]];
491       int $Variable[[y]];
492       int $Function[[f]]();
493       void $Function[[foo]]() {
494         $Macro[[assert]]($Variable[[x]] != $Variable[[y]]);
495         $Macro[[assert]]($Variable[[x]] != $Function[[f]]());
496       }
497     )cpp",
498       // highlighting all macro references
499       R"cpp(
500       #ifndef $Macro[[name]]
501       #define $Macro[[name]]
502       #endif
503 
504       #define $Macro[[test]]
505       #undef $Macro[[test]]
506 $InactiveCode[[#ifdef test]]
507 $InactiveCode[[#endif]]
508 
509 $InactiveCode[[#if defined(test)]]
510 $InactiveCode[[#endif]]
511     )cpp",
512       R"cpp(
513       struct $Class[[S]] {
514         float $Field[[Value]];
515         $Class[[S]] *$Field[[Next]];
516       };
517       $Class[[S]] $Variable[[Global]][2] = {$Class[[S]](), $Class[[S]]()};
518       auto [$Variable[[G1]], $Variable[[G2]]] = $Variable[[Global]];
519       void $Function[[f]]($Class[[S]] $Parameter[[P]]) {
520         int $LocalVariable[[A]][2] = {1,2};
521         auto [$LocalVariable[[B1]], $LocalVariable[[B2]]] = $LocalVariable[[A]];
522         auto [$LocalVariable[[G1]], $LocalVariable[[G2]]] = $Variable[[Global]];
523         $Class[[auto]] [$LocalVariable[[P1]], $LocalVariable[[P2]]] = $Parameter[[P]];
524         // Highlights references to BindingDecls.
525         $LocalVariable[[B1]]++;
526       }
527     )cpp",
528       R"cpp(
529       template<class $TemplateParameter[[T]]>
530       class $Class[[A]] {
531         using $TemplateParameter[[TemplateParam1]] = $TemplateParameter[[T]];
532         typedef $TemplateParameter[[T]] $TemplateParameter[[TemplateParam2]];
533         using $Primitive[[IntType]] = int;
534 
535         using $Typedef[[Pointer]] = $TemplateParameter[[T]] *;
536         using $Typedef[[LVReference]] = $TemplateParameter[[T]] &;
537         using $Typedef[[RVReference]] = $TemplateParameter[[T]]&&;
538         using $Typedef[[Array]] = $TemplateParameter[[T]]*[3];
539         using $Typedef[[MemberPointer]] = int ($Class[[A]]::*)(int);
540 
541         // Use various previously defined typedefs in a function type.
542         void $Method[[func]](
543           $Typedef[[Pointer]], $Typedef[[LVReference]], $Typedef[[RVReference]],
544           $Typedef[[Array]], $Typedef[[MemberPointer]]);
545       };
546     )cpp",
547       R"cpp(
548       template <class $TemplateParameter[[T]]>
549       void $Function[[phase1]]($TemplateParameter[[T]]);
550       template <class $TemplateParameter[[T]]>
551       void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
552         $Function[[phase1]]($Parameter[[P]]);
553         $DependentName[[phase2]]($Parameter[[P]]);
554       }
555     )cpp",
556       R"cpp(
557       class $Class[[A]] {
558         template <class $TemplateParameter[[T]]>
559         void $Method[[bar]]($TemplateParameter[[T]]);
560       };
561 
562       template <class $TemplateParameter[[U]]>
563       void $Function[[foo]]($TemplateParameter[[U]] $Parameter[[P]]) {
564         $Class[[A]]().$Method[[bar]]($Parameter[[P]]);
565       }
566     )cpp",
567       R"cpp(
568       struct $Class[[A]] {
569         template <class $TemplateParameter[[T]]>
570         static void $StaticMethod[[foo]]($TemplateParameter[[T]]);
571       };
572 
573       template <class $TemplateParameter[[T]]>
574       struct $Class[[B]] {
575         void $Method[[bar]]() {
576           $Class[[A]]::$StaticMethod[[foo]]($TemplateParameter[[T]]());
577         }
578       };
579     )cpp",
580       R"cpp(
581       template <class $TemplateParameter[[T]]>
582       void $Function[[foo]](typename $TemplateParameter[[T]]::$DependentType[[Type]]
583                                             = $TemplateParameter[[T]]::$DependentName[[val]]);
584     )cpp",
585       R"cpp(
586       template <class $TemplateParameter[[T]]>
587       void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
588         $Parameter[[P]].$DependentName[[Field]];
589       }
590     )cpp",
591       R"cpp(
592       template <class $TemplateParameter[[T]]>
593       class $Class[[A]] {
594         int $Method[[foo]]() {
595           return $TemplateParameter[[T]]::$DependentName[[Field]];
596         }
597       };
598     )cpp",
599       // Highlighting the using decl as the underlying using shadow decl.
600       R"cpp(
601       void $Function[[foo]]();
602       using ::$Function[[foo]];
603     )cpp",
604       // Highlighting of template template arguments.
605       R"cpp(
606       template <template <class> class $TemplateParameter[[TT]],
607                 template <class> class ...$TemplateParameter[[TTs]]>
608       struct $Class[[Foo]] {
609         $Class[[Foo]]<$TemplateParameter[[TT]], $TemplateParameter[[TTs]]...>
610           *$Field[[t]];
611       };
612     )cpp",
613       // Inactive code highlighting
614       R"cpp(
615       // Code in the preamble.
616       // Inactive lines get an empty InactiveCode token at the beginning.
617 $InactiveCode[[#ifdef test]]
618 $InactiveCode[[#endif]]
619 
620       // A declaration to cause the preamble to end.
621       int $Variable[[EndPreamble]];
622 
623       // Code after the preamble.
624       // Code inside inactive blocks does not get regular highlightings
625       // because it's not part of the AST.
626 $InactiveCode[[#ifdef test]]
627 $InactiveCode[[int Inactive2;]]
628 $InactiveCode[[#endif]]
629 
630       #ifndef $Macro[[test]]
631       int $Variable[[Active1]];
632       #endif
633 
634 $InactiveCode[[#ifdef test]]
635 $InactiveCode[[int Inactive3;]]
636 $InactiveCode[[#else]]
637       int $Variable[[Active2]];
638       #endif
639     )cpp",
640       // Argument to 'sizeof...'
641       R"cpp(
642       template <typename... $TemplateParameter[[Elements]]>
643       struct $Class[[TupleSize]] {
644         static const int $StaticField[[size]] =
645 sizeof...($TemplateParameter[[Elements]]);
646       };
647     )cpp",
648       // More dependent types
649       R"cpp(
650       template <typename $TemplateParameter[[T]]>
651       struct $Class[[Waldo]] {
652         using $Typedef[[Location1]] = typename $TemplateParameter[[T]]
653             ::$DependentType[[Resolver]]::$DependentType[[Location]];
654         using $Typedef[[Location2]] = typename $TemplateParameter[[T]]
655             ::template $DependentType[[Resolver]]<$TemplateParameter[[T]]>
656             ::$DependentType[[Location]];
657         using $Typedef[[Location3]] = typename $TemplateParameter[[T]]
658             ::$DependentType[[Resolver]]
659             ::template $DependentType[[Location]]<$TemplateParameter[[T]]>;
660         static const int $StaticField[[Value]] = $TemplateParameter[[T]]
661             ::$DependentType[[Resolver]]::$DependentName[[Value]];
662       };
663     )cpp",
664       // Dependent name with heuristic target
665       R"cpp(
666       template <typename>
667       struct $Class[[Foo]] {
668         int $Field[[Waldo]];
669         void $Method[[bar]]() {
670           $Class[[Foo]]().$Field[[Waldo]];
671         }
672         template <typename $TemplateParameter[[U]]>
673         void $Method[[bar1]]() {
674           $Class[[Foo]]<$TemplateParameter[[U]]>().$Field[[Waldo]];
675         }
676       };
677     )cpp",
678       // Concepts
679       R"cpp(
680       template <typename $TemplateParameter[[T]]>
681       concept $Concept[[Fooable]] =
682           requires($TemplateParameter[[T]] $Parameter[[F]]) {
683             $Parameter[[F]].$DependentName[[foo]]();
684           };
685       template <typename $TemplateParameter[[T]]>
686           requires $Concept[[Fooable]]<$TemplateParameter[[T]]>
687       void $Function[[bar]]($TemplateParameter[[T]] $Parameter[[F]]) {
688         $Parameter[[F]].$DependentName[[foo]]();
689       }
690     )cpp",
691       // Dependent template name
692       R"cpp(
693       template <template <typename> class> struct $Class[[A]] {};
694       template <typename $TemplateParameter[[T]]>
695       using $Typedef[[W]] = $Class[[A]]<
696         $TemplateParameter[[T]]::template $DependentType[[Waldo]]
697       >;
698     )cpp"};
699   for (const auto &TestCase : TestCases) {
700     checkHighlightings(TestCase);
701   }
702 
703   checkHighlightings(R"cpp(
704     class $Class[[A]] {
705       #include "imp.h"
706     };
707   )cpp",
708                      {{"imp.h", R"cpp(
709     int someMethod();
710     void otherMethod();
711   )cpp"}});
712 
713   // A separate test for macros in headers.
714   checkHighlightings(R"cpp(
715     #include "imp.h"
716     $Macro[[DEFINE_Y]]
717     $Macro[[DXYZ_Y]](A);
718   )cpp",
719                      {{"imp.h", R"cpp(
720     #define DXYZ(X) class X {};
721     #define DXYZ_Y(Y) DXYZ(x##Y)
722     #define DEFINE(X) int X;
723     #define DEFINE_Y DEFINE(Y)
724   )cpp"}});
725 }
726 
TEST(SemanticHighlighting,GeneratesHighlightsWhenFileChange)727 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
728   class HighlightingsCounter : public ClangdServer::Callbacks {
729   public:
730     std::atomic<int> Count = {0};
731 
732     void onHighlightingsReady(
733         PathRef File, llvm::StringRef Version,
734         std::vector<HighlightingToken> Highlightings) override {
735       ++Count;
736     }
737   };
738 
739   auto FooCpp = testPath("foo.cpp");
740   MockFS FS;
741   FS.Files[FooCpp] = "";
742 
743   MockCompilationDatabase MCD;
744   HighlightingsCounter Counter;
745   ClangdServer Server(MCD, FS, ClangdServer::optsForTest(), &Counter);
746   Server.addDocument(FooCpp, "int a;");
747   ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for server";
748   ASSERT_EQ(Counter.Count, 1);
749 }
750 
751 // Ranges are highlighted as variables, unless highlighted as $Function etc.
tokens(llvm::StringRef MarkedText)752 std::vector<HighlightingToken> tokens(llvm::StringRef MarkedText) {
753   Annotations A(MarkedText);
754   std::vector<HighlightingToken> Results;
755   for (const Range& R : A.ranges())
756     Results.push_back({HighlightingKind::Variable, R});
757   for (unsigned I = 0; I < static_cast<unsigned>(HighlightingKind::LastKind); ++I) {
758     HighlightingKind Kind = static_cast<HighlightingKind>(I);
759     for (const Range& R : A.ranges(llvm::to_string(Kind)))
760       Results.push_back({Kind, R});
761   }
762   llvm::sort(Results);
763   return Results;
764 }
765 
TEST(SemanticHighlighting,toSemanticTokens)766 TEST(SemanticHighlighting, toSemanticTokens) {
767   auto Results = toSemanticTokens(tokens(R"(
768  [[blah]]
769 
770     $Function[[big]] [[bang]]
771   )"));
772 
773   ASSERT_THAT(Results, SizeIs(3));
774   EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
775   EXPECT_EQ(Results[0].deltaLine, 1u);
776   EXPECT_EQ(Results[0].deltaStart, 1u);
777   EXPECT_EQ(Results[0].length, 4u);
778 
779   EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function));
780   EXPECT_EQ(Results[1].deltaLine, 2u);
781   EXPECT_EQ(Results[1].deltaStart, 4u);
782   EXPECT_EQ(Results[1].length, 3u);
783 
784   EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable));
785   EXPECT_EQ(Results[2].deltaLine, 0u);
786   EXPECT_EQ(Results[2].deltaStart, 4u);
787   EXPECT_EQ(Results[2].length, 4u);
788 }
789 
TEST(SemanticHighlighting,diffSemanticTokens)790 TEST(SemanticHighlighting, diffSemanticTokens) {
791   auto Before = toSemanticTokens(tokens(R"(
792     [[foo]] [[bar]] [[baz]]
793     [[one]] [[two]] [[three]]
794   )"));
795   EXPECT_THAT(diffTokens(Before, Before), IsEmpty());
796 
797   auto After = toSemanticTokens(tokens(R"(
798     [[foo]] [[hello]] [[world]] [[baz]]
799     [[one]] [[two]] [[three]]
800   )"));
801 
802   // Replace [bar, baz] with [hello, world, baz]
803   auto Diff = diffTokens(Before, After);
804   ASSERT_THAT(Diff, SizeIs(1));
805   EXPECT_EQ(1u, Diff.front().startToken);
806   EXPECT_EQ(2u, Diff.front().deleteTokens);
807   ASSERT_THAT(Diff.front().tokens, SizeIs(3));
808   // hello
809   EXPECT_EQ(0u, Diff.front().tokens[0].deltaLine);
810   EXPECT_EQ(4u, Diff.front().tokens[0].deltaStart);
811   EXPECT_EQ(5u, Diff.front().tokens[0].length);
812   // world
813   EXPECT_EQ(0u, Diff.front().tokens[1].deltaLine);
814   EXPECT_EQ(6u, Diff.front().tokens[1].deltaStart);
815   EXPECT_EQ(5u, Diff.front().tokens[1].length);
816   // baz
817   EXPECT_EQ(0u, Diff.front().tokens[2].deltaLine);
818   EXPECT_EQ(6u, Diff.front().tokens[2].deltaStart);
819   EXPECT_EQ(3u, Diff.front().tokens[2].length);
820 }
821 
TEST(SemanticHighlighting,toTheiaSemanticHighlightingInformation)822 TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
823   auto CreatePosition = [](int Line, int Character) -> Position {
824     Position Pos;
825     Pos.line = Line;
826     Pos.character = Character;
827     return Pos;
828   };
829 
830   std::vector<LineHighlightings> Tokens{
831       {3,
832        {{HighlightingKind::Variable,
833          Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
834         {HighlightingKind::Function,
835          Range{CreatePosition(3, 4), CreatePosition(3, 7)}}},
836        /* IsInactive = */ false},
837       {1,
838        {{HighlightingKind::Variable,
839          Range{CreatePosition(1, 1), CreatePosition(1, 5)}}},
840        /* IsInactive = */ true}};
841   std::vector<TheiaSemanticHighlightingInformation> ActualResults =
842       toTheiaSemanticHighlightingInformation(Tokens);
843   std::vector<TheiaSemanticHighlightingInformation> ExpectedResults = {
844       {3, "AAAACAAEAAAAAAAEAAMAAw=="}, {1, "AAAAAQAEAAA="}};
845   EXPECT_EQ(ActualResults, ExpectedResults);
846 }
847 
TEST(SemanticHighlighting,HighlightingDiffer)848 TEST(SemanticHighlighting, HighlightingDiffer) {
849   struct {
850     llvm::StringRef OldCode;
851     llvm::StringRef NewCode;
852   } TestCases[]{{
853                     R"(
854         $Variable[[A]]
855         $Class[[B]]
856         $Function[[C]]
857       )",
858                     R"(
859         $Variable[[A]]
860         $Class[[D]]
861         $Function[[C]]
862       )"},
863                 {
864                     R"(
865         $Class[[C]]
866         $Field[[F]]
867         $Variable[[V]]
868         $Class[[C]] $Variable[[V]] $Field[[F]]
869       )",
870                     R"(
871         $Class[[C]]
872         $Field[[F]]
873        ^$Function[[F]]
874         $Class[[C]] $Variable[[V]] $Field[[F]]
875       )"},
876                 {
877                     R"(
878 
879         $Class[[A]]
880         $Variable[[A]]
881       )",
882                     R"(
883 
884        ^
885        ^$Class[[A]]
886        ^$Variable[[A]]
887       )"},
888                 {
889                     R"(
890         $Class[[C]]
891         $Field[[F]]
892         $Variable[[V]]
893         $Class[[C]] $Variable[[V]] $Field[[F]]
894       )",
895                     R"(
896         $Class[[C]]
897        ^
898        ^
899         $Class[[C]] $Variable[[V]] $Field[[F]]
900       )"},
901                 {
902                     R"(
903         $Class[[A]]
904         $Variable[[A]]
905         $Variable[[A]]
906       )",
907                     R"(
908         $Class[[A]]
909        ^$Variable[[AA]]
910         $Variable[[A]]
911       )"},
912                 {
913                     R"(
914         $Class[[A]]
915         $Variable[[A]]
916       )",
917                     R"(
918         $Class[[A]]
919         $Variable[[A]]
920        ^$Class[[A]]
921        ^$Variable[[A]]
922       )"},
923                 {
924                     R"(
925         $Variable[[A]]
926         $Variable[[A]]
927         $Variable[[A]]
928       )",
929                     R"(
930        ^$Class[[A]]
931        ^$Class[[A]]
932        ^$Class[[A]]
933       )"}};
934 
935   for (const auto &Test : TestCases)
936     checkDiffedHighlights(Test.OldCode, Test.NewCode);
937 }
938 
TEST(SemanticHighlighting,DiffBeyondTheEndOfFile)939 TEST(SemanticHighlighting, DiffBeyondTheEndOfFile) {
940   llvm::StringRef OldCode =
941       R"(
942         $Class[[A]]
943         $Variable[[A]]
944         $Class[[A]]
945         $Variable[[A]]
946       )";
947   llvm::StringRef NewCode =
948       R"(
949         $Class[[A]] // line 1
950         $Variable[[A]] // line 2
951       )";
952 
953   Annotations OldTest(OldCode);
954   Annotations NewTest(NewCode);
955   std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
956   std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
957 
958   auto ActualDiff = diffHighlightings(NewTokens, OldTokens);
959   EXPECT_THAT(ActualDiff,
960               testing::UnorderedElementsAre(
961                   testing::AllOf(LineNumber(3), EmptyHighlightings()),
962                   testing::AllOf(LineNumber(4), EmptyHighlightings())));
963 }
964 
965 } // namespace
966 } // namespace clangd
967 } // namespace clang
968