1 //===-- QualityTests.cpp ----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Evaluating scoring functions isn't a great fit for assert-based tests.
10 // For interesting cases, both exact scores and "X beats Y" are too brittle to
11 // make good hard assertions.
12 //
13 // Here we test the signal extraction and sanity-check that signals point in
14 // the right direction. This should be supplemented by quality metrics which
15 // we can compute from a corpus of queries and preferred rankings.
16 //
17 //===----------------------------------------------------------------------===//
18 
19 #include "FileDistance.h"
20 #include "Quality.h"
21 #include "TestFS.h"
22 #include "TestTU.h"
23 #include "clang/AST/Decl.h"
24 #include "clang/AST/DeclCXX.h"
25 #include "clang/AST/Type.h"
26 #include "clang/Sema/CodeCompleteConsumer.h"
27 #include "llvm/Support/Casting.h"
28 #include "gmock/gmock.h"
29 #include "gtest/gtest.h"
30 #include <vector>
31 
32 namespace clang {
33 namespace clangd {
34 
35 // Force the unittest URI scheme to be linked,
36 static int LLVM_ATTRIBUTE_UNUSED UnittestSchemeAnchorDest =
37     UnittestSchemeAnchorSource;
38 
39 namespace {
40 
TEST(QualityTests,SymbolQualitySignalExtraction)41 TEST(QualityTests, SymbolQualitySignalExtraction) {
42   auto Header = TestTU::withHeaderCode(R"cpp(
43     int _X;
44 
45     [[deprecated]]
46     int _f() { return _X; }
47 
48     #define DECL_NAME(x, y) x##_##y##_Decl
49     #define DECL(x, y) class DECL_NAME(x, y) {};
50     DECL(X, Y); // X_Y_Decl
51   )cpp");
52 
53   auto Symbols = Header.headerSymbols();
54   auto AST = Header.build();
55 
56   SymbolQualitySignals Quality;
57   Quality.merge(findSymbol(Symbols, "_X"));
58   EXPECT_FALSE(Quality.Deprecated);
59   EXPECT_FALSE(Quality.ImplementationDetail);
60   EXPECT_TRUE(Quality.ReservedName);
61   EXPECT_EQ(Quality.References, SymbolQualitySignals().References);
62   EXPECT_EQ(Quality.Category, SymbolQualitySignals::Variable);
63 
64   Quality.merge(findSymbol(Symbols, "X_Y_Decl"));
65   EXPECT_TRUE(Quality.ImplementationDetail);
66 
67   Symbol F = findSymbol(Symbols, "_f");
68   F.References = 24; // TestTU doesn't count references, so fake it.
69   Quality = {};
70   Quality.merge(F);
71   EXPECT_TRUE(Quality.Deprecated);
72   EXPECT_FALSE(Quality.ReservedName);
73   EXPECT_EQ(Quality.References, 24u);
74   EXPECT_EQ(Quality.Category, SymbolQualitySignals::Function);
75 
76   Quality = {};
77   Quality.merge(CodeCompletionResult(&findDecl(AST, "_f"), /*Priority=*/42));
78   EXPECT_TRUE(Quality.Deprecated);
79   EXPECT_FALSE(Quality.ReservedName);
80   EXPECT_EQ(Quality.References, SymbolQualitySignals().References);
81   EXPECT_EQ(Quality.Category, SymbolQualitySignals::Function);
82 
83   Quality = {};
84   Quality.merge(CodeCompletionResult("if"));
85   EXPECT_EQ(Quality.Category, SymbolQualitySignals::Keyword);
86 }
87 
TEST(QualityTests,SymbolRelevanceSignalExtraction)88 TEST(QualityTests, SymbolRelevanceSignalExtraction) {
89   TestTU Test;
90   Test.HeaderCode = R"cpp(
91   int header();
92   int header_main();
93 
94   namespace hdr { class Bar {}; } // namespace hdr
95 
96   #define DEFINE_FLAG(X) \
97   namespace flags { \
98   int FLAGS_##X; \
99   } \
100 
101   DEFINE_FLAG(FOO)
102   )cpp";
103   Test.Code = R"cpp(
104   using hdr::Bar;
105 
106   using flags::FLAGS_FOO;
107 
108   int ::header_main() {}
109   int main();
110 
111   [[deprecated]]
112   int deprecated() { return 0; }
113 
114   namespace { struct X { void y() { int z; } }; }
115   struct S{};
116   )cpp";
117   auto AST = Test.build();
118 
119   SymbolRelevanceSignals Relevance;
120   Relevance.merge(CodeCompletionResult(&findDecl(AST, "deprecated"),
121                                        /*Priority=*/42, nullptr, false,
122                                        /*Accessible=*/false));
123   EXPECT_EQ(Relevance.NameMatch, SymbolRelevanceSignals().NameMatch);
124   EXPECT_TRUE(Relevance.Forbidden);
125   EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::GlobalScope);
126 
127   Relevance = {};
128   Relevance.merge(CodeCompletionResult(&findDecl(AST, "main"), 42));
129   EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
130       << "Decl in current file";
131   Relevance = {};
132   Relevance.merge(CodeCompletionResult(&findDecl(AST, "header"), 42));
133   EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 0.6f) << "Decl from header";
134   Relevance = {};
135   Relevance.merge(CodeCompletionResult(&findDecl(AST, "header_main"), 42));
136   EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
137       << "Current file and header";
138 
139   auto constructShadowDeclCompletionResult = [&](const std::string DeclName) {
140     auto *Shadow =
141         *dyn_cast<UsingDecl>(&findDecl(AST, [&](const NamedDecl &ND) {
142            if (const UsingDecl *Using = dyn_cast<UsingDecl>(&ND))
143              if (Using->shadow_size() &&
144                  Using->getQualifiedNameAsString() == DeclName)
145                return true;
146            return false;
147          }))->shadow_begin();
148     CodeCompletionResult Result(Shadow->getTargetDecl(), 42);
149     Result.ShadowDecl = Shadow;
150     return Result;
151   };
152 
153   Relevance = {};
154   Relevance.merge(constructShadowDeclCompletionResult("Bar"));
155   EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
156       << "Using declaration in main file";
157   Relevance.merge(constructShadowDeclCompletionResult("FLAGS_FOO"));
158   EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
159       << "Using declaration in main file";
160 
161   Relevance = {};
162   Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "X"), 42));
163   EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FileScope);
164   Relevance = {};
165   Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "y"), 42));
166   EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::ClassScope);
167   Relevance = {};
168   Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "z"), 42));
169   EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FunctionScope);
170   // The injected class name is treated as the outer class name.
171   Relevance = {};
172   Relevance.merge(CodeCompletionResult(&findDecl(AST, "S::S"), 42));
173   EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::GlobalScope);
174 
175   Relevance = {};
176   EXPECT_FALSE(Relevance.InBaseClass);
177   auto BaseMember = CodeCompletionResult(&findUnqualifiedDecl(AST, "y"), 42);
178   BaseMember.InBaseClass = true;
179   Relevance.merge(BaseMember);
180   EXPECT_TRUE(Relevance.InBaseClass);
181 
182   auto Index = Test.index();
183   FuzzyFindRequest Req;
184   Req.Query = "X";
185   Req.AnyScope = true;
186   bool Matched = false;
187   Index->fuzzyFind(Req, [&](const Symbol &S) {
188     Matched = true;
189     Relevance = {};
190     Relevance.merge(S);
191     EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FileScope);
192   });
193   EXPECT_TRUE(Matched);
194 }
195 
196 // Do the signals move the scores in the direction we expect?
TEST(QualityTests,SymbolQualitySignalsSanity)197 TEST(QualityTests, SymbolQualitySignalsSanity) {
198   SymbolQualitySignals Default;
199   EXPECT_EQ(Default.evaluate(), 1);
200 
201   SymbolQualitySignals Deprecated;
202   Deprecated.Deprecated = true;
203   EXPECT_LT(Deprecated.evaluate(), Default.evaluate());
204 
205   SymbolQualitySignals ReservedName;
206   ReservedName.ReservedName = true;
207   EXPECT_LT(ReservedName.evaluate(), Default.evaluate());
208 
209   SymbolQualitySignals ImplementationDetail;
210   ImplementationDetail.ImplementationDetail = true;
211   EXPECT_LT(ImplementationDetail.evaluate(), Default.evaluate());
212 
213   SymbolQualitySignals WithReferences, ManyReferences;
214   WithReferences.References = 20;
215   ManyReferences.References = 1000;
216   EXPECT_GT(WithReferences.evaluate(), Default.evaluate());
217   EXPECT_GT(ManyReferences.evaluate(), WithReferences.evaluate());
218 
219   SymbolQualitySignals Keyword, Variable, Macro, Constructor, Function,
220       Destructor, Operator;
221   Keyword.Category = SymbolQualitySignals::Keyword;
222   Variable.Category = SymbolQualitySignals::Variable;
223   Macro.Category = SymbolQualitySignals::Macro;
224   Constructor.Category = SymbolQualitySignals::Constructor;
225   Destructor.Category = SymbolQualitySignals::Destructor;
226   Destructor.Category = SymbolQualitySignals::Destructor;
227   Operator.Category = SymbolQualitySignals::Operator;
228   Function.Category = SymbolQualitySignals::Function;
229   EXPECT_GT(Variable.evaluate(), Default.evaluate());
230   EXPECT_GT(Keyword.evaluate(), Variable.evaluate());
231   EXPECT_LT(Macro.evaluate(), Default.evaluate());
232   EXPECT_LT(Operator.evaluate(), Default.evaluate());
233   EXPECT_LT(Constructor.evaluate(), Function.evaluate());
234   EXPECT_LT(Destructor.evaluate(), Constructor.evaluate());
235 }
236 
TEST(QualityTests,SymbolRelevanceSignalsSanity)237 TEST(QualityTests, SymbolRelevanceSignalsSanity) {
238   SymbolRelevanceSignals Default;
239   EXPECT_EQ(Default.evaluate(), 1);
240 
241   SymbolRelevanceSignals Forbidden;
242   Forbidden.Forbidden = true;
243   EXPECT_LT(Forbidden.evaluate(), Default.evaluate());
244 
245   SymbolRelevanceSignals PoorNameMatch;
246   PoorNameMatch.NameMatch = 0.2f;
247   EXPECT_LT(PoorNameMatch.evaluate(), Default.evaluate());
248 
249   SymbolRelevanceSignals WithSemaFileProximity;
250   WithSemaFileProximity.SemaFileProximityScore = 0.2f;
251   EXPECT_GT(WithSemaFileProximity.evaluate(), Default.evaluate());
252 
253   ScopeDistance ScopeProximity({"x::y::"});
254 
255   SymbolRelevanceSignals WithSemaScopeProximity;
256   WithSemaScopeProximity.ScopeProximityMatch = &ScopeProximity;
257   WithSemaScopeProximity.SemaSaysInScope = true;
258   EXPECT_GT(WithSemaScopeProximity.evaluate(), Default.evaluate());
259 
260   SymbolRelevanceSignals WithIndexScopeProximity;
261   WithIndexScopeProximity.ScopeProximityMatch = &ScopeProximity;
262   WithIndexScopeProximity.SymbolScope = "x::";
263   EXPECT_GT(WithSemaScopeProximity.evaluate(), Default.evaluate());
264 
265   SymbolRelevanceSignals IndexProximate;
266   IndexProximate.SymbolURI = "unittest:/foo/bar.h";
267   llvm::StringMap<SourceParams> ProxSources;
268   ProxSources.try_emplace(testPath("foo/baz.h"));
269   URIDistance Distance(ProxSources);
270   IndexProximate.FileProximityMatch = &Distance;
271   EXPECT_GT(IndexProximate.evaluate(), Default.evaluate());
272   SymbolRelevanceSignals IndexDistant = IndexProximate;
273   IndexDistant.SymbolURI = "unittest:/elsewhere/path.h";
274   EXPECT_GT(IndexProximate.evaluate(), IndexDistant.evaluate())
275       << IndexProximate << IndexDistant;
276   EXPECT_GT(IndexDistant.evaluate(), Default.evaluate());
277 
278   SymbolRelevanceSignals Scoped;
279   Scoped.Scope = SymbolRelevanceSignals::FileScope;
280   EXPECT_LT(Scoped.evaluate(), Default.evaluate());
281   Scoped.Query = SymbolRelevanceSignals::CodeComplete;
282   EXPECT_GT(Scoped.evaluate(), Default.evaluate());
283 
284   SymbolRelevanceSignals Instance;
285   Instance.IsInstanceMember = false;
286   EXPECT_EQ(Instance.evaluate(), Default.evaluate());
287   Instance.Context = CodeCompletionContext::CCC_DotMemberAccess;
288   EXPECT_LT(Instance.evaluate(), Default.evaluate());
289   Instance.IsInstanceMember = true;
290   EXPECT_EQ(Instance.evaluate(), Default.evaluate());
291 
292   SymbolRelevanceSignals InBaseClass;
293   InBaseClass.InBaseClass = true;
294   EXPECT_LT(InBaseClass.evaluate(), Default.evaluate());
295 
296   llvm::StringSet<> Words = {"one", "two", "three"};
297   SymbolRelevanceSignals WithoutMatchingWord;
298   WithoutMatchingWord.ContextWords = &Words;
299   WithoutMatchingWord.Name = "four";
300   EXPECT_EQ(WithoutMatchingWord.evaluate(), Default.evaluate());
301   SymbolRelevanceSignals WithMatchingWord;
302   WithMatchingWord.ContextWords = &Words;
303   WithMatchingWord.Name = "TheTwoTowers";
304   EXPECT_GT(WithMatchingWord.evaluate(), Default.evaluate());
305 }
306 
TEST(QualityTests,ScopeProximity)307 TEST(QualityTests, ScopeProximity) {
308   SymbolRelevanceSignals Relevance;
309   ScopeDistance ScopeProximity({"x::y::z::", "x::", "llvm::", ""});
310   Relevance.ScopeProximityMatch = &ScopeProximity;
311 
312   Relevance.SymbolScope = "other::";
313   float NotMatched = Relevance.evaluate();
314 
315   Relevance.SymbolScope = "";
316   float Global = Relevance.evaluate();
317   EXPECT_GT(Global, NotMatched);
318 
319   Relevance.SymbolScope = "llvm::";
320   float NonParent = Relevance.evaluate();
321   EXPECT_GT(NonParent, Global);
322 
323   Relevance.SymbolScope = "x::";
324   float GrandParent = Relevance.evaluate();
325   EXPECT_GT(GrandParent, Global);
326 
327   Relevance.SymbolScope = "x::y::";
328   float Parent = Relevance.evaluate();
329   EXPECT_GT(Parent, GrandParent);
330 
331   Relevance.SymbolScope = "x::y::z::";
332   float Enclosing = Relevance.evaluate();
333   EXPECT_GT(Enclosing, Parent);
334 }
335 
TEST(QualityTests,SortText)336 TEST(QualityTests, SortText) {
337   EXPECT_LT(sortText(std::numeric_limits<float>::infinity()),
338             sortText(1000.2f));
339   EXPECT_LT(sortText(1000.2f), sortText(1));
340   EXPECT_LT(sortText(1), sortText(0.3f));
341   EXPECT_LT(sortText(0.3f), sortText(0));
342   EXPECT_LT(sortText(0), sortText(-10));
343   EXPECT_LT(sortText(-10), sortText(-std::numeric_limits<float>::infinity()));
344 
345   EXPECT_LT(sortText(1, "z"), sortText(0, "a"));
346   EXPECT_LT(sortText(0, "a"), sortText(0, "z"));
347 }
348 
TEST(QualityTests,NoBoostForClassConstructor)349 TEST(QualityTests, NoBoostForClassConstructor) {
350   auto Header = TestTU::withHeaderCode(R"cpp(
351     class Foo {
352     public:
353       Foo(int);
354     };
355   )cpp");
356   auto Symbols = Header.headerSymbols();
357   auto AST = Header.build();
358 
359   const NamedDecl *Foo = &findDecl(AST, "Foo");
360   SymbolRelevanceSignals Cls;
361   Cls.merge(CodeCompletionResult(Foo, /*Priority=*/0));
362 
363   const NamedDecl *CtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
364     return (ND.getQualifiedNameAsString() == "Foo::Foo") &&
365            isa<CXXConstructorDecl>(&ND);
366   });
367   SymbolRelevanceSignals Ctor;
368   Ctor.merge(CodeCompletionResult(CtorDecl, /*Priority=*/0));
369 
370   EXPECT_EQ(Cls.Scope, SymbolRelevanceSignals::GlobalScope);
371   EXPECT_EQ(Ctor.Scope, SymbolRelevanceSignals::GlobalScope);
372 }
373 
TEST(QualityTests,IsInstanceMember)374 TEST(QualityTests, IsInstanceMember) {
375   auto Header = TestTU::withHeaderCode(R"cpp(
376     class Foo {
377     public:
378       static void foo() {}
379 
380       template <typename T> void tpl(T *t) {}
381 
382       void bar() {}
383     };
384   )cpp");
385   auto Symbols = Header.headerSymbols();
386 
387   SymbolRelevanceSignals Rel;
388   const Symbol &FooSym = findSymbol(Symbols, "Foo::foo");
389   Rel.merge(FooSym);
390   EXPECT_FALSE(Rel.IsInstanceMember);
391   const Symbol &BarSym = findSymbol(Symbols, "Foo::bar");
392   Rel.merge(BarSym);
393   EXPECT_TRUE(Rel.IsInstanceMember);
394 
395   Rel.IsInstanceMember = false;
396   const Symbol &TplSym = findSymbol(Symbols, "Foo::tpl");
397   Rel.merge(TplSym);
398   EXPECT_TRUE(Rel.IsInstanceMember);
399 
400   auto AST = Header.build();
401   const NamedDecl *Foo = &findDecl(AST, "Foo::foo");
402   const NamedDecl *Bar = &findDecl(AST, "Foo::bar");
403   const NamedDecl *Tpl = &findDecl(AST, "Foo::tpl");
404 
405   Rel.IsInstanceMember = false;
406   Rel.merge(CodeCompletionResult(Foo, /*Priority=*/0));
407   EXPECT_FALSE(Rel.IsInstanceMember);
408   Rel.merge(CodeCompletionResult(Bar, /*Priority=*/0));
409   EXPECT_TRUE(Rel.IsInstanceMember);
410   Rel.IsInstanceMember = false;
411   Rel.merge(CodeCompletionResult(Tpl, /*Priority=*/0));
412   EXPECT_TRUE(Rel.IsInstanceMember);
413 }
414 
TEST(QualityTests,ConstructorDestructor)415 TEST(QualityTests, ConstructorDestructor) {
416   auto Header = TestTU::withHeaderCode(R"cpp(
417     class Foo {
418     public:
419       Foo(int);
420       ~Foo();
421     };
422   )cpp");
423   auto Symbols = Header.headerSymbols();
424   auto AST = Header.build();
425 
426   const NamedDecl *CtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
427     return (ND.getQualifiedNameAsString() == "Foo::Foo") &&
428            isa<CXXConstructorDecl>(&ND);
429   });
430   const NamedDecl *DtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
431     return (ND.getQualifiedNameAsString() == "Foo::~Foo") &&
432            isa<CXXDestructorDecl>(&ND);
433   });
434 
435   SymbolQualitySignals CtorQ;
436   CtorQ.merge(CodeCompletionResult(CtorDecl, /*Priority=*/0));
437   EXPECT_EQ(CtorQ.Category, SymbolQualitySignals::Constructor);
438 
439   CtorQ.Category = SymbolQualitySignals::Unknown;
440   const Symbol &CtorSym = findSymbol(Symbols, "Foo::Foo");
441   CtorQ.merge(CtorSym);
442   EXPECT_EQ(CtorQ.Category, SymbolQualitySignals::Constructor);
443 
444   SymbolQualitySignals DtorQ;
445   DtorQ.merge(CodeCompletionResult(DtorDecl, /*Priority=*/0));
446   EXPECT_EQ(DtorQ.Category, SymbolQualitySignals::Destructor);
447 }
448 
TEST(QualityTests,Operator)449 TEST(QualityTests, Operator) {
450   auto Header = TestTU::withHeaderCode(R"cpp(
451     class Foo {
452     public:
453       bool operator<(const Foo& f1);
454     };
455   )cpp");
456   auto AST = Header.build();
457 
458   const NamedDecl *Operator = &findDecl(AST, [](const NamedDecl &ND) {
459     if (const auto *OD = dyn_cast<FunctionDecl>(&ND))
460       if (OD->isOverloadedOperator())
461         return true;
462     return false;
463   });
464   SymbolQualitySignals Q;
465   Q.merge(CodeCompletionResult(Operator, /*Priority=*/0));
466   EXPECT_EQ(Q.Category, SymbolQualitySignals::Operator);
467 }
468 
TEST(QualityTests,ItemWithFixItsRankedDown)469 TEST(QualityTests, ItemWithFixItsRankedDown) {
470   CodeCompleteOptions Opts;
471   Opts.IncludeFixIts = true;
472 
473   auto Header = TestTU::withHeaderCode(R"cpp(
474         int x;
475       )cpp");
476   auto AST = Header.build();
477 
478   SymbolRelevanceSignals RelevanceWithFixIt;
479   RelevanceWithFixIt.merge(CodeCompletionResult(&findDecl(AST, "x"), 0, nullptr,
480                                                 false, true, {FixItHint{}}));
481   EXPECT_TRUE(RelevanceWithFixIt.NeedsFixIts);
482 
483   SymbolRelevanceSignals RelevanceWithoutFixIt;
484   RelevanceWithoutFixIt.merge(
485       CodeCompletionResult(&findDecl(AST, "x"), 0, nullptr, false, true, {}));
486   EXPECT_FALSE(RelevanceWithoutFixIt.NeedsFixIts);
487 
488   EXPECT_LT(RelevanceWithFixIt.evaluate(), RelevanceWithoutFixIt.evaluate());
489 }
490 
491 } // namespace
492 } // namespace clangd
493 } // namespace clang
494