1 //===-- QualityTests.cpp ----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Evaluating scoring functions isn't a great fit for assert-based tests.
10 // For interesting cases, both exact scores and "X beats Y" are too brittle to
11 // make good hard assertions.
12 //
13 // Here we test the signal extraction and sanity-check that signals point in
14 // the right direction. This should be supplemented by quality metrics which
15 // we can compute from a corpus of queries and preferred rankings.
16 //
17 //===----------------------------------------------------------------------===//
18
19 #include "FileDistance.h"
20 #include "Quality.h"
21 #include "TestFS.h"
22 #include "TestTU.h"
23 #include "clang/AST/Decl.h"
24 #include "clang/AST/DeclCXX.h"
25 #include "clang/AST/Type.h"
26 #include "clang/Sema/CodeCompleteConsumer.h"
27 #include "llvm/Support/Casting.h"
28 #include "gmock/gmock.h"
29 #include "gtest/gtest.h"
30 #include <vector>
31
32 namespace clang {
33 namespace clangd {
34
35 // Force the unittest URI scheme to be linked,
36 static int LLVM_ATTRIBUTE_UNUSED UnittestSchemeAnchorDest =
37 UnittestSchemeAnchorSource;
38
39 namespace {
40
TEST(QualityTests,SymbolQualitySignalExtraction)41 TEST(QualityTests, SymbolQualitySignalExtraction) {
42 auto Header = TestTU::withHeaderCode(R"cpp(
43 int _X;
44
45 [[deprecated]]
46 int _f() { return _X; }
47
48 #define DECL_NAME(x, y) x##_##y##_Decl
49 #define DECL(x, y) class DECL_NAME(x, y) {};
50 DECL(X, Y); // X_Y_Decl
51 )cpp");
52
53 auto Symbols = Header.headerSymbols();
54 auto AST = Header.build();
55
56 SymbolQualitySignals Quality;
57 Quality.merge(findSymbol(Symbols, "_X"));
58 EXPECT_FALSE(Quality.Deprecated);
59 EXPECT_FALSE(Quality.ImplementationDetail);
60 EXPECT_TRUE(Quality.ReservedName);
61 EXPECT_EQ(Quality.References, SymbolQualitySignals().References);
62 EXPECT_EQ(Quality.Category, SymbolQualitySignals::Variable);
63
64 Quality.merge(findSymbol(Symbols, "X_Y_Decl"));
65 EXPECT_TRUE(Quality.ImplementationDetail);
66
67 Symbol F = findSymbol(Symbols, "_f");
68 F.References = 24; // TestTU doesn't count references, so fake it.
69 Quality = {};
70 Quality.merge(F);
71 EXPECT_TRUE(Quality.Deprecated);
72 EXPECT_FALSE(Quality.ReservedName);
73 EXPECT_EQ(Quality.References, 24u);
74 EXPECT_EQ(Quality.Category, SymbolQualitySignals::Function);
75
76 Quality = {};
77 Quality.merge(CodeCompletionResult(&findDecl(AST, "_f"), /*Priority=*/42));
78 EXPECT_TRUE(Quality.Deprecated);
79 EXPECT_FALSE(Quality.ReservedName);
80 EXPECT_EQ(Quality.References, SymbolQualitySignals().References);
81 EXPECT_EQ(Quality.Category, SymbolQualitySignals::Function);
82
83 Quality = {};
84 Quality.merge(CodeCompletionResult("if"));
85 EXPECT_EQ(Quality.Category, SymbolQualitySignals::Keyword);
86 }
87
TEST(QualityTests,SymbolRelevanceSignalExtraction)88 TEST(QualityTests, SymbolRelevanceSignalExtraction) {
89 TestTU Test;
90 Test.HeaderCode = R"cpp(
91 int header();
92 int header_main();
93
94 namespace hdr { class Bar {}; } // namespace hdr
95
96 #define DEFINE_FLAG(X) \
97 namespace flags { \
98 int FLAGS_##X; \
99 } \
100
101 DEFINE_FLAG(FOO)
102 )cpp";
103 Test.Code = R"cpp(
104 using hdr::Bar;
105
106 using flags::FLAGS_FOO;
107
108 int ::header_main() {}
109 int main();
110
111 [[deprecated]]
112 int deprecated() { return 0; }
113
114 namespace { struct X { void y() { int z; } }; }
115 struct S{};
116 )cpp";
117 auto AST = Test.build();
118
119 SymbolRelevanceSignals Relevance;
120 Relevance.merge(CodeCompletionResult(&findDecl(AST, "deprecated"),
121 /*Priority=*/42, nullptr, false,
122 /*Accessible=*/false));
123 EXPECT_EQ(Relevance.NameMatch, SymbolRelevanceSignals().NameMatch);
124 EXPECT_TRUE(Relevance.Forbidden);
125 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::GlobalScope);
126
127 Relevance = {};
128 Relevance.merge(CodeCompletionResult(&findDecl(AST, "main"), 42));
129 EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
130 << "Decl in current file";
131 Relevance = {};
132 Relevance.merge(CodeCompletionResult(&findDecl(AST, "header"), 42));
133 EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 0.6f) << "Decl from header";
134 Relevance = {};
135 Relevance.merge(CodeCompletionResult(&findDecl(AST, "header_main"), 42));
136 EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
137 << "Current file and header";
138
139 auto constructShadowDeclCompletionResult = [&](const std::string DeclName) {
140 auto *Shadow =
141 *dyn_cast<UsingDecl>(&findDecl(AST, [&](const NamedDecl &ND) {
142 if (const UsingDecl *Using = dyn_cast<UsingDecl>(&ND))
143 if (Using->shadow_size() &&
144 Using->getQualifiedNameAsString() == DeclName)
145 return true;
146 return false;
147 }))->shadow_begin();
148 CodeCompletionResult Result(Shadow->getTargetDecl(), 42);
149 Result.ShadowDecl = Shadow;
150 return Result;
151 };
152
153 Relevance = {};
154 Relevance.merge(constructShadowDeclCompletionResult("Bar"));
155 EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
156 << "Using declaration in main file";
157 Relevance.merge(constructShadowDeclCompletionResult("FLAGS_FOO"));
158 EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
159 << "Using declaration in main file";
160
161 Relevance = {};
162 Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "X"), 42));
163 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FileScope);
164 Relevance = {};
165 Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "y"), 42));
166 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::ClassScope);
167 Relevance = {};
168 Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "z"), 42));
169 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FunctionScope);
170 // The injected class name is treated as the outer class name.
171 Relevance = {};
172 Relevance.merge(CodeCompletionResult(&findDecl(AST, "S::S"), 42));
173 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::GlobalScope);
174
175 Relevance = {};
176 EXPECT_FALSE(Relevance.InBaseClass);
177 auto BaseMember = CodeCompletionResult(&findUnqualifiedDecl(AST, "y"), 42);
178 BaseMember.InBaseClass = true;
179 Relevance.merge(BaseMember);
180 EXPECT_TRUE(Relevance.InBaseClass);
181
182 auto Index = Test.index();
183 FuzzyFindRequest Req;
184 Req.Query = "X";
185 Req.AnyScope = true;
186 bool Matched = false;
187 Index->fuzzyFind(Req, [&](const Symbol &S) {
188 Matched = true;
189 Relevance = {};
190 Relevance.merge(S);
191 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FileScope);
192 });
193 EXPECT_TRUE(Matched);
194 }
195
196 // Do the signals move the scores in the direction we expect?
TEST(QualityTests,SymbolQualitySignalsSanity)197 TEST(QualityTests, SymbolQualitySignalsSanity) {
198 SymbolQualitySignals Default;
199 EXPECT_EQ(Default.evaluate(), 1);
200
201 SymbolQualitySignals Deprecated;
202 Deprecated.Deprecated = true;
203 EXPECT_LT(Deprecated.evaluate(), Default.evaluate());
204
205 SymbolQualitySignals ReservedName;
206 ReservedName.ReservedName = true;
207 EXPECT_LT(ReservedName.evaluate(), Default.evaluate());
208
209 SymbolQualitySignals ImplementationDetail;
210 ImplementationDetail.ImplementationDetail = true;
211 EXPECT_LT(ImplementationDetail.evaluate(), Default.evaluate());
212
213 SymbolQualitySignals WithReferences, ManyReferences;
214 WithReferences.References = 20;
215 ManyReferences.References = 1000;
216 EXPECT_GT(WithReferences.evaluate(), Default.evaluate());
217 EXPECT_GT(ManyReferences.evaluate(), WithReferences.evaluate());
218
219 SymbolQualitySignals Keyword, Variable, Macro, Constructor, Function,
220 Destructor, Operator;
221 Keyword.Category = SymbolQualitySignals::Keyword;
222 Variable.Category = SymbolQualitySignals::Variable;
223 Macro.Category = SymbolQualitySignals::Macro;
224 Constructor.Category = SymbolQualitySignals::Constructor;
225 Destructor.Category = SymbolQualitySignals::Destructor;
226 Destructor.Category = SymbolQualitySignals::Destructor;
227 Operator.Category = SymbolQualitySignals::Operator;
228 Function.Category = SymbolQualitySignals::Function;
229 EXPECT_GT(Variable.evaluate(), Default.evaluate());
230 EXPECT_GT(Keyword.evaluate(), Variable.evaluate());
231 EXPECT_LT(Macro.evaluate(), Default.evaluate());
232 EXPECT_LT(Operator.evaluate(), Default.evaluate());
233 EXPECT_LT(Constructor.evaluate(), Function.evaluate());
234 EXPECT_LT(Destructor.evaluate(), Constructor.evaluate());
235 }
236
TEST(QualityTests,SymbolRelevanceSignalsSanity)237 TEST(QualityTests, SymbolRelevanceSignalsSanity) {
238 SymbolRelevanceSignals Default;
239 EXPECT_EQ(Default.evaluate(), 1);
240
241 SymbolRelevanceSignals Forbidden;
242 Forbidden.Forbidden = true;
243 EXPECT_LT(Forbidden.evaluate(), Default.evaluate());
244
245 SymbolRelevanceSignals PoorNameMatch;
246 PoorNameMatch.NameMatch = 0.2f;
247 EXPECT_LT(PoorNameMatch.evaluate(), Default.evaluate());
248
249 SymbolRelevanceSignals WithSemaFileProximity;
250 WithSemaFileProximity.SemaFileProximityScore = 0.2f;
251 EXPECT_GT(WithSemaFileProximity.evaluate(), Default.evaluate());
252
253 ScopeDistance ScopeProximity({"x::y::"});
254
255 SymbolRelevanceSignals WithSemaScopeProximity;
256 WithSemaScopeProximity.ScopeProximityMatch = &ScopeProximity;
257 WithSemaScopeProximity.SemaSaysInScope = true;
258 EXPECT_GT(WithSemaScopeProximity.evaluate(), Default.evaluate());
259
260 SymbolRelevanceSignals WithIndexScopeProximity;
261 WithIndexScopeProximity.ScopeProximityMatch = &ScopeProximity;
262 WithIndexScopeProximity.SymbolScope = "x::";
263 EXPECT_GT(WithSemaScopeProximity.evaluate(), Default.evaluate());
264
265 SymbolRelevanceSignals IndexProximate;
266 IndexProximate.SymbolURI = "unittest:/foo/bar.h";
267 llvm::StringMap<SourceParams> ProxSources;
268 ProxSources.try_emplace(testPath("foo/baz.h"));
269 URIDistance Distance(ProxSources);
270 IndexProximate.FileProximityMatch = &Distance;
271 EXPECT_GT(IndexProximate.evaluate(), Default.evaluate());
272 SymbolRelevanceSignals IndexDistant = IndexProximate;
273 IndexDistant.SymbolURI = "unittest:/elsewhere/path.h";
274 EXPECT_GT(IndexProximate.evaluate(), IndexDistant.evaluate())
275 << IndexProximate << IndexDistant;
276 EXPECT_GT(IndexDistant.evaluate(), Default.evaluate());
277
278 SymbolRelevanceSignals Scoped;
279 Scoped.Scope = SymbolRelevanceSignals::FileScope;
280 EXPECT_LT(Scoped.evaluate(), Default.evaluate());
281 Scoped.Query = SymbolRelevanceSignals::CodeComplete;
282 EXPECT_GT(Scoped.evaluate(), Default.evaluate());
283
284 SymbolRelevanceSignals Instance;
285 Instance.IsInstanceMember = false;
286 EXPECT_EQ(Instance.evaluate(), Default.evaluate());
287 Instance.Context = CodeCompletionContext::CCC_DotMemberAccess;
288 EXPECT_LT(Instance.evaluate(), Default.evaluate());
289 Instance.IsInstanceMember = true;
290 EXPECT_EQ(Instance.evaluate(), Default.evaluate());
291
292 SymbolRelevanceSignals InBaseClass;
293 InBaseClass.InBaseClass = true;
294 EXPECT_LT(InBaseClass.evaluate(), Default.evaluate());
295
296 llvm::StringSet<> Words = {"one", "two", "three"};
297 SymbolRelevanceSignals WithoutMatchingWord;
298 WithoutMatchingWord.ContextWords = &Words;
299 WithoutMatchingWord.Name = "four";
300 EXPECT_EQ(WithoutMatchingWord.evaluate(), Default.evaluate());
301 SymbolRelevanceSignals WithMatchingWord;
302 WithMatchingWord.ContextWords = &Words;
303 WithMatchingWord.Name = "TheTwoTowers";
304 EXPECT_GT(WithMatchingWord.evaluate(), Default.evaluate());
305 }
306
TEST(QualityTests,ScopeProximity)307 TEST(QualityTests, ScopeProximity) {
308 SymbolRelevanceSignals Relevance;
309 ScopeDistance ScopeProximity({"x::y::z::", "x::", "llvm::", ""});
310 Relevance.ScopeProximityMatch = &ScopeProximity;
311
312 Relevance.SymbolScope = "other::";
313 float NotMatched = Relevance.evaluate();
314
315 Relevance.SymbolScope = "";
316 float Global = Relevance.evaluate();
317 EXPECT_GT(Global, NotMatched);
318
319 Relevance.SymbolScope = "llvm::";
320 float NonParent = Relevance.evaluate();
321 EXPECT_GT(NonParent, Global);
322
323 Relevance.SymbolScope = "x::";
324 float GrandParent = Relevance.evaluate();
325 EXPECT_GT(GrandParent, Global);
326
327 Relevance.SymbolScope = "x::y::";
328 float Parent = Relevance.evaluate();
329 EXPECT_GT(Parent, GrandParent);
330
331 Relevance.SymbolScope = "x::y::z::";
332 float Enclosing = Relevance.evaluate();
333 EXPECT_GT(Enclosing, Parent);
334 }
335
TEST(QualityTests,SortText)336 TEST(QualityTests, SortText) {
337 EXPECT_LT(sortText(std::numeric_limits<float>::infinity()),
338 sortText(1000.2f));
339 EXPECT_LT(sortText(1000.2f), sortText(1));
340 EXPECT_LT(sortText(1), sortText(0.3f));
341 EXPECT_LT(sortText(0.3f), sortText(0));
342 EXPECT_LT(sortText(0), sortText(-10));
343 EXPECT_LT(sortText(-10), sortText(-std::numeric_limits<float>::infinity()));
344
345 EXPECT_LT(sortText(1, "z"), sortText(0, "a"));
346 EXPECT_LT(sortText(0, "a"), sortText(0, "z"));
347 }
348
TEST(QualityTests,NoBoostForClassConstructor)349 TEST(QualityTests, NoBoostForClassConstructor) {
350 auto Header = TestTU::withHeaderCode(R"cpp(
351 class Foo {
352 public:
353 Foo(int);
354 };
355 )cpp");
356 auto Symbols = Header.headerSymbols();
357 auto AST = Header.build();
358
359 const NamedDecl *Foo = &findDecl(AST, "Foo");
360 SymbolRelevanceSignals Cls;
361 Cls.merge(CodeCompletionResult(Foo, /*Priority=*/0));
362
363 const NamedDecl *CtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
364 return (ND.getQualifiedNameAsString() == "Foo::Foo") &&
365 isa<CXXConstructorDecl>(&ND);
366 });
367 SymbolRelevanceSignals Ctor;
368 Ctor.merge(CodeCompletionResult(CtorDecl, /*Priority=*/0));
369
370 EXPECT_EQ(Cls.Scope, SymbolRelevanceSignals::GlobalScope);
371 EXPECT_EQ(Ctor.Scope, SymbolRelevanceSignals::GlobalScope);
372 }
373
TEST(QualityTests,IsInstanceMember)374 TEST(QualityTests, IsInstanceMember) {
375 auto Header = TestTU::withHeaderCode(R"cpp(
376 class Foo {
377 public:
378 static void foo() {}
379
380 template <typename T> void tpl(T *t) {}
381
382 void bar() {}
383 };
384 )cpp");
385 auto Symbols = Header.headerSymbols();
386
387 SymbolRelevanceSignals Rel;
388 const Symbol &FooSym = findSymbol(Symbols, "Foo::foo");
389 Rel.merge(FooSym);
390 EXPECT_FALSE(Rel.IsInstanceMember);
391 const Symbol &BarSym = findSymbol(Symbols, "Foo::bar");
392 Rel.merge(BarSym);
393 EXPECT_TRUE(Rel.IsInstanceMember);
394
395 Rel.IsInstanceMember = false;
396 const Symbol &TplSym = findSymbol(Symbols, "Foo::tpl");
397 Rel.merge(TplSym);
398 EXPECT_TRUE(Rel.IsInstanceMember);
399
400 auto AST = Header.build();
401 const NamedDecl *Foo = &findDecl(AST, "Foo::foo");
402 const NamedDecl *Bar = &findDecl(AST, "Foo::bar");
403 const NamedDecl *Tpl = &findDecl(AST, "Foo::tpl");
404
405 Rel.IsInstanceMember = false;
406 Rel.merge(CodeCompletionResult(Foo, /*Priority=*/0));
407 EXPECT_FALSE(Rel.IsInstanceMember);
408 Rel.merge(CodeCompletionResult(Bar, /*Priority=*/0));
409 EXPECT_TRUE(Rel.IsInstanceMember);
410 Rel.IsInstanceMember = false;
411 Rel.merge(CodeCompletionResult(Tpl, /*Priority=*/0));
412 EXPECT_TRUE(Rel.IsInstanceMember);
413 }
414
TEST(QualityTests,ConstructorDestructor)415 TEST(QualityTests, ConstructorDestructor) {
416 auto Header = TestTU::withHeaderCode(R"cpp(
417 class Foo {
418 public:
419 Foo(int);
420 ~Foo();
421 };
422 )cpp");
423 auto Symbols = Header.headerSymbols();
424 auto AST = Header.build();
425
426 const NamedDecl *CtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
427 return (ND.getQualifiedNameAsString() == "Foo::Foo") &&
428 isa<CXXConstructorDecl>(&ND);
429 });
430 const NamedDecl *DtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
431 return (ND.getQualifiedNameAsString() == "Foo::~Foo") &&
432 isa<CXXDestructorDecl>(&ND);
433 });
434
435 SymbolQualitySignals CtorQ;
436 CtorQ.merge(CodeCompletionResult(CtorDecl, /*Priority=*/0));
437 EXPECT_EQ(CtorQ.Category, SymbolQualitySignals::Constructor);
438
439 CtorQ.Category = SymbolQualitySignals::Unknown;
440 const Symbol &CtorSym = findSymbol(Symbols, "Foo::Foo");
441 CtorQ.merge(CtorSym);
442 EXPECT_EQ(CtorQ.Category, SymbolQualitySignals::Constructor);
443
444 SymbolQualitySignals DtorQ;
445 DtorQ.merge(CodeCompletionResult(DtorDecl, /*Priority=*/0));
446 EXPECT_EQ(DtorQ.Category, SymbolQualitySignals::Destructor);
447 }
448
TEST(QualityTests,Operator)449 TEST(QualityTests, Operator) {
450 auto Header = TestTU::withHeaderCode(R"cpp(
451 class Foo {
452 public:
453 bool operator<(const Foo& f1);
454 };
455 )cpp");
456 auto AST = Header.build();
457
458 const NamedDecl *Operator = &findDecl(AST, [](const NamedDecl &ND) {
459 if (const auto *OD = dyn_cast<FunctionDecl>(&ND))
460 if (OD->isOverloadedOperator())
461 return true;
462 return false;
463 });
464 SymbolQualitySignals Q;
465 Q.merge(CodeCompletionResult(Operator, /*Priority=*/0));
466 EXPECT_EQ(Q.Category, SymbolQualitySignals::Operator);
467 }
468
TEST(QualityTests,ItemWithFixItsRankedDown)469 TEST(QualityTests, ItemWithFixItsRankedDown) {
470 CodeCompleteOptions Opts;
471 Opts.IncludeFixIts = true;
472
473 auto Header = TestTU::withHeaderCode(R"cpp(
474 int x;
475 )cpp");
476 auto AST = Header.build();
477
478 SymbolRelevanceSignals RelevanceWithFixIt;
479 RelevanceWithFixIt.merge(CodeCompletionResult(&findDecl(AST, "x"), 0, nullptr,
480 false, true, {FixItHint{}}));
481 EXPECT_TRUE(RelevanceWithFixIt.NeedsFixIts);
482
483 SymbolRelevanceSignals RelevanceWithoutFixIt;
484 RelevanceWithoutFixIt.merge(
485 CodeCompletionResult(&findDecl(AST, "x"), 0, nullptr, false, true, {}));
486 EXPECT_FALSE(RelevanceWithoutFixIt.NeedsFixIts);
487
488 EXPECT_LT(RelevanceWithFixIt.evaluate(), RelevanceWithoutFixIt.evaluate());
489 }
490
491 } // namespace
492 } // namespace clangd
493 } // namespace clang
494