1 /*
2 SPDX-FileCopyrightText: 2011-2012 Sven Brauch <svenbrauch@googlemail.com>
3
4 SPDX-License-Identifier: GPL-2.0-or-later
5 */
6
7 // Note to confused people reading this code: This is not the parser.
8 // It's just a minimalist helper class for code completion. The parser is in the parser/ directory.
9
10 #include "helpers.h"
11
12 #include <language/duchain/abstractfunctiondeclaration.h>
13 #include <language/duchain/duchainutils.h>
14 #include <language/duchain/ducontext.h>
15 #include <language/duchain/declaration.h>
16 #include <language/duchain/types/functiontype.h>
17 #include <language/duchain/types/integraltype.h>
18 #include <language/codecompletion/normaldeclarationcompletionitem.h>
19
20 #include <QStringList>
21 #include <QTextFormat>
22
23 #include <QDebug>
24 #include "codecompletiondebug.h"
25
26 #include "duchain/declarations/functiondeclaration.h"
27 #include "parser/codehelpers.h"
28
29 using namespace KDevelop;
30
31 namespace Python {
32
camelCaseToUnderscore(const QString & camelCase)33 QString camelCaseToUnderscore(const QString& camelCase)
34 {
35 QString underscore;
36 for ( int i = 0; i < camelCase.size(); i++ ) {
37 const QChar& c = camelCase.at(i);
38 if ( c.isUpper() && i != 0 ) {
39 underscore.append('_');
40 }
41 underscore.append(c.toLower());
42 }
43 return underscore;
44 }
45
identifierMatchQuality(const QString & identifier1_,const QString & identifier2_)46 int identifierMatchQuality(const QString& identifier1_, const QString& identifier2_)
47 {
48 QString identifier1 = camelCaseToUnderscore(identifier1_).toLower().replace('.', '_');
49 QString identifier2 = camelCaseToUnderscore(identifier2_).toLower().replace('.', '_');
50
51 if ( identifier1 == identifier2 ) {
52 return 3;
53 }
54 if ( identifier1.contains(identifier2) || identifier2.contains(identifier1) ) {
55 return 2;
56 }
57 QStringList parts1 = identifier1.split('_');
58 QStringList parts2 = identifier2.split('_');
59 parts1.removeAll("");
60 parts2.removeAll("");
61 parts1.removeDuplicates();
62 parts2.removeDuplicates();
63 if ( parts1.length() > 5 || parts2.length() > 5 ) {
64 // don't waste time comparing huge identifiers,
65 // the matching is probably pointless anyways for people using
66 // more than 5 words for their variable names
67 return 0;
68 }
69 foreach ( const QString& part1, parts1 ) {
70 foreach ( const QString& part2, parts2 ) {
71 // Don't take very short name parts into account,
72 // those are not very descriptive eventually
73 if ( part1.size() < 3 || part2.size() < 3 ) {
74 continue;
75 }
76 if ( part1 == part2 ) {
77 // partial match
78 return 1;
79 }
80 }
81 }
82 return 0;
83 }
84
85 typedef QPair<QString, ExpressionParser::Status> keyword;
86
87 static QList<keyword> supportedKeywords;
88 static QList<keyword> controlChars;
89 static QList<QString> miscKeywords;
90 static QList<QString> noCompletionKeywords;
91 static QMutex keywordPopulationLock;
92
93 // Keywords known to me:
94 // and del for is raise
95 // assert elif from lambda return
96 // break else global not try
97 // class except if or while
98 // continue exec import pass yield
99 // def finally in print with
100 // async await
101
ExpressionParser(QString code)102 ExpressionParser::ExpressionParser(QString code)
103 : m_code(code)
104 , m_cursorPositionInString(m_code.length())
105 {
106 keywordPopulationLock.lock();
107 if ( supportedKeywords.isEmpty() ) {
108 noCompletionKeywords << "break" << "class" << "continue" << "pass" << "try"
109 << "else" << "as" << "finally" << "global" << "lambda";
110 miscKeywords << "and" << "assert" << "del" << "elif" << "exec" << "if" << "is" << "not"
111 << "or" << "print" << "return" << "while" << "yield" << "with" << "await";
112 supportedKeywords << keyword("import", ExpressionParser::ImportFound);
113 supportedKeywords << keyword("from", ExpressionParser::FromFound);
114 supportedKeywords << keyword("raise", ExpressionParser::RaiseFound);
115 supportedKeywords << keyword("in", ExpressionParser::InFound);
116 supportedKeywords << keyword("for", ExpressionParser::ForFound);
117 supportedKeywords << keyword("class", ExpressionParser::ClassFound);
118 supportedKeywords << keyword("def", ExpressionParser::DefFound);
119 supportedKeywords << keyword("except", ExpressionParser::ExceptFound);
120 controlChars << keyword(":", ExpressionParser::ColonFound);
121 controlChars << keyword(",", ExpressionParser::CommaFound);
122 controlChars << keyword("(", ExpressionParser::InitializerFound);
123 controlChars << keyword("{", ExpressionParser::InitializerFound);
124 controlChars << keyword("[", ExpressionParser::InitializerFound);
125 controlChars << keyword(".", ExpressionParser::MemberAccessFound);
126 controlChars << keyword("=", ExpressionParser::EqualsFound);
127 }
128 keywordPopulationLock.unlock();
129 }
130
getRemainingCode()131 QString ExpressionParser::getRemainingCode()
132 {
133 return m_code.mid(0, m_cursorPositionInString);
134 }
135
getScannedCode()136 QString ExpressionParser::getScannedCode()
137 {
138 return m_code.mid(m_cursorPositionInString, m_code.length() - m_cursorPositionInString);
139 }
140
trailingWhitespace()141 int ExpressionParser::trailingWhitespace()
142 {
143 int ws = 0;
144 int index = m_cursorPositionInString - 1;
145 while ( index >= 0 ) {
146 if ( m_code.at(index).isSpace() ) {
147 ws++;
148 index --;
149 }
150 else {
151 break;
152 }
153 }
154 return ws;
155 }
156
reset()157 void ExpressionParser::reset()
158 {
159 m_cursorPositionInString = m_code.length();
160 }
161
skipUntilStatus(ExpressionParser::Status requestedStatus,bool * ok,int * expressionsSkipped)162 QString ExpressionParser::skipUntilStatus(ExpressionParser::Status requestedStatus, bool* ok, int* expressionsSkipped)
163 {
164 if ( expressionsSkipped ) {
165 *expressionsSkipped = 0;
166 }
167 QString lastExpression;
168 Status currentStatus = InvalidStatus;
169 while ( currentStatus != requestedStatus ) {
170 lastExpression = popExpression(¤tStatus);
171 qCDebug(KDEV_PYTHON_CODECOMPLETION) << lastExpression << currentStatus;
172 if ( currentStatus == NothingFound ) {
173 *ok = ( requestedStatus == NothingFound ); // ok exactly if the caller requested NothingFound as end status
174 return QString();
175 }
176 if ( expressionsSkipped && currentStatus == ExpressionFound ) {
177 *expressionsSkipped += 1;
178 }
179 }
180 *ok = true;
181 return lastExpression;
182 }
183
popAll()184 TokenList ExpressionParser::popAll()
185 {
186 Status currentStatus = InvalidStatus;
187 TokenList items;
188 while ( currentStatus != NothingFound ) {
189 QString result = popExpression(¤tStatus);
190 items << TokenListEntry(currentStatus, result, m_cursorPositionInString);
191 }
192 std::reverse(items.begin(), items.end());
193 return items;
194 }
195
endsWithSeperatedKeyword(const QString & str,const QString & shouldEndWith)196 bool endsWithSeperatedKeyword(const QString& str, const QString& shouldEndWith) {
197 bool endsWith = str.endsWith(shouldEndWith);
198 if ( ! endsWith ) {
199 return false;
200 }
201 int l = shouldEndWith.length();
202 if ( str.length() == l ) {
203 return true;
204 }
205 if ( str.right(l + 1).at(0).isSpace() ) {
206 return true;
207 }
208 return false;
209 }
210
popExpression(ExpressionParser::Status * status)211 QString ExpressionParser::popExpression(ExpressionParser::Status* status)
212 {
213 const auto remaining = getRemainingCode();
214 auto trimmed = remaining.trimmed();
215 auto operatingOn = trimmed.replace('\t', ' ');
216 bool lineIsEmpty = false;
217 for ( auto it = remaining.constEnd()-1; it != remaining.constEnd(); it-- ) {
218 if ( ! it->isSpace() ) {
219 break;
220 }
221 if ( *it == '\n' ) {
222 lineIsEmpty = true;
223 break;
224 }
225 }
226 if ( operatingOn.isEmpty() || lineIsEmpty ) {
227 m_cursorPositionInString = 0;
228 *status = NothingFound;
229 return QString();
230 }
231 bool lastCharIsSpace = getRemainingCode().right(1).at(0).isSpace();
232 m_cursorPositionInString -= trailingWhitespace();
233 if ( operatingOn.endsWith('(') ) {
234 qCDebug(KDEV_PYTHON_CODECOMPLETION) << "eventual call found";
235 m_cursorPositionInString -= 1;
236 *status = EventualCallFound;
237 return QString();
238 }
239 foreach ( const keyword& kw, controlChars ) {
240 if ( operatingOn.endsWith(kw.first) ) {
241 m_cursorPositionInString -= kw.first.length();
242 *status = kw.second;
243 return QString();
244 }
245 }
246 if ( lastCharIsSpace ) {
247 foreach ( const keyword& kw, supportedKeywords ) {
248 if ( endsWithSeperatedKeyword(operatingOn, kw.first) ) {
249 m_cursorPositionInString -= kw.first.length();
250 *status = kw.second;
251 return QString();
252 }
253 }
254 foreach ( const QString& kw, miscKeywords ) {
255 if ( endsWithSeperatedKeyword(operatingOn, kw) ) {
256 m_cursorPositionInString -= kw.length();
257 *status = MeaninglessKeywordFound;
258 return QString();
259 }
260 }
261 foreach ( const QString& kw, noCompletionKeywords ) {
262 if ( endsWithSeperatedKeyword(operatingOn, kw) ) {
263 m_cursorPositionInString -= kw.length();
264 *status = NoCompletionKeywordFound;
265 return QString();
266 }
267 }
268 }
269 // Otherwise, there's a real expression at the cursor, so scan it.
270 QStringList lines = operatingOn.split('\n');
271 Python::TrivialLazyLineFetcher f(lines);
272 int lastLine = lines.length()-1;
273 KTextEditor::Cursor startCursor;
274 QString expr = CodeHelpers::expressionUnderCursor(f, KTextEditor::Cursor(lastLine, f.fetchLine(lastLine).length() - 1),
275 startCursor, true);
276 if ( expr.isEmpty() ) {
277 *status = NothingFound;
278 }
279 else {
280 *status = ExpressionFound;
281 }
282 m_cursorPositionInString -= expr.length();
283 return expr;
284 }
285
286
287 // This is stolen from PHP. For credits, see helpers.cpp in PHP.
createArgumentList(Declaration * dec_,QString & ret,QList<QVariant> * highlighting,int atArg,bool includeTypes)288 void createArgumentList(Declaration* dec_, QString& ret, QList< QVariant >* highlighting, int atArg, bool includeTypes)
289 {
290 auto dec = dynamic_cast<Python::FunctionDeclaration*>(dec_);
291 if ( ! dec ) {
292 return;
293 }
294 int textFormatStart = 0;
295 QTextFormat normalFormat(QTextFormat::CharFormat);
296 QTextFormat highlightFormat(QTextFormat::CharFormat);
297 highlightFormat.setBackground(QColor::fromRgb(142, 186, 255));
298 highlightFormat.setProperty(QTextFormat::FontWeight, 99);
299
300 AbstractFunctionDeclaration* decl = dynamic_cast<AbstractFunctionDeclaration*>(dec);
301 FunctionType::Ptr functionType = dec->type<FunctionType>();
302
303 if (functionType && decl) {
304
305 QVector<Declaration*> parameters;
306 if (DUChainUtils::argumentContext(dec))
307 parameters = DUChainUtils::argumentContext(dec)->localDeclarations();
308
309 ret = '(';
310 bool first = true;
311 int num = 0;
312
313 bool skipFirst = false;
314 if ( dec->context() && dec->context()->type() == DUContext::Class && ! dec->isStatic() ) {
315 // the function is a class method, and its first argument is "self". Don't display that.
316 skipFirst = true;
317 }
318
319 uint defaultParamNum = 0;
320 int firstDefaultParam = parameters.count() - decl->defaultParametersSize() - skipFirst;
321
322 // disable highlighting when in default arguments, it doesn't make much sense then
323 bool disableHighlighting = false;
324
325 foreach(Declaration* dec, parameters) {
326 if ( skipFirst ) {
327 skipFirst = false;
328 continue;
329 }
330 // that has nothing to do with the skip, it's just for the comma
331 if (first)
332 first = false;
333 else
334 ret += ", ";
335
336 bool doHighlight = false;
337 QTextFormat doFormat;
338
339 if ( num == atArg - 1 )
340 doFormat = highlightFormat;
341 else
342 doFormat = normalFormat;
343
344 if ( num == firstDefaultParam ) {
345 ret += "[";
346 ++defaultParamNum;
347 disableHighlighting = true;
348 }
349
350 if ( ! disableHighlighting ) {
351 doHighlight = true;
352 }
353
354 if ( includeTypes ) {
355 if (num < functionType->arguments().count()) {
356 if (AbstractType::Ptr type = functionType->arguments().at(num)) {
357 if ( type->toString() != "<unknown>" ) {
358 ret += type->toString() + ' ';
359 }
360 }
361 }
362
363 if (doHighlight) {
364 if (highlighting && ret.length() != textFormatStart) {
365 //Add a default-highlighting for the passed text
366 *highlighting << QVariant(textFormatStart);
367 *highlighting << QVariant(ret.length() - textFormatStart);
368 *highlighting << QVariant(normalFormat);
369 textFormatStart = ret.length();
370 }
371 }
372 }
373
374
375 ret += dec->identifier().toString();
376
377 if (doHighlight) {
378 if (highlighting && ret.length() != textFormatStart) {
379 *highlighting << QVariant(textFormatStart + 1);
380 *highlighting << QVariant(ret.length() - textFormatStart - 1);
381 *highlighting << doFormat;
382 textFormatStart = ret.length();
383 }
384 }
385
386 ++num;
387 }
388 if ( defaultParamNum != 0 ) {
389 ret += "]";
390 }
391 ret += ')';
392
393 if (highlighting && ret.length() != textFormatStart) {
394 *highlighting << QVariant(textFormatStart);
395 *highlighting << QVariant(ret.length());
396 *highlighting << normalFormat;
397 textFormatStart = ret.length();
398 }
399 return;
400 }
401 }
402
StringFormatter(const QString & string)403 StringFormatter::StringFormatter(const QString &string)
404 : m_string(string)
405 {
406 qCDebug(KDEV_PYTHON_CODECOMPLETION) << "String being parsed: " << string;
407 QRegExp regex("\\{(\\w+)(?:!([rs]))?(?:\\:(.*))?\\}");
408 regex.setMinimal(true);
409 int pos = 0;
410 while ( (pos = regex.indexIn(string, pos)) != -1 ) {
411 QString identifier = regex.cap(1);
412 QString conversionStr = regex.cap(2);
413 QChar conversion = (conversionStr.isNull() || conversionStr.isEmpty()) ? QChar() : conversionStr.at(0);
414 QString formatSpec = regex.cap(3);
415
416 qCDebug(KDEV_PYTHON_CODECOMPLETION) << "variable: " << regex.cap(0);
417
418 // The regex guarantees that conversion is only a single character
419 ReplacementVariable variable(identifier, conversion, formatSpec);
420 m_replacementVariables.append(variable);
421
422 RangeInString variablePosition(pos, pos + regex.matchedLength());
423 m_variablePositions.append(variablePosition);
424
425 pos += regex.matchedLength();
426 }
427 }
428
isInsideReplacementVariable(int cursorPosition) const429 bool StringFormatter::isInsideReplacementVariable(int cursorPosition) const
430 {
431 return getReplacementVariable(cursorPosition) != nullptr;
432 }
433
getReplacementVariable(int cursorPosition) const434 const ReplacementVariable *StringFormatter::getReplacementVariable(int cursorPosition) const
435 {
436 int index = 0;
437 foreach ( const RangeInString &variablePosition, m_variablePositions ) {
438 if ( cursorPosition >= variablePosition.beginIndex && cursorPosition <= variablePosition.endIndex ) {
439 return &m_replacementVariables.at(index);
440 }
441 index++;
442 }
443
444 return nullptr;
445 }
446
getVariablePosition(int cursorPosition) const447 RangeInString StringFormatter::getVariablePosition(int cursorPosition) const
448 {
449 int index = 0;
450 foreach ( const RangeInString &variablePosition, m_variablePositions ) {
451 if ( cursorPosition >= variablePosition.beginIndex && cursorPosition <= variablePosition.endIndex ) {
452 return m_variablePositions.at(index);
453 }
454 index++;
455 }
456 return RangeInString();
457 }
458
nextIdentifierId() const459 int StringFormatter::nextIdentifierId() const
460 {
461 int highestIdFound = -1;
462 foreach ( const ReplacementVariable &variable, m_replacementVariables ) {
463 bool isNumeric;
464 int identifier = variable.identifier().toInt(&isNumeric);
465 if ( isNumeric && identifier > highestIdFound ) {
466 highestIdFound = identifier;
467 }
468 }
469 return highestIdFound + 1;
470 }
471
472 }
473