1 /*
2     SPDX-FileCopyrightText: 2011-2012 Sven Brauch <svenbrauch@googlemail.com>
3 
4     SPDX-License-Identifier: GPL-2.0-or-later
5 */
6 
7 // Note to confused people reading this code: This is not the parser.
8 // It's just a minimalist helper class for code completion. The parser is in the parser/ directory.
9 
10 #include "helpers.h"
11 
12 #include <language/duchain/abstractfunctiondeclaration.h>
13 #include <language/duchain/duchainutils.h>
14 #include <language/duchain/ducontext.h>
15 #include <language/duchain/declaration.h>
16 #include <language/duchain/types/functiontype.h>
17 #include <language/duchain/types/integraltype.h>
18 #include <language/codecompletion/normaldeclarationcompletionitem.h>
19 
20 #include <QStringList>
21 #include <QTextFormat>
22 
23 #include <QDebug>
24 #include "codecompletiondebug.h"
25 
26 #include "duchain/declarations/functiondeclaration.h"
27 #include "parser/codehelpers.h"
28 
29 using namespace KDevelop;
30 
31 namespace Python {
32 
camelCaseToUnderscore(const QString & camelCase)33 QString camelCaseToUnderscore(const QString& camelCase)
34 {
35     QString underscore;
36     for ( int i = 0; i < camelCase.size(); i++ ) {
37         const QChar& c = camelCase.at(i);
38         if ( c.isUpper() && i != 0 ) {
39             underscore.append('_');
40         }
41         underscore.append(c.toLower());
42     }
43     return underscore;
44 }
45 
identifierMatchQuality(const QString & identifier1_,const QString & identifier2_)46 int identifierMatchQuality(const QString& identifier1_, const QString& identifier2_)
47 {
48     QString identifier1 = camelCaseToUnderscore(identifier1_).toLower().replace('.', '_');
49     QString identifier2 = camelCaseToUnderscore(identifier2_).toLower().replace('.', '_');
50 
51     if ( identifier1 == identifier2 ) {
52         return 3;
53     }
54     if ( identifier1.contains(identifier2) || identifier2.contains(identifier1) ) {
55         return 2;
56     }
57     QStringList parts1 = identifier1.split('_');
58     QStringList parts2 = identifier2.split('_');
59     parts1.removeAll("");
60     parts2.removeAll("");
61     parts1.removeDuplicates();
62     parts2.removeDuplicates();
63     if ( parts1.length() > 5 || parts2.length() > 5 ) {
64         // don't waste time comparing huge identifiers,
65         // the matching is probably pointless anyways for people using
66         // more than 5 words for their variable names
67         return 0;
68     }
69     foreach ( const QString& part1, parts1 ) {
70         foreach ( const QString& part2, parts2 ) {
71             // Don't take very short name parts into account,
72             // those are not very descriptive eventually
73             if ( part1.size() < 3 || part2.size() < 3 ) {
74                 continue;
75             }
76             if ( part1 == part2 ) {
77                 // partial match
78                 return 1;
79             }
80         }
81     }
82     return 0;
83 }
84 
85 typedef QPair<QString, ExpressionParser::Status> keyword;
86 
87 static QList<keyword> supportedKeywords;
88 static QList<keyword> controlChars;
89 static QList<QString> miscKeywords;
90 static QList<QString> noCompletionKeywords;
91 static QMutex keywordPopulationLock;
92 
93 // Keywords known to me:
94 // and       del       for       is        raise
95 // assert    elif      from      lambda    return
96 // break     else      global    not       try
97 // class     except    if        or        while
98 // continue  exec      import    pass      yield
99 // def       finally   in        print     with
100 // async     await
101 
ExpressionParser(QString code)102 ExpressionParser::ExpressionParser(QString code)
103     : m_code(code)
104     , m_cursorPositionInString(m_code.length())
105 {
106     keywordPopulationLock.lock();
107     if ( supportedKeywords.isEmpty() ) {
108         noCompletionKeywords << "break" << "class" << "continue" << "pass" << "try"
109                              << "else" << "as" << "finally" << "global" << "lambda";
110         miscKeywords << "and" << "assert" << "del" << "elif" << "exec" << "if" << "is" << "not"
111                      << "or" << "print" << "return" << "while" << "yield" << "with" << "await";
112         supportedKeywords << keyword("import", ExpressionParser::ImportFound);
113         supportedKeywords << keyword("from", ExpressionParser::FromFound);
114         supportedKeywords << keyword("raise", ExpressionParser::RaiseFound);
115         supportedKeywords << keyword("in", ExpressionParser::InFound);
116         supportedKeywords << keyword("for", ExpressionParser::ForFound);
117         supportedKeywords << keyword("class", ExpressionParser::ClassFound);
118         supportedKeywords << keyword("def", ExpressionParser::DefFound);
119         supportedKeywords << keyword("except", ExpressionParser::ExceptFound);
120         controlChars << keyword(":", ExpressionParser::ColonFound);
121         controlChars << keyword(",", ExpressionParser::CommaFound);
122         controlChars << keyword("(", ExpressionParser::InitializerFound);
123         controlChars << keyword("{", ExpressionParser::InitializerFound);
124         controlChars << keyword("[", ExpressionParser::InitializerFound);
125         controlChars << keyword(".", ExpressionParser::MemberAccessFound);
126         controlChars << keyword("=", ExpressionParser::EqualsFound);
127     }
128     keywordPopulationLock.unlock();
129 }
130 
getRemainingCode()131 QString ExpressionParser::getRemainingCode()
132 {
133     return m_code.mid(0, m_cursorPositionInString);
134 }
135 
getScannedCode()136 QString ExpressionParser::getScannedCode()
137 {
138     return m_code.mid(m_cursorPositionInString, m_code.length() - m_cursorPositionInString);
139 }
140 
trailingWhitespace()141 int ExpressionParser::trailingWhitespace()
142 {
143     int ws = 0;
144     int index = m_cursorPositionInString - 1;
145     while ( index >= 0 ) {
146         if ( m_code.at(index).isSpace() ) {
147             ws++;
148             index --;
149         }
150         else {
151             break;
152         }
153     }
154     return ws;
155 }
156 
reset()157 void ExpressionParser::reset()
158 {
159     m_cursorPositionInString = m_code.length();
160 }
161 
skipUntilStatus(ExpressionParser::Status requestedStatus,bool * ok,int * expressionsSkipped)162 QString ExpressionParser::skipUntilStatus(ExpressionParser::Status requestedStatus, bool* ok, int* expressionsSkipped)
163 {
164     if ( expressionsSkipped ) {
165         *expressionsSkipped = 0;
166     }
167     QString lastExpression;
168     Status currentStatus = InvalidStatus;
169     while ( currentStatus != requestedStatus ) {
170         lastExpression = popExpression(&currentStatus);
171         qCDebug(KDEV_PYTHON_CODECOMPLETION) << lastExpression << currentStatus;
172         if ( currentStatus == NothingFound ) {
173             *ok = ( requestedStatus == NothingFound ); // ok exactly if the caller requested NothingFound as end status
174             return QString();
175         }
176         if ( expressionsSkipped && currentStatus == ExpressionFound ) {
177             *expressionsSkipped += 1;
178         }
179     }
180     *ok = true;
181     return lastExpression;
182 }
183 
popAll()184 TokenList ExpressionParser::popAll()
185 {
186     Status currentStatus = InvalidStatus;
187     TokenList items;
188     while ( currentStatus != NothingFound ) {
189         QString result = popExpression(&currentStatus);
190         items << TokenListEntry(currentStatus, result, m_cursorPositionInString);
191     }
192     std::reverse(items.begin(), items.end());
193     return items;
194 }
195 
endsWithSeperatedKeyword(const QString & str,const QString & shouldEndWith)196 bool endsWithSeperatedKeyword(const QString& str, const QString& shouldEndWith) {
197     bool endsWith = str.endsWith(shouldEndWith);
198     if ( ! endsWith ) {
199         return false;
200     }
201     int l = shouldEndWith.length();
202     if ( str.length() == l ) {
203         return true;
204     }
205     if ( str.right(l + 1).at(0).isSpace() ) {
206         return true;
207     }
208     return false;
209 }
210 
popExpression(ExpressionParser::Status * status)211 QString ExpressionParser::popExpression(ExpressionParser::Status* status)
212 {
213     const auto remaining = getRemainingCode();
214     auto trimmed = remaining.trimmed();
215     auto operatingOn = trimmed.replace('\t', ' ');
216     bool lineIsEmpty = false;
217     for ( auto it = remaining.constEnd()-1; it != remaining.constEnd(); it-- ) {
218         if ( ! it->isSpace() ) {
219             break;
220         }
221         if ( *it == '\n' ) {
222             lineIsEmpty = true;
223             break;
224         }
225     }
226     if ( operatingOn.isEmpty() || lineIsEmpty ) {
227         m_cursorPositionInString = 0;
228         *status = NothingFound;
229         return QString();
230     }
231     bool lastCharIsSpace = getRemainingCode().right(1).at(0).isSpace();
232     m_cursorPositionInString -= trailingWhitespace();
233     if ( operatingOn.endsWith('(') ) {
234         qCDebug(KDEV_PYTHON_CODECOMPLETION) << "eventual call found";
235         m_cursorPositionInString -= 1;
236         *status = EventualCallFound;
237         return QString();
238     }
239     foreach ( const keyword& kw, controlChars ) {
240         if ( operatingOn.endsWith(kw.first) ) {
241             m_cursorPositionInString -= kw.first.length();
242             *status = kw.second;
243             return QString();
244         }
245     }
246     if ( lastCharIsSpace ) {
247         foreach ( const keyword& kw, supportedKeywords ) {
248             if ( endsWithSeperatedKeyword(operatingOn, kw.first) ) {
249                 m_cursorPositionInString -= kw.first.length();
250                 *status = kw.second;
251                 return QString();
252             }
253         }
254         foreach ( const QString& kw, miscKeywords ) {
255             if ( endsWithSeperatedKeyword(operatingOn, kw) ) {
256                 m_cursorPositionInString -= kw.length();
257                 *status = MeaninglessKeywordFound;
258                 return QString();
259             }
260         }
261         foreach ( const QString& kw, noCompletionKeywords ) {
262             if ( endsWithSeperatedKeyword(operatingOn, kw) ) {
263                 m_cursorPositionInString -= kw.length();
264                 *status = NoCompletionKeywordFound;
265                 return QString();
266             }
267         }
268     }
269     // Otherwise, there's a real expression at the cursor, so scan it.
270     QStringList lines = operatingOn.split('\n');
271     Python::TrivialLazyLineFetcher f(lines);
272     int lastLine = lines.length()-1;
273     KTextEditor::Cursor startCursor;
274     QString expr = CodeHelpers::expressionUnderCursor(f, KTextEditor::Cursor(lastLine, f.fetchLine(lastLine).length() - 1),
275                                                       startCursor, true);
276     if ( expr.isEmpty() ) {
277         *status = NothingFound;
278     }
279     else {
280         *status = ExpressionFound;
281     }
282     m_cursorPositionInString -= expr.length();
283     return expr;
284 }
285 
286 
287 // This is stolen from PHP. For credits, see helpers.cpp in PHP.
createArgumentList(Declaration * dec_,QString & ret,QList<QVariant> * highlighting,int atArg,bool includeTypes)288 void createArgumentList(Declaration* dec_, QString& ret, QList< QVariant >* highlighting, int atArg, bool includeTypes)
289 {
290     auto dec = dynamic_cast<Python::FunctionDeclaration*>(dec_);
291     if ( ! dec ) {
292         return;
293     }
294     int textFormatStart = 0;
295     QTextFormat normalFormat(QTextFormat::CharFormat);
296     QTextFormat highlightFormat(QTextFormat::CharFormat);
297     highlightFormat.setBackground(QColor::fromRgb(142, 186, 255));
298     highlightFormat.setProperty(QTextFormat::FontWeight, 99);
299 
300     AbstractFunctionDeclaration* decl = dynamic_cast<AbstractFunctionDeclaration*>(dec);
301     FunctionType::Ptr functionType = dec->type<FunctionType>();
302 
303     if (functionType && decl) {
304 
305         QVector<Declaration*> parameters;
306         if (DUChainUtils::argumentContext(dec))
307             parameters = DUChainUtils::argumentContext(dec)->localDeclarations();
308 
309         ret = '(';
310         bool first = true;
311         int num = 0;
312 
313         bool skipFirst = false;
314         if ( dec->context() && dec->context()->type() == DUContext::Class && ! dec->isStatic() ) {
315             // the function is a class method, and its first argument is "self". Don't display that.
316             skipFirst = true;
317         }
318 
319         uint defaultParamNum = 0;
320         int firstDefaultParam = parameters.count() - decl->defaultParametersSize() - skipFirst;
321 
322         // disable highlighting when in default arguments, it doesn't make much sense then
323         bool disableHighlighting = false;
324 
325         foreach(Declaration* dec, parameters) {
326             if ( skipFirst ) {
327                 skipFirst = false;
328                 continue;
329             }
330             // that has nothing to do with the skip, it's just for the comma
331             if (first)
332                 first = false;
333             else
334                 ret += ", ";
335 
336             bool doHighlight = false;
337             QTextFormat doFormat;
338 
339             if ( num == atArg - 1 )
340                 doFormat = highlightFormat;
341             else
342                 doFormat = normalFormat;
343 
344             if ( num == firstDefaultParam ) {
345                 ret += "[";
346                 ++defaultParamNum;
347                 disableHighlighting = true;
348             }
349 
350             if ( ! disableHighlighting ) {
351                 doHighlight = true;
352             }
353 
354             if ( includeTypes ) {
355                 if (num < functionType->arguments().count()) {
356                     if (AbstractType::Ptr type = functionType->arguments().at(num)) {
357                         if ( type->toString() != "<unknown>" ) {
358                             ret += type->toString() + ' ';
359                         }
360                     }
361                 }
362 
363                 if (doHighlight) {
364                     if (highlighting && ret.length() != textFormatStart) {
365                         //Add a default-highlighting for the passed text
366                         *highlighting << QVariant(textFormatStart);
367                         *highlighting << QVariant(ret.length() - textFormatStart);
368                         *highlighting << QVariant(normalFormat);
369                         textFormatStart = ret.length();
370                     }
371                 }
372             }
373 
374 
375             ret += dec->identifier().toString();
376 
377             if (doHighlight) {
378                 if (highlighting && ret.length() != textFormatStart) {
379                     *highlighting << QVariant(textFormatStart + 1);
380                     *highlighting << QVariant(ret.length() - textFormatStart - 1);
381                     *highlighting << doFormat;
382                     textFormatStart = ret.length();
383                 }
384             }
385 
386             ++num;
387         }
388         if ( defaultParamNum != 0 ) {
389             ret += "]";
390         }
391         ret += ')';
392 
393         if (highlighting && ret.length() != textFormatStart) {
394             *highlighting <<  QVariant(textFormatStart);
395             *highlighting << QVariant(ret.length());
396             *highlighting << normalFormat;
397             textFormatStart = ret.length();
398         }
399         return;
400     }
401 }
402 
StringFormatter(const QString & string)403 StringFormatter::StringFormatter(const QString &string)
404     : m_string(string)
405 {
406     qCDebug(KDEV_PYTHON_CODECOMPLETION) << "String being parsed: " << string;
407     QRegExp regex("\\{(\\w+)(?:!([rs]))?(?:\\:(.*))?\\}");
408     regex.setMinimal(true);
409     int pos = 0;
410     while ( (pos = regex.indexIn(string, pos)) != -1 ) {
411         QString identifier = regex.cap(1);
412         QString conversionStr = regex.cap(2);
413         QChar conversion = (conversionStr.isNull() || conversionStr.isEmpty()) ? QChar() : conversionStr.at(0);
414         QString formatSpec = regex.cap(3);
415 
416         qCDebug(KDEV_PYTHON_CODECOMPLETION) << "variable: " << regex.cap(0);
417 
418         // The regex guarantees that conversion is only a single character
419         ReplacementVariable variable(identifier, conversion, formatSpec);
420         m_replacementVariables.append(variable);
421 
422         RangeInString variablePosition(pos, pos + regex.matchedLength());
423         m_variablePositions.append(variablePosition);
424 
425         pos += regex.matchedLength();
426     }
427 }
428 
isInsideReplacementVariable(int cursorPosition) const429 bool StringFormatter::isInsideReplacementVariable(int cursorPosition) const
430 {
431     return getReplacementVariable(cursorPosition) != nullptr;
432 }
433 
getReplacementVariable(int cursorPosition) const434 const ReplacementVariable *StringFormatter::getReplacementVariable(int cursorPosition) const
435 {
436     int index = 0;
437     foreach ( const RangeInString &variablePosition, m_variablePositions ) {
438         if ( cursorPosition >= variablePosition.beginIndex && cursorPosition <= variablePosition.endIndex ) {
439             return &m_replacementVariables.at(index);
440         }
441         index++;
442     }
443 
444     return nullptr;
445 }
446 
getVariablePosition(int cursorPosition) const447 RangeInString StringFormatter::getVariablePosition(int cursorPosition) const
448 {
449     int index = 0;
450     foreach ( const RangeInString &variablePosition, m_variablePositions ) {
451         if ( cursorPosition >= variablePosition.beginIndex && cursorPosition <= variablePosition.endIndex ) {
452             return m_variablePositions.at(index);
453         }
454         index++;
455     }
456     return RangeInString();
457 }
458 
nextIdentifierId() const459 int StringFormatter::nextIdentifierId() const
460 {
461     int highestIdFound = -1;
462     foreach ( const ReplacementVariable &variable, m_replacementVariables ) {
463         bool isNumeric;
464         int identifier = variable.identifier().toInt(&isNumeric);
465         if ( isNumeric && identifier > highestIdFound ) {
466             highestIdFound = identifier;
467         }
468     }
469     return highestIdFound + 1;
470 }
471 
472 }
473