1 #include "token.h"
2 #include "lexer.h"
3 #include <stdio.h>
4 #include <QStringList>
5
Token()6 Token::Token()
7 : lemonType(0), type(INVALID), value(QString()), start(-1), end(-1)
8 {
9 }
10
Token(int lemonType,Type type,QString value,qint64 start,qint64 end)11 Token::Token(int lemonType, Type type, QString value, qint64 start, qint64 end)
12 : lemonType(lemonType), type(type), value(value), start(start), end(end)
13 {}
14
Token(int lemonType,Type type,QChar value,qint64 start,qint64 end)15 Token::Token(int lemonType, Type type, QChar value, qint64 start, qint64 end)
16 : lemonType(lemonType), type(type), value(value), start(start), end(end)
17 {}
18
Token(int lemonType,Token::Type type,QString value)19 Token::Token(int lemonType, Token::Type type, QString value)
20 : lemonType(lemonType), type(type), value(value), start(-1), end(-1)
21 {
22 }
23
Token(QString value)24 Token::Token(QString value)
25 : lemonType(0), type(INVALID), value(value), start(0), end(0)
26 {}
27
Token(Token::Type type,QString value)28 Token::Token(Token::Type type, QString value)
29 : lemonType(0), type(type), value(value), start(0), end(0)
30 {
31 }
32
Token(Token::Type type,QString value,qint64 start,qint64 end)33 Token::Token(Token::Type type, QString value, qint64 start, qint64 end)
34 : lemonType(0), type(type), value(value), start(start), end(end)
35 {
36 }
37
~Token()38 Token::~Token()
39 {
40 }
41
toString()42 QString Token::toString()
43 {
44 return "{" +
45 typeToString(type) +
46 " " +
47 value +
48 " " +
49 QString::number(start) +
50 " " +
51 QString::number(end) +
52 "}";
53 }
54
typeToString(Token::Type type)55 const QString Token::typeToString(Token::Type type)
56 {
57 switch (type)
58 {
59 case Token::CTX_ROWID_KW:
60 return "CTX_ROWID_KW";
61 case Token::CTX_NEW_KW:
62 return "CTX_NEW_KW";
63 case Token::CTX_OLD_KW:
64 return "CTX_OLD_KW";
65 case Token::CTX_TABLE_NEW:
66 return "CTX_TABLE_NEW";
67 case Token::CTX_INDEX_NEW:
68 return "CTX_INDEX_NEW";
69 case Token::CTX_VIEW_NEW:
70 return "CTX_VIEW_NEW";
71 case Token::CTX_TRIGGER_NEW:
72 return "CTX_TRIGGER_NEW";
73 case Token::CTX_ALIAS:
74 return "CTX_ALIAS";
75 case Token::CTX_TRANSACTION:
76 return "CTX_transaction";
77 case Token::CTX_COLUMN_NEW:
78 return "CTX_COLUMN_NEW";
79 case Token::CTX_COLUMN_TYPE:
80 return "CTX_COLUMN_TYPE";
81 case Token::CTX_CONSTRAINT:
82 return "CTX_CONSTRAINT";
83 case Token::CTX_FK_MATCH:
84 return "CTX_FK_MATCH";
85 case Token::CTX_PRAGMA:
86 return "CTX_PRAGMA";
87 case Token::CTX_ERROR_MESSAGE:
88 return "CTX_ERROR_MESSAGE";
89 case Token::CTX_COLUMN:
90 return "CTX_COLUMN";
91 case Token::CTX_TABLE:
92 return "CTX_TABLE";
93 case Token::CTX_DATABASE:
94 return "CTX_DATABASE";
95 case Token::CTX_FUNCTION:
96 return "CTX_FUNCTION";
97 case Token::CTX_COLLATION:
98 return "CTX_COLLATION";
99 case Token::CTX_INDEX:
100 return "CTX_INDEX";
101 case Token::CTX_TRIGGER:
102 return "CTX_TRIGGER";
103 case Token::CTX_VIEW:
104 return "CTX_VIEW";
105 case Token::CTX_JOIN_OPTS:
106 return "CTX_JOIN_OPTS";
107 case Token::INVALID:
108 return "INVALID";
109 case Token::OTHER:
110 return "OTHER";
111 case Token::STRING:
112 return "STRING";
113 case Token::COMMENT:
114 return "COMMENT";
115 case Token::FLOAT:
116 return "FLOAT";
117 case Token::INTEGER:
118 return "INTEGER";
119 case Token::BIND_PARAM:
120 return "BIND_PARAM";
121 case Token::OPERATOR:
122 return "OPERATOR";
123 case Token::PAR_LEFT:
124 return "PAR_LEFT";
125 case Token::PAR_RIGHT:
126 return "PAR_RIGHT";
127 case Token::SPACE:
128 return "SPACE";
129 case Token::BLOB:
130 return "BLOB";
131 case Token::KEYWORD:
132 return "KEYWORD";
133 }
134
135 return "";
136 }
137
getRange()138 Range Token::getRange()
139 {
140 return Range(start, end);
141 }
142
isWhitespace(bool includeComments) const143 bool Token::isWhitespace(bool includeComments) const
144 {
145 return (type == SPACE || (includeComments && type == COMMENT));
146 }
147
isSeparating() const148 bool Token::isSeparating() const
149 {
150 switch (type)
151 {
152 case Token::SPACE:
153 case Token::PAR_LEFT:
154 case Token::PAR_RIGHT:
155 case Token::OPERATOR:
156 return true;
157 default:
158 break;
159 }
160 return false;
161 }
162
isMeaningful() const163 bool Token::isMeaningful() const
164 {
165 switch (type)
166 {
167 case Token::BIND_PARAM:
168 case Token::PAR_LEFT:
169 case Token::PAR_RIGHT:
170 case Token::BLOB:
171 case Token::FLOAT:
172 case Token::INTEGER:
173 case Token::INVALID:
174 case Token::KEYWORD:
175 case Token::OTHER:
176 case Token::STRING:
177 case Token::OPERATOR:
178 return true;
179 default:
180 break;
181 }
182 return false;
183 }
184
isDbObjectType() const185 bool Token::isDbObjectType() const
186 {
187 return ((type & TOKEN_TYPE_MASK_DB_OBJECT) == TOKEN_TYPE_MASK_DB_OBJECT);
188 }
189
typeString() const190 QString Token::typeString() const
191 {
192 return typeToString(type);
193 }
194
operator ==(const Token & other)195 int Token::operator ==(const Token &other)
196 {
197 return type == other.type && value == other.value && start == other.start && end == other.end;
198 }
199
operator <(const Token & other) const200 bool Token::operator <(const Token &other) const
201 {
202 if (start == other.start)
203 return end < other.end;
204 else
205 return start < other.start;
206 }
207
qHash(const TokenPtr & token)208 uint qHash(const TokenPtr& token)
209 {
210 // This doesn't look nice, but it's good enough to satisfy a hash table.
211 // It's fast and quite distinguishable.
212 // It's rare to have two pointers with the same int type representation,
213 // and if that happens, there's always a comparision operator.
214 return (uint)reinterpret_cast<qint64>(token.data());
215 }
216
TokenList()217 TokenList::TokenList()
218 : QList<TokenPtr>()
219 {
220 }
221
TokenList(const QList<TokenPtr> & other)222 TokenList::TokenList(const QList<TokenPtr>& other)
223 : QList<TokenPtr>(other)
224 {
225 }
226
toString() const227 QString TokenList::toString() const
228 {
229 return toStringList().join(" ");
230 }
231
toStringList() const232 QStringList TokenList::toStringList() const
233 {
234 QStringList strList;
235 for (const TokenPtr& t : *this)
236 strList << t->toString();
237
238 return strList;
239 }
240
toValueList() const241 QStringList TokenList::toValueList() const
242 {
243 QStringList strList;
244 for (const TokenPtr& t : *this)
245 strList << t->value;
246
247 return strList;
248 }
249
indexOf(TokenPtr token) const250 int TokenList::indexOf(TokenPtr token) const
251 {
252 return QList<TokenPtr>::indexOf(token);
253 }
254
indexOf(Token::Type type) const255 int TokenList::indexOf(Token::Type type) const
256 {
257 int i;
258 findFirst(type, &i);
259 return i;
260 }
261
indexOf(Token::Type type,const QString & value,Qt::CaseSensitivity caseSensitivity) const262 int TokenList::indexOf(Token::Type type, const QString &value, Qt::CaseSensitivity caseSensitivity) const
263 {
264 int i;
265 findFirst(type, value, caseSensitivity, &i);
266 return i;
267 }
268
indexOf(const QString & value,Qt::CaseSensitivity caseSensitivity) const269 int TokenList::indexOf(const QString &value, Qt::CaseSensitivity caseSensitivity) const
270 {
271 int i;
272 findFirst(value, caseSensitivity, &i);
273 return i;
274 }
275
lastIndexOf(TokenPtr token) const276 int TokenList::lastIndexOf(TokenPtr token) const
277 {
278 return QList<TokenPtr>::lastIndexOf(token);
279 }
280
lastIndexOf(Token::Type type) const281 int TokenList::lastIndexOf(Token::Type type) const
282 {
283 int i;
284 findLast(type, &i);
285 return i;
286 }
287
lastIndexOf(Token::Type type,const QString & value,Qt::CaseSensitivity caseSensitivity) const288 int TokenList::lastIndexOf(Token::Type type, const QString& value, Qt::CaseSensitivity caseSensitivity) const
289 {
290 int i;
291 findLast(type, value, caseSensitivity, &i);
292 return i;
293 }
294
lastIndexOf(const QString & value,Qt::CaseSensitivity caseSensitivity) const295 int TokenList::lastIndexOf(const QString& value, Qt::CaseSensitivity caseSensitivity) const
296 {
297 int i;
298 findLast(value, caseSensitivity, &i);
299 return i;
300 }
301
find(Token::Type type) const302 TokenPtr TokenList::find(Token::Type type) const
303 {
304 return findFirst(type, nullptr);
305 }
306
find(Token::Type type,const QString & value,Qt::CaseSensitivity caseSensitivity) const307 TokenPtr TokenList::find(Token::Type type, const QString &value, Qt::CaseSensitivity caseSensitivity) const
308 {
309 return findFirst(type, value, caseSensitivity, nullptr);
310 }
311
find(const QString & value,Qt::CaseSensitivity caseSensitivity) const312 TokenPtr TokenList::find(const QString &value, Qt::CaseSensitivity caseSensitivity) const
313 {
314 return findFirst(value, caseSensitivity, nullptr);
315 }
316
findLast(Token::Type type) const317 TokenPtr TokenList::findLast(Token::Type type) const
318 {
319 return findLast(type, nullptr);
320 }
321
findLast(Token::Type type,const QString & value,Qt::CaseSensitivity caseSensitivity) const322 TokenPtr TokenList::findLast(Token::Type type, const QString& value, Qt::CaseSensitivity caseSensitivity) const
323 {
324 return findLast(type, value, caseSensitivity, nullptr);
325 }
326
findLast(const QString & value,Qt::CaseSensitivity caseSensitivity) const327 TokenPtr TokenList::findLast(const QString& value, Qt::CaseSensitivity caseSensitivity) const
328 {
329 return findLast(value, caseSensitivity, nullptr);
330 }
331
atCursorPosition(quint64 cursorPosition) const332 TokenPtr TokenList::atCursorPosition(quint64 cursorPosition) const
333 {
334 for (TokenPtr token : *this)
335 {
336 if (token->getRange().contains(cursorPosition))
337 return token;
338 }
339 return TokenPtr();
340 }
341
insert(int i,const TokenList & list)342 void TokenList::insert(int i, const TokenList &list)
343 {
344 for (TokenPtr token : list)
345 QList<TokenPtr>::insert(i++, token);
346 }
347
insert(int i,TokenPtr token)348 void TokenList::insert(int i, TokenPtr token)
349 {
350 QList<TokenPtr>::insert(i, token);
351 }
352
operator =(const QList<TokenPtr> & other)353 TokenList &TokenList::operator =(const QList<TokenPtr> &other)
354 {
355 QList<TokenPtr>::operator =(other);
356 return *this;
357 }
358
detokenize() const359 QString TokenList::detokenize() const
360 {
361 return Lexer::detokenize(*this);
362 }
363
replace(int startIdx,int length,const TokenList & newTokens)364 void TokenList::replace(int startIdx, int length, const TokenList& newTokens)
365 {
366 for (int i = 0; i < length; i++)
367 removeAt(startIdx);
368
369 insert(startIdx, newTokens);
370 }
371
replace(int startIdx,int length,TokenPtr newToken)372 void TokenList::replace(int startIdx, int length, TokenPtr newToken)
373 {
374 for (int i = 0; i < length; i++)
375 removeAt(startIdx);
376
377 insert(startIdx, newToken);
378 }
379
replace(int startIdx,TokenPtr newToken)380 void TokenList::replace(int startIdx, TokenPtr newToken)
381 {
382 QList<TokenPtr>::replace(startIdx, newToken);
383 }
384
replace(int startIdx,const TokenList & newTokens)385 void TokenList::replace(int startIdx, const TokenList& newTokens)
386 {
387 replace(startIdx, 1, newTokens);
388 }
389
replace(TokenPtr startToken,TokenPtr endToken,const TokenList & newTokens)390 int TokenList::replace(TokenPtr startToken, TokenPtr endToken, const TokenList& newTokens)
391 {
392 int startIdx = indexOf(startToken);
393 if (startIdx < 0)
394 return 0;
395
396 int endIdx = indexOf(endToken);
397 if (endIdx < 0)
398 return 0;
399
400 replace(startIdx, endIdx - startIdx, newTokens);
401 return endIdx - startIdx;
402 }
403
replace(TokenPtr startToken,TokenPtr endToken,TokenPtr newToken)404 int TokenList::replace(TokenPtr startToken, TokenPtr endToken, TokenPtr newToken)
405 {
406 int startIdx = indexOf(startToken);
407 if (startIdx < 0)
408 return 0;
409
410 int endIdx = indexOf(endToken);
411 if (endIdx < 0)
412 return 0;
413
414 replace(startIdx, endIdx - startIdx, newToken);
415 return endIdx - startIdx;
416 }
417
replace(TokenPtr oldToken,TokenPtr newToken)418 bool TokenList::replace(TokenPtr oldToken, TokenPtr newToken)
419 {
420 int idx = indexOf(oldToken);
421 if (idx < 0)
422 return false;
423
424 replace(idx, newToken);
425 return true;
426 }
427
replace(TokenPtr oldToken,const TokenList & newTokens)428 bool TokenList::replace(TokenPtr oldToken, const TokenList& newTokens)
429 {
430 int idx = indexOf(oldToken);
431 if (idx < 0)
432 return false;
433
434 replace(idx, newTokens);
435 return true;
436 }
437
remove(TokenPtr startToken,TokenPtr endToken)438 bool TokenList::remove(TokenPtr startToken, TokenPtr endToken)
439 {
440 int startIdx = indexOf(startToken);
441 if (startIdx < 0)
442 return false;
443
444 int endIdx = indexOf(endToken);
445 if (endIdx < 0)
446 return false;
447
448 if (endIdx < startIdx)
449 return false;
450
451 for (int i = startIdx; i < endIdx; i++)
452 removeAt(startIdx);
453
454 return true;
455 }
456
remove(Token::Type type)457 bool TokenList::remove(Token::Type type)
458 {
459 int idx = indexOf(type);
460 if (idx == -1)
461 return false;
462
463 removeAt(idx);
464 return true;
465 }
466
trimLeft()467 TokenList& TokenList::trimLeft()
468 {
469 while (size() > 0 && first()->isWhitespace())
470 removeFirst();
471
472 return *this;
473 }
474
trimRight()475 TokenList& TokenList::trimRight()
476 {
477 while (size() > 0 && last()->isWhitespace())
478 removeLast();
479
480 return *this;
481 }
482
trim()483 TokenList& TokenList::trim()
484 {
485 trimLeft();
486 trimRight();
487 return *this;
488 }
489
trimLeft(Token::Type type,const QString & alsoTrim)490 TokenList& TokenList::trimLeft(Token::Type type, const QString& alsoTrim)
491 {
492 while (size() > 0 && (first()->isWhitespace() || (first()->type == type && first()->value == alsoTrim)))
493 removeFirst();
494
495 return *this;
496 }
497
trimRight(Token::Type type,const QString & alsoTrim)498 TokenList& TokenList::trimRight(Token::Type type, const QString& alsoTrim)
499 {
500 while (size() > 0 && (last()->isWhitespace() || (last()->type == type && last()->value == alsoTrim)))
501 removeLast();
502
503 return *this;
504 }
505
trim(Token::Type type,const QString & alsoTrim)506 TokenList& TokenList::trim(Token::Type type, const QString& alsoTrim)
507 {
508 trimLeft(type, alsoTrim);
509 trimRight(type, alsoTrim);
510 return *this;
511 }
512
filter(Token::Type type) const513 TokenList TokenList::filter(Token::Type type) const
514 {
515 TokenList filtered;
516 for (TokenPtr token : *this)
517 if (token->type == type)
518 filtered << token;
519
520 return filtered;
521 }
522
filterOut(Token::Type type) const523 TokenList TokenList::filterOut(Token::Type type) const
524 {
525 TokenList filtered;
526 for (TokenPtr token : *this)
527 if (token->type != type)
528 filtered << token;
529
530 return filtered;
531 }
532
filterWhiteSpaces(bool includeComments) const533 TokenList TokenList::filterWhiteSpaces(bool includeComments) const
534 {
535 TokenList filtered;
536 for (TokenPtr token : *this)
537 if (!token->isWhitespace(includeComments))
538 filtered << token;
539
540 return filtered;
541 }
542
mid(int pos,int length) const543 TokenList TokenList::mid(int pos, int length) const
544 {
545 TokenList newList = QList<TokenPtr>::mid(pos, length);
546 return newList;
547 }
548
findFirst(Token::Type type,int * idx) const549 TokenPtr TokenList::findFirst(Token::Type type, int *idx) const
550 {
551 int i = -1;
552 TokenPtr token;
553 QListIterator<TokenPtr> it(*this);
554 while (it.hasNext())
555 {
556 token = it.next();
557 i++;
558 if (token->type == type)
559 {
560 if (idx) (*idx) = i;
561 return token;
562 }
563 }
564 if (idx) (*idx) = -1;
565 return TokenPtr();
566 }
567
findFirst(Token::Type type,const QString & value,Qt::CaseSensitivity caseSensitivity,int * idx) const568 TokenPtr TokenList::findFirst(Token::Type type, const QString &value, Qt::CaseSensitivity caseSensitivity, int *idx) const
569 {
570 int i = -1;
571 TokenPtr token;
572 QListIterator<TokenPtr> it(*this);
573 while (it.hasNext())
574 {
575 token = it.next();
576 i++;
577 if (token->type != type)
578 continue;
579
580 if (token->value.compare(value, caseSensitivity) == 0)
581 {
582 if (idx) (*idx) = i;
583 return token;
584 }
585 }
586 if (idx) (*idx) = -1;
587 return TokenPtr();
588 }
589
findFirst(const QString & value,Qt::CaseSensitivity caseSensitivity,int * idx) const590 TokenPtr TokenList::findFirst(const QString &value, Qt::CaseSensitivity caseSensitivity, int *idx) const
591 {
592 int i = -1;
593 TokenPtr token;
594 QListIterator<TokenPtr> it(*this);
595 while (it.hasNext())
596 {
597 token = it.next();
598 i++;
599 if (token->value.compare(value, caseSensitivity) == 0)
600 {
601 if (idx) (*idx) = i;
602 return token;
603 }
604 }
605 if (idx) (*idx) = -1;
606 return TokenPtr();
607 }
608
609
findLast(Token::Type type,int * idx) const610 TokenPtr TokenList::findLast(Token::Type type, int* idx) const
611 {
612 int i = size();
613 TokenPtr token;
614 QListIterator<TokenPtr> it(*this);
615 it.toBack();
616 while (it.hasPrevious())
617 {
618 token = it.previous();
619 i--;
620 if (token->type == type)
621 {
622 if (idx) (*idx) = i;
623 return token;
624 }
625 }
626 if (idx) (*idx) = -1;
627 return TokenPtr();
628 }
629
findLast(Token::Type type,const QString & value,Qt::CaseSensitivity caseSensitivity,int * idx) const630 TokenPtr TokenList::findLast(Token::Type type, const QString& value, Qt::CaseSensitivity caseSensitivity, int* idx) const
631 {
632 int i = size();
633 TokenPtr token;
634 QListIterator<TokenPtr> it(*this);
635 it.toBack();
636 while (it.hasPrevious())
637 {
638 token = it.previous();
639 i--;
640 if (token->type != type)
641 continue;
642
643 if (token->value.compare(value, caseSensitivity) == 0)
644 {
645 if (idx) (*idx) = i;
646 return token;
647 }
648 }
649 if (idx) (*idx) = -1;
650 return TokenPtr();
651 }
652
findLast(const QString & value,Qt::CaseSensitivity caseSensitivity,int * idx) const653 TokenPtr TokenList::findLast(const QString& value, Qt::CaseSensitivity caseSensitivity, int* idx) const
654 {
655 int i = size();
656 TokenPtr token;
657 QListIterator<TokenPtr> it(*this);
658 it.toBack();
659 while (it.hasPrevious())
660 {
661 token = it.previous();
662 i--;
663 if (token->value.compare(value, caseSensitivity) == 0)
664 {
665 if (idx) (*idx) = i;
666 return token;
667 }
668 }
669 if (idx) (*idx) = -1;
670 return TokenPtr();
671 }
672