1 /****************************************************************************
2 **
3 ** Copyright (C) 2016 The Qt Company Ltd.
4 ** Contact: https://www.qt.io/licensing/
5 **
6 ** This file is part of Qt Creator.
7 **
8 ** Commercial License Usage
9 ** Licensees holding valid commercial Qt licenses may use this file in
10 ** accordance with the commercial license agreement provided with the
11 ** Software or, alternatively, in accordance with the terms contained in
12 ** a written agreement between you and The Qt Company. For licensing terms
13 ** and conditions see https://www.qt.io/terms-conditions. For further
14 ** information use the contact form at https://www.qt.io/contact-us.
15 **
16 ** GNU General Public License Usage
17 ** Alternatively, this file may be used under the terms of the GNU
18 ** General Public License version 3 as published by the Free Software
19 ** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
20 ** included in the packaging of this file. Please review the following
21 ** information to ensure the GNU General Public License requirements will
22 ** be met: https://www.gnu.org/licenses/gpl-3.0.html.
23 **
24 ****************************************************************************/
25 
26 #pragma once
27 
28 #include <qmljs/qmljs_global.h>
29 
30 #include <QStringList>
31 
32 namespace QmlJS {
33 
34 class QMLJS_EXPORT Token
35 {
36 public:
37     enum Kind {
38         EndOfFile,
39         Keyword,
40         Identifier,
41         String,
42         Comment,
43         Number,
44         LeftParenthesis,
45         RightParenthesis,
46         LeftBrace,
47         RightBrace,
48         LeftBracket,
49         RightBracket,
50         Semicolon,
51         Colon,
52         Comma,
53         Dot,
54         Delimiter,
55         RegExp
56     };
57 
Token()58     inline Token(): offset(0), length(0), kind(EndOfFile) {}
Token(int o,int l,Kind k)59     inline Token(int o, int l, Kind k): offset(o), length(l), kind(k) {}
begin()60     inline int begin() const { return offset; }
end()61     inline int end() const { return offset + length; }
is(int k)62     inline bool is(int k) const { return k == kind; }
isNot(int k)63     inline bool isNot(int k) const { return k != kind; }
64 
compare(const Token & t1,const Token & t2)65     static int compare(const Token &t1, const Token &t2) {
66         if (int c = t1.offset - t2.offset)
67             return c;
68         if (int c = t1.length - t2.length)
69             return c;
70         return int(t1.kind) - int(t2.kind);
71     }
72 
73 public:
74     int offset = 0;
75     int length = 0;
76     Kind kind = EndOfFile;
77 };
78 
79 inline int operator == (const Token &t1, const Token &t2) {
80     return Token::compare(t1, t2) == 0;
81 }
82 inline int operator != (const Token &t1, const Token &t2) {
83     return Token::compare(t1, t2) != 0;
84 }
85 
86 class QMLJS_EXPORT Scanner
87 {
88 public:
89     enum {
90         FlagsBits = 4,
91         BraceCounterBits = 7
92     };
93     enum {
94         Normal = 0,
95         MultiLineComment = 1,
96         MultiLineStringDQuote = 2,
97         MultiLineStringSQuote = 3,
98         MultiLineStringBQuote = 4,
99         MultiLineMask = 7,
100 
101         RegexpMayFollow = 8, // flag that may be combined with the above
102 
103         // templates can be nested, which means that the scanner/lexer cannot
104         // be a simple state machine anymore, but should have a stack to store
105         // the state (the number of open braces in the current template
106         // string).
107         // The lexer stare is currently stored in an int, so we abuse that and
108         // store a the number of open braces (maximum 0x7f = 127) for at most 5
109         // nested templates in the int after the flags for the multiline
110         // comments and strings.
111         TemplateExpression = 0x1 << 4,
112         TemplateExpressionOpenBracesMask0 = 0x7F,
113         TemplateExpressionOpenBracesMask1 = 0x7F << 4,
114         TemplateExpressionOpenBracesMask2 = 0x7F << 11,
115         TemplateExpressionOpenBracesMask3 = 0x7F << 18,
116         TemplateExpressionOpenBracesMask4 = 0x7F << 25,
117         TemplateExpressionOpenBracesMask = TemplateExpressionOpenBracesMask1 | TemplateExpressionOpenBracesMask2
118            | TemplateExpressionOpenBracesMask3 | TemplateExpressionOpenBracesMask4
119     };
120 
121     Scanner();
122     virtual ~Scanner();
123 
124     bool scanComments() const;
125     void setScanComments(bool scanComments);
126 
127     QList<Token> operator()(const QString &text, int startState = Normal);
128     int state() const;
129 
130     bool isKeyword(const QString &text) const;
131     static QStringList keywords();
132 
133 private:
134     int _state;
135     bool _scanComments: 1;
136 };
137 
138 } // namespace QmlJS
139