1
2 /*
3 * Copyright (C) Igor Sysoev
4 * Copyright (C) NGINX, Inc.
5 */
6
7 #ifndef _NJS_LEXER_H_INCLUDED_
8 #define _NJS_LEXER_H_INCLUDED_
9
10
11 typedef enum {
12 NJS_TOKEN_ERROR = -1,
13 NJS_TOKEN_ILLEGAL = 0,
14
15 NJS_TOKEN_END,
16 NJS_TOKEN_SPACE,
17 NJS_TOKEN_LINE_END,
18
19 NJS_TOKEN_DOUBLE_QUOTE,
20 NJS_TOKEN_SINGLE_QUOTE,
21
22 NJS_TOKEN_OPEN_PARENTHESIS,
23 NJS_TOKEN_CLOSE_PARENTHESIS,
24 NJS_TOKEN_OPEN_BRACKET,
25 NJS_TOKEN_CLOSE_BRACKET,
26 NJS_TOKEN_OPEN_BRACE,
27 NJS_TOKEN_CLOSE_BRACE,
28
29 NJS_TOKEN_COMMA,
30 NJS_TOKEN_DOT,
31 NJS_TOKEN_ELLIPSIS,
32 NJS_TOKEN_SEMICOLON,
33
34 NJS_TOKEN_COLON,
35 NJS_TOKEN_CONDITIONAL,
36
37 NJS_TOKEN_COMMENT,
38
39 NJS_TOKEN_ASSIGNMENT,
40 NJS_TOKEN_ARROW,
41 NJS_TOKEN_ADDITION_ASSIGNMENT,
42 NJS_TOKEN_SUBSTRACTION_ASSIGNMENT,
43 NJS_TOKEN_MULTIPLICATION_ASSIGNMENT,
44 NJS_TOKEN_EXPONENTIATION_ASSIGNMENT,
45 NJS_TOKEN_DIVISION_ASSIGNMENT,
46 NJS_TOKEN_REMAINDER_ASSIGNMENT,
47 NJS_TOKEN_LEFT_SHIFT_ASSIGNMENT,
48 NJS_TOKEN_RIGHT_SHIFT_ASSIGNMENT,
49 NJS_TOKEN_UNSIGNED_RIGHT_SHIFT_ASSIGNMENT,
50 NJS_TOKEN_BITWISE_OR_ASSIGNMENT,
51 NJS_TOKEN_BITWISE_XOR_ASSIGNMENT,
52 NJS_TOKEN_BITWISE_AND_ASSIGNMENT,
53
54 NJS_TOKEN_INCREMENT,
55 NJS_TOKEN_DECREMENT,
56 NJS_TOKEN_POST_INCREMENT,
57 NJS_TOKEN_POST_DECREMENT,
58
59 #define NJS_TOKEN_LAST_ASSIGNMENT NJS_TOKEN_POST_DECREMENT
60
61 NJS_TOKEN_EQUAL,
62 NJS_TOKEN_STRICT_EQUAL,
63 NJS_TOKEN_NOT_EQUAL,
64 NJS_TOKEN_STRICT_NOT_EQUAL,
65
66 NJS_TOKEN_ADDITION,
67 NJS_TOKEN_UNARY_PLUS,
68
69 NJS_TOKEN_SUBSTRACTION,
70 NJS_TOKEN_UNARY_NEGATION,
71
72 NJS_TOKEN_MULTIPLICATION,
73
74 NJS_TOKEN_EXPONENTIATION,
75
76 NJS_TOKEN_DIVISION,
77
78 NJS_TOKEN_REMAINDER,
79
80 NJS_TOKEN_LESS,
81 NJS_TOKEN_LESS_OR_EQUAL,
82 NJS_TOKEN_LEFT_SHIFT,
83
84 NJS_TOKEN_GREATER,
85 NJS_TOKEN_GREATER_OR_EQUAL,
86 NJS_TOKEN_RIGHT_SHIFT,
87 NJS_TOKEN_UNSIGNED_RIGHT_SHIFT,
88
89 NJS_TOKEN_BITWISE_OR,
90 NJS_TOKEN_LOGICAL_OR,
91
92 NJS_TOKEN_BITWISE_XOR,
93
94 NJS_TOKEN_BITWISE_AND,
95 NJS_TOKEN_LOGICAL_AND,
96
97 NJS_TOKEN_BITWISE_NOT,
98 NJS_TOKEN_LOGICAL_NOT,
99
100 NJS_TOKEN_COALESCE,
101
102 NJS_TOKEN_IN,
103 NJS_TOKEN_OF,
104 NJS_TOKEN_INSTANCEOF,
105 NJS_TOKEN_TYPEOF,
106 NJS_TOKEN_VOID,
107 NJS_TOKEN_NEW,
108 NJS_TOKEN_DELETE,
109 NJS_TOKEN_YIELD,
110
111 NJS_TOKEN_DIGIT,
112 NJS_TOKEN_LETTER,
113
114 #define NJS_TOKEN_FIRST_CONST NJS_TOKEN_NULL
115
116 NJS_TOKEN_NULL,
117 NJS_TOKEN_NUMBER,
118 NJS_TOKEN_TRUE,
119 NJS_TOKEN_UNDEFINED,
120 NJS_TOKEN_FALSE,
121 NJS_TOKEN_STRING,
122
123 #define NJS_TOKEN_LAST_CONST NJS_TOKEN_STRING
124
125 NJS_TOKEN_ESCAPE_STRING,
126 NJS_TOKEN_UNTERMINATED_STRING,
127 NJS_TOKEN_NAME,
128
129 NJS_TOKEN_OBJECT,
130 NJS_TOKEN_OBJECT_VALUE,
131 NJS_TOKEN_PROPERTY,
132 NJS_TOKEN_PROPERTY_INIT,
133 NJS_TOKEN_PROPERTY_DELETE,
134 NJS_TOKEN_PROPERTY_GETTER,
135 NJS_TOKEN_PROPERTY_SETTER,
136 NJS_TOKEN_PROTO_INIT,
137
138 NJS_TOKEN_ARRAY,
139
140 NJS_TOKEN_GRAVE,
141 NJS_TOKEN_TEMPLATE_LITERAL,
142
143 NJS_TOKEN_FUNCTION,
144 NJS_TOKEN_FUNCTION_DECLARATION,
145 NJS_TOKEN_FUNCTION_EXPRESSION,
146 NJS_TOKEN_FUNCTION_CALL,
147 NJS_TOKEN_METHOD_CALL,
148 NJS_TOKEN_ARGUMENT,
149 NJS_TOKEN_RETURN,
150
151 NJS_TOKEN_ASYNC_FUNCTION,
152 NJS_TOKEN_ASYNC_FUNCTION_DECLARATION,
153 NJS_TOKEN_ASYNC_FUNCTION_EXPRESSION,
154
155 NJS_TOKEN_REGEXP,
156
157 NJS_TOKEN_EXTERNAL,
158
159 NJS_TOKEN_STATEMENT,
160 NJS_TOKEN_BLOCK,
161 NJS_TOKEN_VAR,
162 NJS_TOKEN_IF,
163 NJS_TOKEN_ELSE,
164 NJS_TOKEN_BRANCHING,
165 NJS_TOKEN_WHILE,
166 NJS_TOKEN_DO,
167 NJS_TOKEN_FOR,
168 NJS_TOKEN_FOR_IN,
169 NJS_TOKEN_BREAK,
170 NJS_TOKEN_CONTINUE,
171 NJS_TOKEN_SWITCH,
172 NJS_TOKEN_CASE,
173 NJS_TOKEN_DEFAULT,
174 NJS_TOKEN_WITH,
175 NJS_TOKEN_TRY,
176 NJS_TOKEN_CATCH,
177 NJS_TOKEN_FINALLY,
178 NJS_TOKEN_THROW,
179
180 NJS_TOKEN_THIS,
181 NJS_TOKEN_ARGUMENTS,
182 NJS_TOKEN_EVAL,
183
184 NJS_TOKEN_IMPORT,
185 NJS_TOKEN_EXPORT,
186
187 NJS_TOKEN_TARGET,
188
189 NJS_TOKEN_FROM,
190
191 NJS_TOKEN_META,
192
193 NJS_TOKEN_AWAIT,
194 NJS_TOKEN_ASYNC,
195 NJS_TOKEN_CLASS,
196 NJS_TOKEN_CONST,
197 NJS_TOKEN_DEBUGGER,
198 NJS_TOKEN_ENUM,
199 NJS_TOKEN_EXTENDS,
200 NJS_TOKEN_IMPLEMENTS,
201 NJS_TOKEN_INTERFACE,
202 NJS_TOKEN_LET,
203 NJS_TOKEN_PACKAGE,
204 NJS_TOKEN_PRIVATE,
205 NJS_TOKEN_PROTECTED,
206 NJS_TOKEN_PUBLIC,
207 NJS_TOKEN_STATIC,
208 NJS_TOKEN_SUPER,
209
210 NJS_TOKEN_RESERVED,
211 } njs_token_type_t;
212
213
214 typedef enum {
215 NJS_KEYWORD_TYPE_UNDEF = 0,
216 NJS_KEYWORD_TYPE_RESERVED = 1,
217 NJS_KEYWORD_TYPE_KEYWORD = 2
218 } njs_keyword_type_t;
219
220
221 typedef struct {
222 njs_str_t name;
223 } njs_lexer_entry_t;
224
225
226 typedef struct {
227 njs_lexer_entry_t entry;
228 njs_token_type_t type;
229 njs_bool_t reserved;
230 } njs_keyword_t;
231
232
233 typedef struct {
234 const char *key;
235 const njs_keyword_t *value;
236
237 size_t length;
238 size_t next;
239 } njs_lexer_keyword_entry_t;
240
241
242 typedef struct {
243 njs_token_type_t type:16;
244 njs_keyword_type_t keyword_type;
245 uint32_t line;
246 uintptr_t unique_id;
247 njs_str_t text;
248 double number;
249 njs_queue_link_t link;
250 } njs_lexer_token_t;
251
252
253 typedef struct {
254 njs_lexer_token_t *token;
255 njs_queue_t preread; /* of njs_lexer_token_t */
256
257 u_char *prev_start;
258 njs_token_type_t prev_type:16;
259 njs_token_type_t last_type:16;
260
261 uint32_t line;
262 njs_str_t file;
263
264 njs_lvlhsh_t *keywords_hash;
265
266 njs_mp_t *mem_pool;
267
268 u_char *start;
269 u_char *end;
270 } njs_lexer_t;
271
272
273 njs_int_t njs_lexer_init(njs_vm_t *vm, njs_lexer_t *lexer, njs_str_t *file,
274 u_char *start, u_char *end, njs_uint_t runtime);
275
276 njs_lexer_token_t *njs_lexer_token(njs_lexer_t *lexer,
277 njs_bool_t with_end_line);
278 njs_lexer_token_t *njs_lexer_peek_token(njs_lexer_t *lexer,
279 njs_lexer_token_t *current, njs_bool_t with_end_line);
280 void njs_lexer_consume_token(njs_lexer_t *lexer, unsigned length);
281 njs_int_t njs_lexer_make_token(njs_lexer_t *lexer, njs_lexer_token_t *token);
282
283 const njs_lexer_keyword_entry_t *njs_lexer_keyword(const u_char *key,
284 size_t length);
285 njs_int_t njs_lexer_keywords(njs_arr_t *array);
286
287
288 njs_inline const njs_lexer_entry_t *
njs_lexer_entry(uintptr_t unique_id)289 njs_lexer_entry(uintptr_t unique_id)
290 {
291 return (const njs_lexer_entry_t *) unique_id;
292 }
293
294
295 njs_inline njs_bool_t
njs_lexer_token_is_keyword(njs_lexer_token_t * token)296 njs_lexer_token_is_keyword(njs_lexer_token_t *token)
297 {
298 return token->keyword_type & NJS_KEYWORD_TYPE_KEYWORD;
299 }
300
301
302 njs_inline njs_bool_t
njs_lexer_token_is_reserved(njs_lexer_token_t * token)303 njs_lexer_token_is_reserved(njs_lexer_token_t *token)
304 {
305 return token->keyword_type & NJS_KEYWORD_TYPE_RESERVED;
306 }
307
308
309 njs_inline njs_bool_t
njs_lexer_token_is_name(njs_lexer_token_t * token)310 njs_lexer_token_is_name(njs_lexer_token_t *token)
311 {
312 return token->type == NJS_TOKEN_NAME
313 || (!njs_lexer_token_is_reserved(token)
314 && njs_lexer_token_is_keyword(token));
315 }
316
317
318 njs_inline njs_bool_t
njs_lexer_token_is_identifier_name(njs_lexer_token_t * token)319 njs_lexer_token_is_identifier_name(njs_lexer_token_t *token)
320 {
321 return token->type == NJS_TOKEN_NAME || njs_lexer_token_is_keyword(token);
322 }
323
324
325 njs_inline njs_bool_t
njs_lexer_token_is_binding_identifier(njs_lexer_token_t * token)326 njs_lexer_token_is_binding_identifier(njs_lexer_token_t *token)
327 {
328 switch (token->type) {
329 case NJS_TOKEN_NAME:
330 case NJS_TOKEN_YIELD:
331 case NJS_TOKEN_AWAIT:
332 return 1;
333
334 default:
335 return (!njs_lexer_token_is_reserved(token)
336 && njs_lexer_token_is_keyword(token));
337 };
338 }
339
340
341 njs_inline njs_bool_t
njs_lexer_token_is_label_identifier(njs_lexer_token_t * token)342 njs_lexer_token_is_label_identifier(njs_lexer_token_t *token)
343 {
344 return njs_lexer_token_is_binding_identifier(token);
345 }
346
347
348 njs_inline njs_bool_t
njs_lexer_token_is_identifier_reference(njs_lexer_token_t * token)349 njs_lexer_token_is_identifier_reference(njs_lexer_token_t *token)
350 {
351 return njs_lexer_token_is_binding_identifier(token);
352 }
353
354
355 extern const njs_lvlhsh_proto_t njs_lexer_hash_proto;
356
357
358 #endif /* _NJS_LEXER_H_INCLUDED_ */
359