1 /* This file is part of KDevelop
2 SPDX-FileCopyrightText: 2006 Hamish Rodda <rodda@kde.org>
3 SPDX-FileCopyrightText: 2008 Niko Sams <niko.sams@gmail.com>
4
5 SPDX-License-Identifier: LGPL-2.0-only
6 */
7
8 #include "lexertest.h"
9
10 #include <QtTest/QtTest>
11
12 #include "parsesession.h"
13 #include "phplexer.h"
14 #include "phptokentext.h"
15
16 QTEST_MAIN(Php::LexerTest)
17 namespace Php
18 {
19
20 #define COMPARE_TOKEN(tokenStream, index, tokenKind, startLine, startColumn, endLine, endColumn) \
21 { \
22 QVERIFY(tokenStream->at(index).kind == tokenKind); \
23 qint64 line; qint64 column; \
24 tokenStream->startPosition(index, &line, &column); \
25 QCOMPARE(line, (qint64) startLine); \
26 QCOMPARE(column, (qint64) startColumn); \
27 tokenStream->endPosition(index, &line, &column); \
28 QCOMPARE(line, (qint64) endLine); \
29 QCOMPARE(column, (qint64) endColumn); \
30 }
31
LexerTest()32 LexerTest::LexerTest()
33 {
34 }
35
testOpenTagWithNewline()36 void LexerTest::testOpenTagWithNewline()
37 {
38 TokenStream* ts = tokenize(QStringLiteral("<?php\nfoo;"));
39 QVERIFY(ts->size() == 3);
40
41 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
42 COMPARE_TOKEN(ts, 1, Parser::Token_STRING, 1, 0, 1, 2);
43 COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 3, 1, 3);
44
45 delete ts;
46 }
47
testOpenTagWithSpace()48 void LexerTest::testOpenTagWithSpace()
49 {
50 TokenStream* ts = tokenize(QStringLiteral("<?php foo;"));
51 QVERIFY(ts->size() == 3);
52
53 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
54 COMPARE_TOKEN(ts, 1, Parser::Token_STRING, 0, 6, 0, 8);
55 COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 0, 9, 0, 9);
56 delete ts;
57 }
58
testCommentOneLine()59 void LexerTest::testCommentOneLine()
60 {
61 TokenStream* ts = tokenize(QStringLiteral("<?php\n//comment\nfoo;"));
62 QVERIFY(ts->size() == 4);
63
64 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
65 COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 1, 9);
66 COMPARE_TOKEN(ts, 2, Parser::Token_STRING, 2, 0, 2, 2);
67 COMPARE_TOKEN(ts, 3, Parser::Token_SEMICOLON, 2, 3, 2, 3);
68 delete ts;
69 }
70
testCommentOneLine2()71 void LexerTest::testCommentOneLine2()
72 {
73 TokenStream* ts = tokenize(QStringLiteral("<?php\n#comment\nfoo;"));
74 QVERIFY(ts->size() == 4);
75
76 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
77 COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 1, 8);
78 COMPARE_TOKEN(ts, 2, Parser::Token_STRING, 2, 0, 2, 2);
79 COMPARE_TOKEN(ts, 3, Parser::Token_SEMICOLON, 2, 3, 2, 3);
80 delete ts;
81 }
82
testCommentMultiLine()83 void LexerTest::testCommentMultiLine()
84 {
85 TokenStream* ts = tokenize(QStringLiteral("<?php\n/*com\nment*/\nfoo;"), true);
86 QVERIFY(ts->size() == 5);
87
88 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
89 COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 2, 5);
90 COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 2, 6, 2, 6);
91 COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 3, 0, 3, 2);
92 COMPARE_TOKEN(ts, 4, Parser::Token_SEMICOLON, 3, 3, 3, 3);
93 delete ts;
94 }
95
testCommentMultiLine2()96 void LexerTest::testCommentMultiLine2()
97 {
98 TokenStream* ts = tokenize(QStringLiteral("<?php\n/*\nment*/\nfoo;"), true);
99 QVERIFY(ts->size() == 5);
100
101 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
102 COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 2, 5);
103 COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 2, 6, 2, 6);
104 COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 3, 0, 3, 2);
105 COMPARE_TOKEN(ts, 4, Parser::Token_SEMICOLON, 3, 3, 3, 3);
106 delete ts;
107 }
108
testEndTag()109 void LexerTest::testEndTag()
110 {
111 TokenStream* ts = tokenize(QStringLiteral("<?\n':\n'?>\n>"), true, Lexer::DefaultState);
112 //don't crash and we are fine
113 delete ts;
114 }
115
testNewlineInString()116 void LexerTest::testNewlineInString()
117 {
118 //0 1
119 //012345 6 7 890123456789
120 TokenStream* ts = tokenize(QStringLiteral("<?php \"\n\";"), true);
121 QVERIFY(ts->size() == 3);
122
123 COMPARE_TOKEN(ts, 1, Parser::Token_CONSTANT_ENCAPSED_STRING, 0, 6, 1, 0);
124 COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 1, 1, 1);
125 delete ts;
126 }
127
testNewlineInString2()128 void LexerTest::testNewlineInString2()
129 {
130 //0
131 //0123 4567
132 TokenStream* ts = tokenize(QStringLiteral("<?php '\n';"), true);
133 QCOMPARE((int)ts->size(), 3);
134
135 COMPARE_TOKEN(ts, 1, Parser::Token_CONSTANT_ENCAPSED_STRING, 0, 6, 1, 0);
136 COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 1, 1, 1);
137 delete ts;
138 }
139
testNewlineInStringWithVar()140 void LexerTest::testNewlineInStringWithVar()
141 {
142 TokenStream* ts = tokenize(QStringLiteral("<?php \"$a\n\";"), true);
143 QCOMPARE((int)ts->size(), 6);
144
145 COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6);
146 COMPARE_TOKEN(ts, 2, Parser::Token_VARIABLE, 0, 7, 0, 8);
147 COMPARE_TOKEN(ts, 3, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 9, 0, 9);
148 COMPARE_TOKEN(ts, 4, Parser::Token_DOUBLE_QUOTE, 1, 0, 1, 0);
149 COMPARE_TOKEN(ts, 5, Parser::Token_SEMICOLON, 1, 1, 1, 1);
150 delete ts;
151 }
152
testNewlineInStringWithVar2()153 void LexerTest::testNewlineInStringWithVar2()
154 {
155 //0 1
156 //012345 6 789 0123456789
157 TokenStream* ts = tokenize(QStringLiteral("<?php \"\n$a\n\";"), true);
158 QCOMPARE((int)ts->size(), 7);
159
160 COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6);
161 COMPARE_TOKEN(ts, 2, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 7, 0, 7);
162 COMPARE_TOKEN(ts, 3, Parser::Token_VARIABLE, 1, 0, 1, 1);
163 COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 1, 2, 1, 2);
164 COMPARE_TOKEN(ts, 5, Parser::Token_DOUBLE_QUOTE, 2, 0, 2, 0);
165 COMPARE_TOKEN(ts, 6, Parser::Token_SEMICOLON, 2, 1, 2, 1);
166 delete ts;
167 }
168
testNewlineInStringWithVar3()169 void LexerTest::testNewlineInStringWithVar3()
170 {
171 //0 1
172 //012345 6 789 0123456789
173 TokenStream* ts = tokenize(QStringLiteral("<?php \"{$$a}\";"), true);
174 QCOMPARE((int)ts->size(), 7);
175
176 COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6);
177 COMPARE_TOKEN(ts, 2, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 7, 0, 8);
178 COMPARE_TOKEN(ts, 3, Parser::Token_VARIABLE, 0, 9, 0, 10);
179 COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 11, 0, 11);
180 COMPARE_TOKEN(ts, 5, Parser::Token_DOUBLE_QUOTE, 0, 12, 0, 12);
181 COMPARE_TOKEN(ts, 6, Parser::Token_SEMICOLON, 0, 13, 0, 13);
182 delete ts;
183 }
184
testMultiplePhpSections()185 void LexerTest::testMultiplePhpSections()
186 {
187
188 //0 1
189 //012345 6 789 0123456789
190 TokenStream* ts = tokenize(QStringLiteral("<?php $a;?>\n<html>\n<?php $a;?>"), true);
191 QCOMPARE((int)ts->size(), 9);
192
193 qint64 index = 0;
194 for (qint64 line = 0; line <= 2; ++line) {
195 if (line == 1) {
196 // the html stuff in the middle
197 COMPARE_TOKEN(ts, index, Parser::Token_INLINE_HTML, 0, 11, 1, 6);
198 ++index;
199 } else {
200 // the php stuff (symmetric) at the start and end
201 COMPARE_TOKEN(ts, index, Parser::Token_OPEN_TAG, line, 0, line, 5);
202 ++index;
203
204 COMPARE_TOKEN(ts, index, Parser::Token_VARIABLE, line, 6, line, 7);
205 ++index;
206
207 COMPARE_TOKEN(ts, index, Parser::Token_SEMICOLON, line, 8, line, 8);
208 ++index;
209
210 COMPARE_TOKEN(ts, index, Parser::Token_CLOSE_TAG, line, 9, line, 10);
211 ++index;
212 }
213 }
214 delete ts;
215 }
216
testHereDoc()217 void LexerTest::testHereDoc()
218 {
219 TokenStream* ts = tokenize(QStringLiteral("<?php\necho <<<EOD1\nstart $text\nend\nEOD1;\n$extern;"), true);
220 QCOMPARE((int)ts->size(), 12);
221
222 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
223 COMPARE_TOKEN(ts, 1, Parser::Token_ECHO, 1, 0, 1, 3);
224 COMPARE_TOKEN(ts, 3, Parser::Token_START_HEREDOC, 1, 5, 1, 12);
225 COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 0, 2, 5);
226 COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 2, 6, 2, 10);
227 COMPARE_TOKEN(ts, 6, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 11, 3, 3);
228 COMPARE_TOKEN(ts, 7, Parser::Token_END_HEREDOC, 4, 0, 4, 3);
229 COMPARE_TOKEN(ts, 8, Parser::Token_SEMICOLON, 4, 4, 4, 4);
230 COMPARE_TOKEN(ts, 10, Parser::Token_VARIABLE, 5, 0, 5, 6);
231 COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 5, 7, 5, 7);
232 delete ts;
233 }
234
testHereDocQuoted()235 void LexerTest::testHereDocQuoted()
236 {
237 TokenStream* ts = tokenize(QStringLiteral("<?php\necho <<<\"EOD1\"\nstart $text\nend\nEOD1;\n$extern;"), true);
238 QCOMPARE((int)ts->size(), 12);
239
240 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
241 COMPARE_TOKEN(ts, 1, Parser::Token_ECHO, 1, 0, 1, 3);
242 COMPARE_TOKEN(ts, 3, Parser::Token_START_HEREDOC, 1, 5, 1, 14);
243 COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 0, 2, 5);
244 COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 2, 6, 2, 10);
245 COMPARE_TOKEN(ts, 6, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 11, 3, 3);
246 COMPARE_TOKEN(ts, 7, Parser::Token_END_HEREDOC, 4, 0, 4, 3);
247 COMPARE_TOKEN(ts, 8, Parser::Token_SEMICOLON, 4, 4, 4, 4);
248 COMPARE_TOKEN(ts, 10, Parser::Token_VARIABLE, 5, 0, 5, 6);
249 COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 5, 7, 5, 7);
250 delete ts;
251 }
252
testNowdoc()253 void LexerTest::testNowdoc()
254 {
255 TokenStream* ts = tokenize(QStringLiteral("<?php\necho <<<'EOD1'\nstart $text\nend\nEOD1;\n$extern;"), true);
256 QCOMPARE((int)ts->size(), 10);
257
258 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
259 COMPARE_TOKEN(ts, 1, Parser::Token_ECHO, 1, 0, 1, 3);
260 COMPARE_TOKEN(ts, 3, Parser::Token_START_NOWDOC, 1, 5, 1, 14);
261 COMPARE_TOKEN(ts, 4, Parser::Token_STRING, 2, 0, 3, 3);
262 COMPARE_TOKEN(ts, 5, Parser::Token_END_NOWDOC, 4, 0, 4, 3);
263 COMPARE_TOKEN(ts, 6, Parser::Token_SEMICOLON, 4, 4, 4, 4);
264 COMPARE_TOKEN(ts, 8, Parser::Token_VARIABLE, 5, 0, 5, 6);
265 COMPARE_TOKEN(ts, 9, Parser::Token_SEMICOLON, 5, 7, 5, 7);
266 delete ts;
267 }
268
testCommonStringTokens()269 void LexerTest::testCommonStringTokens()
270 {
271 // all these should have open_tag followed by constant encapsed string
272 foreach ( const QString& code, QStringList() << "<?php ''" << "<?php \"\"" << "<?php '" << "<?php \"" ) {
273 qDebug() << code;
274 TokenStream* ts = tokenize(code, true);
275
276 QCOMPARE((int)ts->size(), 2);
277
278 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
279 COMPARE_TOKEN(ts, 1, Parser::Token_CONSTANT_ENCAPSED_STRING, 0, 6, 0, code.size() - 1);
280
281 delete ts;
282 }
283 }
284
testNonTerminatedStringWithVar()285 void LexerTest::testNonTerminatedStringWithVar()
286 {
287 TokenStream* ts = tokenize(QStringLiteral("<?php \"$a"), true);
288
289 QCOMPARE((int)ts->size(), 3);
290
291 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
292 COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6);
293 COMPARE_TOKEN(ts, 2, Parser::Token_VARIABLE, 0, 7, 0, 8);
294 delete ts;
295 }
296
testPhpBlockWithComment()297 void LexerTest::testPhpBlockWithComment()
298 {
299 TokenStream* ts = tokenize(
300 QStringLiteral("<?php\n"
301 "//asdf\n"
302 "?>\n"
303 "<?php\n")
304 , true);
305
306 QCOMPARE((int)ts->size(), 5);
307
308 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
309 COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 1, 6);
310 COMPARE_TOKEN(ts, 2, Parser::Token_CLOSE_TAG, 2, 0, 2, 1);
311 COMPARE_TOKEN(ts, 3, Parser::Token_INLINE_HTML, 2, 2, 2, 2);
312 COMPARE_TOKEN(ts, 4, Parser::Token_OPEN_TAG, 3, 0, 3, 5);
313 delete ts;
314 }
315
testNamespaces()316 void LexerTest::testNamespaces()
317 {
318 TokenStream* ts = tokenize(
319 QStringLiteral("<?php\n"
320 "namespace Foo;\n"
321 "namespace Foo\\Bar;\n"
322 "namespace Foo\\Bar\\Asd {\n"
323 "}\n")
324 , true);
325 QCOMPARE((int)ts->size(), 25);
326
327 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
328
329 COMPARE_TOKEN(ts, 1, Parser::Token_NAMESPACE, 1, 0, 1, 8);
330 COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 9, 1, 9);
331 COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 1, 10, 1, 12);
332 COMPARE_TOKEN(ts, 4, Parser::Token_SEMICOLON, 1, 13, 1, 13);
333
334 COMPARE_TOKEN(ts, 6, Parser::Token_NAMESPACE, 2, 0, 2, 8);
335 COMPARE_TOKEN(ts, 7, Parser::Token_WHITESPACE, 2, 9, 2, 9);
336 COMPARE_TOKEN(ts, 8, Parser::Token_STRING, 2, 10, 2, 12);
337 COMPARE_TOKEN(ts, 9, Parser::Token_BACKSLASH, 2, 13, 2, 13);
338 COMPARE_TOKEN(ts, 10, Parser::Token_STRING, 2, 14, 2, 16);
339 COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 2, 17, 2, 17);
340
341 COMPARE_TOKEN(ts, 13, Parser::Token_NAMESPACE, 3, 0, 3, 8);
342 COMPARE_TOKEN(ts, 14, Parser::Token_WHITESPACE, 3, 9, 3, 9);
343 COMPARE_TOKEN(ts, 15, Parser::Token_STRING, 3, 10, 3, 12);
344 COMPARE_TOKEN(ts, 16, Parser::Token_BACKSLASH, 3, 13, 3, 13);
345 COMPARE_TOKEN(ts, 17, Parser::Token_STRING, 3, 14, 3, 16);
346 COMPARE_TOKEN(ts, 18, Parser::Token_BACKSLASH, 3, 17, 3, 17);
347 COMPARE_TOKEN(ts, 19, Parser::Token_STRING, 3, 18, 3, 20);
348 COMPARE_TOKEN(ts, 20, Parser::Token_WHITESPACE, 3, 21, 3, 21);
349 COMPARE_TOKEN(ts, 21, Parser::Token_LBRACE, 3, 22, 3, 22);
350 COMPARE_TOKEN(ts, 23, Parser::Token_RBRACE, 4, 0, 4, 0);
351
352 delete ts;
353 }
354
testCloseTagInComment()355 void LexerTest::testCloseTagInComment()
356 {
357 {
358 TokenStream* ts = tokenize(
359 QStringLiteral("<?php // asdf ?>")
360 , true);
361 QCOMPARE((int)ts->size(), 3);
362
363 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
364 COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 0, 6, 0, 13);
365 COMPARE_TOKEN(ts, 2, Parser::Token_CLOSE_TAG, 0, 14, 0, 15);
366
367 delete ts;
368 }
369 {
370 TokenStream* ts = tokenize(
371 QStringLiteral("<?php # asdf ?>")
372 , true);
373 QCOMPARE((int)ts->size(), 3);
374
375 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
376 COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 0, 6, 0, 13);
377 COMPARE_TOKEN(ts, 2, Parser::Token_CLOSE_TAG, 0, 14, 0, 15);
378
379 delete ts;
380 }
381 }
382
testBinaryNumber()383 void LexerTest::testBinaryNumber()
384 {
385 TokenStream* ts = tokenize(QStringLiteral("<?php\n0b01;\n0B01;"), true);
386 QCOMPARE((int)ts->size(), 6);
387
388 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
389 COMPARE_TOKEN(ts, 1, Parser::Token_LNUMBER, 1, 0, 1, 3);
390 COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 4, 1, 4);
391 COMPARE_TOKEN(ts, 3, Parser::Token_WHITESPACE, 1, 5, 1, 5);
392 COMPARE_TOKEN(ts, 4, Parser::Token_LNUMBER, 2, 0, 2, 3);
393 COMPARE_TOKEN(ts, 5, Parser::Token_SEMICOLON, 2, 4, 2, 4);
394 delete ts;
395 }
396
testHexadecimalNumber()397 void LexerTest::testHexadecimalNumber()
398 {
399 TokenStream* ts = tokenize(QStringLiteral("<?php\n0x01;\n0X01;\n0xABC12;\n0Xab10A;"), true);
400 QCOMPARE((int)ts->size(), 12);
401
402 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
403 COMPARE_TOKEN(ts, 1, Parser::Token_LNUMBER, 1, 0, 1, 3);
404 COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 4, 1, 4);
405 COMPARE_TOKEN(ts, 3, Parser::Token_WHITESPACE, 1, 5, 1, 5);
406 COMPARE_TOKEN(ts, 4, Parser::Token_LNUMBER, 2, 0, 2, 3);
407 COMPARE_TOKEN(ts, 5, Parser::Token_SEMICOLON, 2, 4, 2, 4);
408 COMPARE_TOKEN(ts, 6, Parser::Token_WHITESPACE, 2, 5, 2, 5);
409 COMPARE_TOKEN(ts, 7, Parser::Token_LNUMBER, 3, 0, 3, 6);
410 COMPARE_TOKEN(ts, 8, Parser::Token_SEMICOLON, 3, 7, 3, 7);
411 COMPARE_TOKEN(ts, 9, Parser::Token_WHITESPACE, 3, 8, 3, 8);
412 COMPARE_TOKEN(ts, 10, Parser::Token_LNUMBER, 4, 0, 4, 6);
413 COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 4, 7, 4, 7);
414 delete ts;
415 }
416
testTypeHintsOnFunction()417 void LexerTest::testTypeHintsOnFunction()
418 {
419 QScopedPointer<TokenStream> ts(tokenize(QStringLiteral("<?php\nfunction a($a, array $b = [], callable $c) {}"), true));
420 QCOMPARE((int)ts->size(), 25);
421
422 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
423 COMPARE_TOKEN(ts, 1, Parser::Token_FUNCTION, 1, 0, 1, 7);
424 COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 8, 1, 8);
425 COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 1, 9, 1, 9);
426 COMPARE_TOKEN(ts, 4, Parser::Token_LPAREN, 1, 10, 1, 10);
427 COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 1, 11, 1, 12);
428 COMPARE_TOKEN(ts, 6, Parser::Token_COMMA, 1, 13, 1, 13);
429 COMPARE_TOKEN(ts, 7, Parser::Token_WHITESPACE, 1, 14, 1, 14);
430 COMPARE_TOKEN(ts, 8, Parser::Token_ARRAY, 1, 15, 1, 19);
431 COMPARE_TOKEN(ts, 9, Parser::Token_WHITESPACE, 1, 20, 1, 20);
432 COMPARE_TOKEN(ts, 10, Parser::Token_VARIABLE, 1, 21, 1, 22);
433 COMPARE_TOKEN(ts, 11, Parser::Token_WHITESPACE, 1, 23, 1, 23);
434 COMPARE_TOKEN(ts, 12, Parser::Token_ASSIGN, 1, 24, 1, 24);
435 COMPARE_TOKEN(ts, 13, Parser::Token_WHITESPACE, 1, 25, 1, 25);
436 COMPARE_TOKEN(ts, 14, Parser::Token_LBRACKET, 1, 26, 1, 26);
437 COMPARE_TOKEN(ts, 15, Parser::Token_RBRACKET, 1, 27, 1, 27);
438 COMPARE_TOKEN(ts, 16, Parser::Token_COMMA, 1, 28, 1, 28);
439 COMPARE_TOKEN(ts, 17, Parser::Token_WHITESPACE, 1, 29, 1, 29);
440 COMPARE_TOKEN(ts, 18, Parser::Token_CALLABLE, 1, 30, 1, 37);
441 COMPARE_TOKEN(ts, 19, Parser::Token_WHITESPACE, 1, 38, 1, 38);
442 COMPARE_TOKEN(ts, 20, Parser::Token_VARIABLE, 1, 39, 1, 40);
443 COMPARE_TOKEN(ts, 21, Parser::Token_RPAREN, 1, 41, 1, 41);
444 COMPARE_TOKEN(ts, 22, Parser::Token_WHITESPACE, 1, 42, 1, 42);
445 COMPARE_TOKEN(ts, 23, Parser::Token_LBRACE, 1, 43, 1, 43);
446 COMPARE_TOKEN(ts, 24, Parser::Token_RBRACE, 1, 44, 1, 44);
447 }
448
testExponentiation()449 void LexerTest::testExponentiation()
450 {
451 QScopedPointer<TokenStream> ts(tokenize(QStringLiteral("<?php\n$a = 2 ** 3; $a **= 2;"), true));
452 QCOMPARE((int)ts->size(), 18);
453
454 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
455 COMPARE_TOKEN(ts, 1, Parser::Token_VARIABLE, 1, 0, 1, 1);
456 COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 2, 1, 2);
457 COMPARE_TOKEN(ts, 3, Parser::Token_ASSIGN, 1, 3, 1, 3);
458 COMPARE_TOKEN(ts, 4, Parser::Token_WHITESPACE, 1, 4, 1, 4);
459 COMPARE_TOKEN(ts, 5, Parser::Token_LNUMBER, 1, 5, 1, 5);
460 COMPARE_TOKEN(ts, 6, Parser::Token_WHITESPACE, 1, 6, 1, 6);
461 COMPARE_TOKEN(ts, 7, Parser::Token_EXP, 1, 7, 1, 8);
462 COMPARE_TOKEN(ts, 8, Parser::Token_WHITESPACE, 1, 9, 1, 9);
463 COMPARE_TOKEN(ts, 9, Parser::Token_LNUMBER, 1, 10, 1, 10);
464 COMPARE_TOKEN(ts, 10, Parser::Token_SEMICOLON, 1, 11, 1, 11);
465 COMPARE_TOKEN(ts, 11, Parser::Token_WHITESPACE, 1, 12, 1, 12);
466 COMPARE_TOKEN(ts, 12, Parser::Token_VARIABLE, 1, 13, 1, 14);
467 COMPARE_TOKEN(ts, 13, Parser::Token_WHITESPACE, 1, 15, 1, 15);
468 COMPARE_TOKEN(ts, 14, Parser::Token_EXP_ASSIGN, 1, 16, 1, 18);
469 COMPARE_TOKEN(ts, 15, Parser::Token_WHITESPACE, 1, 19, 1, 19);
470 COMPARE_TOKEN(ts, 16, Parser::Token_LNUMBER, 1, 20, 1, 20);
471 COMPARE_TOKEN(ts, 17, Parser::Token_SEMICOLON, 1, 21, 1, 21);
472
473 }
474
testExceptionFinally()475 void LexerTest::testExceptionFinally()
476 {
477 QScopedPointer<TokenStream> ts(tokenize(QStringLiteral("<?php\ntry { $a = 1; } finally { }"), true));
478 QCOMPARE((int)ts->size(), 19);
479
480 COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
481 COMPARE_TOKEN(ts, 1, Parser::Token_TRY, 1, 0, 1, 2);
482 COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 3, 1, 3);
483 COMPARE_TOKEN(ts, 3, Parser::Token_LBRACE, 1, 4, 1, 4);
484 COMPARE_TOKEN(ts, 4, Parser::Token_WHITESPACE, 1, 5, 1, 5);
485 COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 1, 6, 1, 7);
486 COMPARE_TOKEN(ts, 6, Parser::Token_WHITESPACE, 1, 8, 1, 8);
487 COMPARE_TOKEN(ts, 7, Parser::Token_ASSIGN, 1, 9, 1, 9);
488 COMPARE_TOKEN(ts, 8, Parser::Token_WHITESPACE, 1, 10, 1, 10);
489 COMPARE_TOKEN(ts, 9, Parser::Token_LNUMBER, 1, 11, 1, 11);
490 COMPARE_TOKEN(ts, 10, Parser::Token_SEMICOLON, 1, 12, 1, 12);
491 COMPARE_TOKEN(ts, 11, Parser::Token_WHITESPACE, 1, 13, 1, 13);
492 COMPARE_TOKEN(ts, 12, Parser::Token_RBRACE, 1, 14, 1, 14);
493 COMPARE_TOKEN(ts, 13, Parser::Token_WHITESPACE, 1, 15, 1, 15);
494 COMPARE_TOKEN(ts, 14, Parser::Token_FINALLY, 1, 16, 1, 22);
495 COMPARE_TOKEN(ts, 15, Parser::Token_WHITESPACE, 1, 23, 1, 23);
496 COMPARE_TOKEN(ts, 16, Parser::Token_LBRACE, 1, 24, 1, 24);
497 COMPARE_TOKEN(ts, 17, Parser::Token_WHITESPACE, 1, 25, 1, 25);
498 COMPARE_TOKEN(ts, 18, Parser::Token_RBRACE, 1, 26, 1, 26);
499 }
500
tokenize(const QString & unit,bool debug,int initialState)501 TokenStream* LexerTest::tokenize(const QString& unit, bool debug, int initialState)
502 {
503 TokenStream* tokenStream = new TokenStream;
504 Lexer lexer(tokenStream, unit, initialState);
505 int token;
506 int i = 0;
507 QList<Parser::Token> tokens;
508 while ((token = lexer.nextTokenKind())) {
509 Parser::Token &t = tokenStream->push();
510 t.begin = lexer.tokenBegin();
511 t.end = lexer.tokenEnd();
512 t.kind = token;
513 tokens << t;
514 }
515 if (debug) {
516 foreach(const Parser::Token &t, tokens) {
517 qint64 beginLine;
518 qint64 beginColumn;
519 tokenStream->startPosition(i, &beginLine, &beginColumn);
520 qint64 endLine;
521 qint64 endColumn;
522 tokenStream->endPosition(i, &endLine, &endColumn);
523 qDebug() << tokenText(t.kind)
524 << unit.mid(t.begin, t.end - t.begin + 1).replace('\n', QLatin1String("\\n"))
525 << QStringLiteral("[%0-%1] - [%2-%3]").arg(beginLine).arg(beginColumn).arg(endLine).arg(endColumn);
526 ++i;
527 }
528 }
529 return tokenStream;
530 }
531 }
532
533