1 /* This file is part of KDevelop
2     SPDX-FileCopyrightText: 2006 Hamish Rodda <rodda@kde.org>
3     SPDX-FileCopyrightText: 2008 Niko Sams <niko.sams@gmail.com>
4 
5     SPDX-License-Identifier: LGPL-2.0-only
6 */
7 
8 #include "lexertest.h"
9 
10 #include <QtTest/QtTest>
11 
12 #include "parsesession.h"
13 #include "phplexer.h"
14 #include "phptokentext.h"
15 
16 QTEST_MAIN(Php::LexerTest)
17 namespace Php
18 {
19 
20 #define COMPARE_TOKEN(tokenStream, index, tokenKind, startLine, startColumn, endLine, endColumn) \
21     { \
22         QVERIFY(tokenStream->at(index).kind == tokenKind); \
23         qint64 line; qint64 column; \
24         tokenStream->startPosition(index, &line, &column); \
25         QCOMPARE(line, (qint64) startLine); \
26         QCOMPARE(column, (qint64) startColumn); \
27         tokenStream->endPosition(index, &line, &column); \
28         QCOMPARE(line, (qint64) endLine); \
29         QCOMPARE(column, (qint64) endColumn); \
30     }
31 
LexerTest()32 LexerTest::LexerTest()
33 {
34 }
35 
testOpenTagWithNewline()36 void LexerTest::testOpenTagWithNewline()
37 {
38     TokenStream* ts = tokenize(QStringLiteral("<?php\nfoo;"));
39     QVERIFY(ts->size() == 3);
40 
41     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
42     COMPARE_TOKEN(ts, 1, Parser::Token_STRING, 1, 0, 1, 2);
43     COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 3, 1, 3);
44 
45     delete ts;
46 }
47 
testOpenTagWithSpace()48 void LexerTest::testOpenTagWithSpace()
49 {
50     TokenStream* ts = tokenize(QStringLiteral("<?php foo;"));
51     QVERIFY(ts->size() == 3);
52 
53     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
54     COMPARE_TOKEN(ts, 1, Parser::Token_STRING, 0, 6, 0, 8);
55     COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 0, 9, 0, 9);
56     delete ts;
57 }
58 
testCommentOneLine()59 void LexerTest::testCommentOneLine()
60 {
61     TokenStream* ts = tokenize(QStringLiteral("<?php\n//comment\nfoo;"));
62     QVERIFY(ts->size() == 4);
63 
64     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
65     COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 1, 9);
66     COMPARE_TOKEN(ts, 2, Parser::Token_STRING, 2, 0, 2, 2);
67     COMPARE_TOKEN(ts, 3, Parser::Token_SEMICOLON, 2, 3, 2, 3);
68     delete ts;
69 }
70 
testCommentOneLine2()71 void LexerTest::testCommentOneLine2()
72 {
73     TokenStream* ts = tokenize(QStringLiteral("<?php\n#comment\nfoo;"));
74     QVERIFY(ts->size() == 4);
75 
76     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
77     COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 1, 8);
78     COMPARE_TOKEN(ts, 2, Parser::Token_STRING, 2, 0, 2, 2);
79     COMPARE_TOKEN(ts, 3, Parser::Token_SEMICOLON, 2, 3, 2, 3);
80     delete ts;
81 }
82 
testCommentMultiLine()83 void LexerTest::testCommentMultiLine()
84 {
85     TokenStream* ts = tokenize(QStringLiteral("<?php\n/*com\nment*/\nfoo;"), true);
86     QVERIFY(ts->size() == 5);
87 
88     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
89     COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 2, 5);
90     COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 2, 6, 2, 6);
91     COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 3, 0, 3, 2);
92     COMPARE_TOKEN(ts, 4, Parser::Token_SEMICOLON, 3, 3, 3, 3);
93     delete ts;
94 }
95 
testCommentMultiLine2()96 void LexerTest::testCommentMultiLine2()
97 {
98     TokenStream* ts = tokenize(QStringLiteral("<?php\n/*\nment*/\nfoo;"), true);
99     QVERIFY(ts->size() == 5);
100 
101     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
102     COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 2, 5);
103     COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 2, 6, 2, 6);
104     COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 3, 0, 3, 2);
105     COMPARE_TOKEN(ts, 4, Parser::Token_SEMICOLON, 3, 3, 3, 3);
106     delete ts;
107 }
108 
testEndTag()109 void LexerTest::testEndTag()
110 {
111     TokenStream* ts = tokenize(QStringLiteral("<?\n':\n'?>\n>"), true, Lexer::DefaultState);
112     //don't crash and we are fine
113     delete ts;
114 }
115 
testNewlineInString()116 void LexerTest::testNewlineInString()
117 {
118     //0            1
119     //012345 6 7 890123456789
120     TokenStream* ts = tokenize(QStringLiteral("<?php \"\n\";"), true);
121     QVERIFY(ts->size() == 3);
122 
123     COMPARE_TOKEN(ts, 1, Parser::Token_CONSTANT_ENCAPSED_STRING, 0, 6, 1, 0);
124     COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 1, 1, 1);
125     delete ts;
126 }
127 
testNewlineInString2()128 void LexerTest::testNewlineInString2()
129 {
130     //0
131     //0123 4567
132     TokenStream* ts = tokenize(QStringLiteral("<?php '\n';"), true);
133     QCOMPARE((int)ts->size(), 3);
134 
135     COMPARE_TOKEN(ts, 1, Parser::Token_CONSTANT_ENCAPSED_STRING, 0, 6, 1, 0);
136     COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 1, 1, 1);
137     delete ts;
138 }
139 
testNewlineInStringWithVar()140 void LexerTest::testNewlineInStringWithVar()
141 {
142     TokenStream* ts = tokenize(QStringLiteral("<?php \"$a\n\";"), true);
143     QCOMPARE((int)ts->size(), 6);
144 
145     COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6);
146     COMPARE_TOKEN(ts, 2, Parser::Token_VARIABLE, 0, 7, 0, 8);
147     COMPARE_TOKEN(ts, 3, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 9, 0, 9);
148     COMPARE_TOKEN(ts, 4, Parser::Token_DOUBLE_QUOTE, 1, 0, 1, 0);
149     COMPARE_TOKEN(ts, 5, Parser::Token_SEMICOLON, 1, 1, 1, 1);
150     delete ts;
151 }
152 
testNewlineInStringWithVar2()153 void LexerTest::testNewlineInStringWithVar2()
154 {
155     //0            1
156     //012345 6 789 0123456789
157     TokenStream* ts = tokenize(QStringLiteral("<?php \"\n$a\n\";"), true);
158     QCOMPARE((int)ts->size(), 7);
159 
160     COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6);
161     COMPARE_TOKEN(ts, 2, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 7, 0, 7);
162     COMPARE_TOKEN(ts, 3, Parser::Token_VARIABLE, 1, 0, 1, 1);
163     COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 1, 2, 1, 2);
164     COMPARE_TOKEN(ts, 5, Parser::Token_DOUBLE_QUOTE, 2, 0, 2, 0);
165     COMPARE_TOKEN(ts, 6, Parser::Token_SEMICOLON, 2, 1, 2, 1);
166     delete ts;
167 }
168 
testNewlineInStringWithVar3()169 void LexerTest::testNewlineInStringWithVar3()
170 {
171     //0            1
172     //012345 6 789 0123456789
173     TokenStream* ts = tokenize(QStringLiteral("<?php \"{$$a}\";"), true);
174     QCOMPARE((int)ts->size(), 7);
175 
176     COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6);
177     COMPARE_TOKEN(ts, 2, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 7, 0, 8);
178     COMPARE_TOKEN(ts, 3, Parser::Token_VARIABLE, 0, 9, 0, 10);
179     COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 11, 0, 11);
180     COMPARE_TOKEN(ts, 5, Parser::Token_DOUBLE_QUOTE, 0, 12, 0, 12);
181     COMPARE_TOKEN(ts, 6, Parser::Token_SEMICOLON, 0, 13, 0, 13);
182     delete ts;
183 }
184 
testMultiplePhpSections()185 void LexerTest::testMultiplePhpSections()
186 {
187 
188     //0            1
189     //012345 6 789 0123456789
190     TokenStream* ts = tokenize(QStringLiteral("<?php $a;?>\n<html>\n<?php $a;?>"), true);
191     QCOMPARE((int)ts->size(), 9);
192 
193     qint64 index = 0;
194     for (qint64 line = 0; line <= 2; ++line) {
195         if (line == 1) {
196             // the html stuff in the middle
197             COMPARE_TOKEN(ts, index, Parser::Token_INLINE_HTML, 0, 11, 1, 6);
198             ++index;
199         } else {
200             // the php stuff (symmetric) at the start and end
201             COMPARE_TOKEN(ts, index, Parser::Token_OPEN_TAG, line, 0, line, 5);
202             ++index;
203 
204             COMPARE_TOKEN(ts, index, Parser::Token_VARIABLE, line, 6, line, 7);
205             ++index;
206 
207             COMPARE_TOKEN(ts, index, Parser::Token_SEMICOLON, line, 8, line, 8);
208             ++index;
209 
210             COMPARE_TOKEN(ts, index, Parser::Token_CLOSE_TAG, line, 9, line, 10);
211             ++index;
212         }
213     }
214     delete ts;
215 }
216 
testHereDoc()217 void LexerTest::testHereDoc()
218 {
219     TokenStream* ts = tokenize(QStringLiteral("<?php\necho <<<EOD1\nstart $text\nend\nEOD1;\n$extern;"), true);
220     QCOMPARE((int)ts->size(), 12);
221 
222     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
223     COMPARE_TOKEN(ts, 1, Parser::Token_ECHO, 1, 0, 1, 3);
224     COMPARE_TOKEN(ts, 3, Parser::Token_START_HEREDOC, 1, 5, 1, 12);
225     COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 0, 2, 5);
226     COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 2, 6, 2, 10);
227     COMPARE_TOKEN(ts, 6, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 11, 3, 3);
228     COMPARE_TOKEN(ts, 7, Parser::Token_END_HEREDOC, 4, 0, 4, 3);
229     COMPARE_TOKEN(ts, 8, Parser::Token_SEMICOLON, 4, 4, 4, 4);
230     COMPARE_TOKEN(ts, 10, Parser::Token_VARIABLE, 5, 0, 5, 6);
231     COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 5, 7, 5, 7);
232     delete ts;
233 }
234 
testHereDocQuoted()235 void LexerTest::testHereDocQuoted()
236 {
237     TokenStream* ts = tokenize(QStringLiteral("<?php\necho <<<\"EOD1\"\nstart $text\nend\nEOD1;\n$extern;"), true);
238     QCOMPARE((int)ts->size(), 12);
239 
240     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
241     COMPARE_TOKEN(ts, 1, Parser::Token_ECHO, 1, 0, 1, 3);
242     COMPARE_TOKEN(ts, 3, Parser::Token_START_HEREDOC, 1, 5, 1, 14);
243     COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 0, 2, 5);
244     COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 2, 6, 2, 10);
245     COMPARE_TOKEN(ts, 6, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 11, 3, 3);
246     COMPARE_TOKEN(ts, 7, Parser::Token_END_HEREDOC, 4, 0, 4, 3);
247     COMPARE_TOKEN(ts, 8, Parser::Token_SEMICOLON, 4, 4, 4, 4);
248     COMPARE_TOKEN(ts, 10, Parser::Token_VARIABLE, 5, 0, 5, 6);
249     COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 5, 7, 5, 7);
250     delete ts;
251 }
252 
testNowdoc()253 void LexerTest::testNowdoc()
254 {
255     TokenStream* ts = tokenize(QStringLiteral("<?php\necho <<<'EOD1'\nstart $text\nend\nEOD1;\n$extern;"), true);
256     QCOMPARE((int)ts->size(), 10);
257 
258     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
259     COMPARE_TOKEN(ts, 1, Parser::Token_ECHO, 1, 0, 1, 3);
260     COMPARE_TOKEN(ts, 3, Parser::Token_START_NOWDOC, 1, 5, 1, 14);
261     COMPARE_TOKEN(ts, 4, Parser::Token_STRING, 2, 0, 3, 3);
262     COMPARE_TOKEN(ts, 5, Parser::Token_END_NOWDOC, 4, 0, 4, 3);
263     COMPARE_TOKEN(ts, 6, Parser::Token_SEMICOLON, 4, 4, 4, 4);
264     COMPARE_TOKEN(ts, 8, Parser::Token_VARIABLE, 5, 0, 5, 6);
265     COMPARE_TOKEN(ts, 9, Parser::Token_SEMICOLON, 5, 7, 5, 7);
266     delete ts;
267 }
268 
testCommonStringTokens()269 void LexerTest::testCommonStringTokens()
270 {
271     // all these should have open_tag followed by constant encapsed string
272     foreach ( const QString& code, QStringList() << "<?php ''" << "<?php \"\"" << "<?php '" << "<?php \"" ) {
273         qDebug() << code;
274         TokenStream* ts = tokenize(code, true);
275 
276         QCOMPARE((int)ts->size(), 2);
277 
278         COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
279         COMPARE_TOKEN(ts, 1, Parser::Token_CONSTANT_ENCAPSED_STRING, 0, 6, 0, code.size() - 1);
280 
281         delete ts;
282     }
283 }
284 
testNonTerminatedStringWithVar()285 void LexerTest::testNonTerminatedStringWithVar()
286 {
287     TokenStream* ts = tokenize(QStringLiteral("<?php \"$a"), true);
288 
289     QCOMPARE((int)ts->size(), 3);
290 
291     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
292     COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6);
293     COMPARE_TOKEN(ts, 2, Parser::Token_VARIABLE, 0, 7, 0, 8);
294     delete ts;
295 }
296 
testPhpBlockWithComment()297 void LexerTest::testPhpBlockWithComment()
298 {
299     TokenStream* ts = tokenize(
300         QStringLiteral("<?php\n"
301         "//asdf\n"
302         "?>\n"
303         "<?php\n")
304     , true);
305 
306     QCOMPARE((int)ts->size(), 5);
307 
308     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
309     COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 1, 6);
310     COMPARE_TOKEN(ts, 2, Parser::Token_CLOSE_TAG, 2, 0, 2, 1);
311     COMPARE_TOKEN(ts, 3, Parser::Token_INLINE_HTML, 2, 2, 2, 2);
312     COMPARE_TOKEN(ts, 4, Parser::Token_OPEN_TAG, 3, 0, 3, 5);
313     delete ts;
314 }
315 
testNamespaces()316 void LexerTest::testNamespaces()
317 {
318     TokenStream* ts = tokenize(
319         QStringLiteral("<?php\n"
320         "namespace Foo;\n"
321         "namespace Foo\\Bar;\n"
322         "namespace Foo\\Bar\\Asd {\n"
323         "}\n")
324     , true);
325     QCOMPARE((int)ts->size(), 25);
326 
327     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
328 
329     COMPARE_TOKEN(ts, 1, Parser::Token_NAMESPACE, 1, 0, 1, 8);
330     COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 9, 1, 9);
331     COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 1, 10, 1, 12);
332     COMPARE_TOKEN(ts, 4, Parser::Token_SEMICOLON, 1, 13, 1, 13);
333 
334     COMPARE_TOKEN(ts, 6, Parser::Token_NAMESPACE, 2, 0, 2, 8);
335     COMPARE_TOKEN(ts, 7, Parser::Token_WHITESPACE, 2, 9, 2, 9);
336     COMPARE_TOKEN(ts, 8, Parser::Token_STRING, 2, 10, 2, 12);
337     COMPARE_TOKEN(ts, 9, Parser::Token_BACKSLASH, 2, 13, 2, 13);
338     COMPARE_TOKEN(ts, 10, Parser::Token_STRING, 2, 14, 2, 16);
339     COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 2, 17, 2, 17);
340 
341     COMPARE_TOKEN(ts, 13, Parser::Token_NAMESPACE, 3, 0, 3, 8);
342     COMPARE_TOKEN(ts, 14, Parser::Token_WHITESPACE, 3, 9, 3, 9);
343     COMPARE_TOKEN(ts, 15, Parser::Token_STRING, 3, 10, 3, 12);
344     COMPARE_TOKEN(ts, 16, Parser::Token_BACKSLASH, 3, 13, 3, 13);
345     COMPARE_TOKEN(ts, 17, Parser::Token_STRING, 3, 14, 3, 16);
346     COMPARE_TOKEN(ts, 18, Parser::Token_BACKSLASH, 3, 17, 3, 17);
347     COMPARE_TOKEN(ts, 19, Parser::Token_STRING, 3, 18, 3, 20);
348     COMPARE_TOKEN(ts, 20, Parser::Token_WHITESPACE, 3, 21, 3, 21);
349     COMPARE_TOKEN(ts, 21, Parser::Token_LBRACE, 3, 22, 3, 22);
350     COMPARE_TOKEN(ts, 23, Parser::Token_RBRACE, 4, 0, 4, 0);
351 
352     delete ts;
353 }
354 
testCloseTagInComment()355 void LexerTest::testCloseTagInComment()
356 {
357     {
358     TokenStream* ts = tokenize(
359         QStringLiteral("<?php // asdf ?>")
360     , true);
361     QCOMPARE((int)ts->size(), 3);
362 
363     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
364     COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 0, 6, 0, 13);
365     COMPARE_TOKEN(ts, 2, Parser::Token_CLOSE_TAG, 0, 14, 0, 15);
366 
367     delete ts;
368     }
369     {
370     TokenStream* ts = tokenize(
371         QStringLiteral("<?php #  asdf ?>")
372     , true);
373     QCOMPARE((int)ts->size(), 3);
374 
375     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
376     COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 0, 6, 0, 13);
377     COMPARE_TOKEN(ts, 2, Parser::Token_CLOSE_TAG, 0, 14, 0, 15);
378 
379     delete ts;
380     }
381 }
382 
testBinaryNumber()383 void LexerTest::testBinaryNumber()
384 {
385     TokenStream* ts = tokenize(QStringLiteral("<?php\n0b01;\n0B01;"), true);
386     QCOMPARE((int)ts->size(), 6);
387 
388     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
389     COMPARE_TOKEN(ts, 1, Parser::Token_LNUMBER, 1, 0, 1, 3);
390     COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 4, 1, 4);
391     COMPARE_TOKEN(ts, 3, Parser::Token_WHITESPACE, 1, 5, 1, 5);
392     COMPARE_TOKEN(ts, 4, Parser::Token_LNUMBER, 2, 0, 2, 3);
393     COMPARE_TOKEN(ts, 5, Parser::Token_SEMICOLON, 2, 4, 2, 4);
394     delete ts;
395 }
396 
testHexadecimalNumber()397 void LexerTest::testHexadecimalNumber()
398 {
399     TokenStream* ts = tokenize(QStringLiteral("<?php\n0x01;\n0X01;\n0xABC12;\n0Xab10A;"), true);
400     QCOMPARE((int)ts->size(), 12);
401 
402     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
403     COMPARE_TOKEN(ts, 1, Parser::Token_LNUMBER, 1, 0, 1, 3);
404     COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 4, 1, 4);
405     COMPARE_TOKEN(ts, 3, Parser::Token_WHITESPACE, 1, 5, 1, 5);
406     COMPARE_TOKEN(ts, 4, Parser::Token_LNUMBER, 2, 0, 2, 3);
407     COMPARE_TOKEN(ts, 5, Parser::Token_SEMICOLON, 2, 4, 2, 4);
408     COMPARE_TOKEN(ts, 6, Parser::Token_WHITESPACE, 2, 5, 2, 5);
409     COMPARE_TOKEN(ts, 7, Parser::Token_LNUMBER, 3, 0, 3, 6);
410     COMPARE_TOKEN(ts, 8, Parser::Token_SEMICOLON, 3, 7, 3, 7);
411     COMPARE_TOKEN(ts, 9, Parser::Token_WHITESPACE, 3, 8, 3, 8);
412     COMPARE_TOKEN(ts, 10, Parser::Token_LNUMBER, 4, 0, 4, 6);
413     COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 4, 7, 4, 7);
414     delete ts;
415 }
416 
testTypeHintsOnFunction()417 void LexerTest::testTypeHintsOnFunction()
418 {
419     QScopedPointer<TokenStream> ts(tokenize(QStringLiteral("<?php\nfunction a($a, array $b = [], callable $c) {}"), true));
420     QCOMPARE((int)ts->size(), 25);
421 
422     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
423     COMPARE_TOKEN(ts, 1, Parser::Token_FUNCTION, 1, 0, 1, 7);
424     COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 8, 1, 8);
425     COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 1, 9, 1, 9);
426     COMPARE_TOKEN(ts, 4, Parser::Token_LPAREN, 1, 10, 1, 10);
427     COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE,  1, 11, 1, 12);
428     COMPARE_TOKEN(ts, 6, Parser::Token_COMMA, 1, 13, 1, 13);
429     COMPARE_TOKEN(ts, 7, Parser::Token_WHITESPACE, 1, 14, 1, 14);
430     COMPARE_TOKEN(ts, 8, Parser::Token_ARRAY, 1, 15, 1, 19);
431     COMPARE_TOKEN(ts, 9, Parser::Token_WHITESPACE, 1, 20, 1, 20);
432     COMPARE_TOKEN(ts, 10, Parser::Token_VARIABLE,  1, 21, 1, 22);
433     COMPARE_TOKEN(ts, 11, Parser::Token_WHITESPACE, 1, 23, 1, 23);
434     COMPARE_TOKEN(ts, 12, Parser::Token_ASSIGN, 1, 24, 1, 24);
435     COMPARE_TOKEN(ts, 13, Parser::Token_WHITESPACE, 1, 25, 1, 25);
436     COMPARE_TOKEN(ts, 14, Parser::Token_LBRACKET, 1, 26, 1, 26);
437     COMPARE_TOKEN(ts, 15, Parser::Token_RBRACKET, 1, 27, 1, 27);
438     COMPARE_TOKEN(ts, 16, Parser::Token_COMMA, 1, 28, 1, 28);
439     COMPARE_TOKEN(ts, 17, Parser::Token_WHITESPACE, 1, 29, 1, 29);
440     COMPARE_TOKEN(ts, 18, Parser::Token_CALLABLE, 1, 30, 1, 37);
441     COMPARE_TOKEN(ts, 19, Parser::Token_WHITESPACE, 1, 38, 1, 38);
442     COMPARE_TOKEN(ts, 20, Parser::Token_VARIABLE,  1, 39, 1, 40);
443     COMPARE_TOKEN(ts, 21, Parser::Token_RPAREN, 1, 41, 1, 41);
444     COMPARE_TOKEN(ts, 22, Parser::Token_WHITESPACE, 1, 42, 1, 42);
445     COMPARE_TOKEN(ts, 23, Parser::Token_LBRACE, 1, 43, 1, 43);
446     COMPARE_TOKEN(ts, 24, Parser::Token_RBRACE, 1, 44, 1, 44);
447 }
448 
testExponentiation()449 void LexerTest::testExponentiation()
450 {
451     QScopedPointer<TokenStream> ts(tokenize(QStringLiteral("<?php\n$a = 2 ** 3; $a **= 2;"), true));
452     QCOMPARE((int)ts->size(), 18);
453 
454     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
455     COMPARE_TOKEN(ts, 1, Parser::Token_VARIABLE, 1, 0, 1, 1);
456     COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 2, 1, 2);
457     COMPARE_TOKEN(ts, 3, Parser::Token_ASSIGN, 1, 3, 1, 3);
458     COMPARE_TOKEN(ts, 4, Parser::Token_WHITESPACE, 1, 4, 1, 4);
459     COMPARE_TOKEN(ts, 5, Parser::Token_LNUMBER, 1, 5, 1, 5);
460     COMPARE_TOKEN(ts, 6, Parser::Token_WHITESPACE, 1, 6, 1, 6);
461     COMPARE_TOKEN(ts, 7, Parser::Token_EXP, 1, 7, 1, 8);
462     COMPARE_TOKEN(ts, 8, Parser::Token_WHITESPACE, 1, 9, 1, 9);
463     COMPARE_TOKEN(ts, 9, Parser::Token_LNUMBER, 1, 10, 1, 10);
464     COMPARE_TOKEN(ts, 10, Parser::Token_SEMICOLON, 1, 11, 1, 11);
465     COMPARE_TOKEN(ts, 11, Parser::Token_WHITESPACE, 1, 12, 1, 12);
466     COMPARE_TOKEN(ts, 12, Parser::Token_VARIABLE, 1, 13, 1, 14);
467     COMPARE_TOKEN(ts, 13, Parser::Token_WHITESPACE, 1, 15, 1, 15);
468     COMPARE_TOKEN(ts, 14, Parser::Token_EXP_ASSIGN, 1, 16, 1, 18);
469     COMPARE_TOKEN(ts, 15, Parser::Token_WHITESPACE, 1, 19, 1, 19);
470     COMPARE_TOKEN(ts, 16, Parser::Token_LNUMBER, 1, 20, 1, 20);
471     COMPARE_TOKEN(ts, 17, Parser::Token_SEMICOLON, 1, 21, 1, 21);
472 
473 }
474 
testExceptionFinally()475 void LexerTest::testExceptionFinally()
476 {
477     QScopedPointer<TokenStream> ts(tokenize(QStringLiteral("<?php\ntry { $a = 1; } finally { }"), true));
478     QCOMPARE((int)ts->size(), 19);
479 
480     COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5);
481     COMPARE_TOKEN(ts, 1, Parser::Token_TRY, 1, 0, 1, 2);
482     COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 3, 1, 3);
483     COMPARE_TOKEN(ts, 3, Parser::Token_LBRACE, 1, 4, 1, 4);
484     COMPARE_TOKEN(ts, 4, Parser::Token_WHITESPACE, 1, 5, 1, 5);
485     COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 1, 6, 1, 7);
486     COMPARE_TOKEN(ts, 6, Parser::Token_WHITESPACE, 1, 8, 1, 8);
487     COMPARE_TOKEN(ts, 7, Parser::Token_ASSIGN, 1, 9, 1, 9);
488     COMPARE_TOKEN(ts, 8, Parser::Token_WHITESPACE, 1, 10, 1, 10);
489     COMPARE_TOKEN(ts, 9, Parser::Token_LNUMBER, 1, 11, 1, 11);
490     COMPARE_TOKEN(ts, 10, Parser::Token_SEMICOLON, 1, 12, 1, 12);
491     COMPARE_TOKEN(ts, 11, Parser::Token_WHITESPACE, 1, 13, 1, 13);
492     COMPARE_TOKEN(ts, 12, Parser::Token_RBRACE, 1, 14, 1, 14);
493     COMPARE_TOKEN(ts, 13, Parser::Token_WHITESPACE, 1, 15, 1, 15);
494     COMPARE_TOKEN(ts, 14, Parser::Token_FINALLY, 1, 16, 1, 22);
495     COMPARE_TOKEN(ts, 15, Parser::Token_WHITESPACE, 1, 23, 1, 23);
496     COMPARE_TOKEN(ts, 16, Parser::Token_LBRACE, 1, 24, 1, 24);
497     COMPARE_TOKEN(ts, 17, Parser::Token_WHITESPACE, 1, 25, 1, 25);
498     COMPARE_TOKEN(ts, 18, Parser::Token_RBRACE, 1, 26, 1, 26);
499 }
500 
tokenize(const QString & unit,bool debug,int initialState)501 TokenStream* LexerTest::tokenize(const QString& unit, bool debug, int initialState)
502 {
503     TokenStream* tokenStream = new TokenStream;
504     Lexer lexer(tokenStream, unit, initialState);
505     int token;
506     int i = 0;
507     QList<Parser::Token> tokens;
508     while ((token = lexer.nextTokenKind())) {
509         Parser::Token &t = tokenStream->push();
510         t.begin = lexer.tokenBegin();
511         t.end = lexer.tokenEnd();
512         t.kind = token;
513         tokens << t;
514     }
515     if (debug) {
516         foreach(const Parser::Token &t, tokens) {
517             qint64 beginLine;
518             qint64 beginColumn;
519             tokenStream->startPosition(i, &beginLine, &beginColumn);
520             qint64 endLine;
521             qint64 endColumn;
522             tokenStream->endPosition(i, &endLine, &endColumn);
523             qDebug() << tokenText(t.kind)
524             << unit.mid(t.begin, t.end - t.begin + 1).replace('\n', QLatin1String("\\n"))
525             << QStringLiteral("[%0-%1] - [%2-%3]").arg(beginLine).arg(beginColumn).arg(endLine).arg(endColumn);
526             ++i;
527         }
528     }
529     return tokenStream;
530 }
531 }
532 
533