1 // Copyright (C) 2012-2015 Internet Systems Consortium, Inc. ("ISC")
2 //
3 // This Source Code Form is subject to the terms of the Mozilla Public
4 // License, v. 2.0. If a copy of the MPL was not distributed with this
5 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7 #include <config.h>
8
9 #include <dns/master_lexer.h>
10 #include <dns/master_lexer_inputsource.h>
11 #include <dns/master_lexer_state.h>
12
13 #include <gtest/gtest.h>
14
15 #include <sstream>
16
17 using namespace isc::dns;
18 using namespace master_lexer_internal;
19
20 namespace {
21 typedef MasterToken Token; // shortcut
22
23 class MasterLexerStateTest : public ::testing::Test {
24 protected:
MasterLexerStateTest()25 MasterLexerStateTest() : common_options(MasterLexer::INITIAL_WS),
26 s_null(NULL),
27 s_crlf(State::getInstance(State::CRLF)),
28 s_string(State::getInstance(State::String)),
29 s_qstring(State::getInstance(State::QString)),
30 s_number(State::getInstance(State::Number)),
31 options(MasterLexer::NONE),
32 orig_options(options)
33 {}
34
35 // Specify INITIAL_WS as common initial options.
36 const MasterLexer::Options common_options;
37 MasterLexer lexer;
38 const State* const s_null;
39 const State& s_crlf;
40 const State& s_string;
41 const State& s_qstring;
42 const State& s_number;
43 std::stringstream ss;
44 MasterLexer::Options options, orig_options;
45 };
46
47 // Common check for the end-of-file condition.
48 // Token is set to END_OF_FILE, and the lexer was NOT last eol state.
49 // Passed state can be any valid one; they are stateless, just providing the
50 // interface for inspection.
51 void
eofCheck(const State & state,MasterLexer & lexer)52 eofCheck(const State& state, MasterLexer& lexer) {
53 EXPECT_EQ(Token::END_OF_FILE, state.getToken(lexer).getType());
54 EXPECT_FALSE(state.wasLastEOL(lexer));
55 }
56
TEST_F(MasterLexerStateTest,startAndEnd)57 TEST_F(MasterLexerStateTest, startAndEnd) {
58 // A simple case: the input is empty, so we begin with start and
59 // are immediately done.
60 lexer.pushSource(ss);
61 EXPECT_EQ(s_null, State::start(lexer, common_options));
62 eofCheck(s_crlf, lexer);
63 }
64
TEST_F(MasterLexerStateTest,startToEOL)65 TEST_F(MasterLexerStateTest, startToEOL) {
66 ss << "\n";
67 lexer.pushSource(ss);
68
69 EXPECT_EQ(s_null, State::start(lexer, common_options));
70 EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
71 EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
72
73 // The next lexer session will reach EOF. Same eof check should pass.
74 EXPECT_EQ(s_null, State::start(lexer, common_options));
75 eofCheck(s_crlf, lexer);
76 }
77
TEST_F(MasterLexerStateTest,space)78 TEST_F(MasterLexerStateTest, space) {
79 // repeat '\t\n' twice (see below), then space after EOL
80 ss << " \t\n\t\n ";
81 lexer.pushSource(ss);
82
83 // by default space characters and tabs will be ignored. We check this
84 // twice; at the second iteration, it's a white space at the beginning
85 // of line, but since we don't specify INITIAL_WS option, it's treated as
86 // normal space and ignored.
87 for (size_t i = 0; i < 2; ++i) {
88 EXPECT_EQ(s_null, State::start(lexer, MasterLexer::NONE));
89 EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
90 EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
91 }
92
93 // Now we specify the INITIAL_WS option. It will be recognized and the
94 // corresponding token will be returned.
95 EXPECT_EQ(s_null, State::start(lexer, MasterLexer::INITIAL_WS));
96 EXPECT_FALSE(s_crlf.wasLastEOL(lexer));
97 EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
98 }
99
TEST_F(MasterLexerStateTest,parentheses)100 TEST_F(MasterLexerStateTest, parentheses) {
101 ss << "\n(\na\n )\n "; // 1st \n is to check if 'was EOL' is set to false
102 lexer.pushSource(ss);
103
104 EXPECT_EQ(s_null, State::start(lexer, common_options)); // handle \n
105
106 // Now handle '('. It skips \n and recognize 'a' as string
107 EXPECT_EQ(0, s_crlf.getParenCount(lexer)); // check pre condition
108 EXPECT_EQ(&s_string, State::start(lexer, common_options));
109 EXPECT_EQ(1, s_crlf.getParenCount(lexer)); // check post condition
110 EXPECT_FALSE(s_crlf.wasLastEOL(lexer));
111
112 // skip 'a'
113 s_string.handle(lexer);
114
115 // Then handle ')'. '\n' before ')' isn't recognized because
116 // it's canceled due to the '('. Likewise, the space after the '\n'
117 // shouldn't be recognized but should be just ignored.
118 EXPECT_EQ(s_null, State::start(lexer, common_options));
119 EXPECT_EQ(0, s_crlf.getParenCount(lexer));
120
121 // Now, temporarily disabled options are restored: Both EOL and the
122 // initial WS are recognized
123 EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
124 EXPECT_EQ(s_null, State::start(lexer, common_options));
125 EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
126 }
127
TEST_F(MasterLexerStateTest,nestedParentheses)128 TEST_F(MasterLexerStateTest, nestedParentheses) {
129 // This is an unusual, but allowed (in this implementation) case.
130 ss << "(a(b)\n c)\n ";
131 lexer.pushSource(ss);
132
133 EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '('
134 s_string.handle(lexer); // consume 'a'
135 EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '('
136 s_string.handle(lexer); // consume 'b'
137 EXPECT_EQ(2, s_crlf.getParenCount(lexer)); // now the count is 2
138
139 // Close the inner most parentheses. count will be decreased, but option
140 // shouldn't be restored yet, so the intermediate EOL or initial WS won't
141 // be recognized.
142 EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume ')'
143 s_string.handle(lexer); // consume 'c'
144 EXPECT_EQ(1, s_crlf.getParenCount(lexer));
145
146 // Close the outermost parentheses. count will be reset to 0, and original
147 // options are restored.
148 EXPECT_EQ(s_null, State::start(lexer, common_options));
149
150 // Now, temporarily disabled options are restored: Both EOL and the
151 // initial WS are recognized
152 EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
153 EXPECT_EQ(s_null, State::start(lexer, common_options));
154 EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
155 }
156
TEST_F(MasterLexerStateTest,unbalancedParentheses)157 TEST_F(MasterLexerStateTest, unbalancedParentheses) {
158 // Only closing paren is provided. We prepend a \n to check if it's
159 // correctly canceled after detecting the error.
160 ss << "\n)";
161 ss << "(a";
162 lexer.pushSource(ss);
163
164 EXPECT_EQ(s_null, State::start(lexer, common_options)); // consume '\n'
165 EXPECT_TRUE(s_crlf.wasLastEOL(lexer)); // this \n was remembered
166
167 // Now checking ')'. The result should be error, count shouldn't be
168 // changed. "last EOL" should be canceled.
169 EXPECT_EQ(0, s_crlf.getParenCount(lexer));
170 EXPECT_EQ(s_null, State::start(lexer, common_options));
171 EXPECT_EQ(0, s_crlf.getParenCount(lexer));
172 ASSERT_EQ(Token::ERROR, s_crlf.getToken(lexer).getType());
173 EXPECT_EQ(Token::UNBALANCED_PAREN, s_crlf.getToken(lexer).getErrorCode());
174 EXPECT_FALSE(s_crlf.wasLastEOL(lexer));
175
176 // Reach EOF with a dangling open parenthesis.
177 EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '('
178 s_string.handle(lexer); // consume 'a'
179 EXPECT_EQ(1, s_crlf.getParenCount(lexer));
180 EXPECT_EQ(s_null, State::start(lexer, common_options)); // reach EOF
181 ASSERT_EQ(Token::ERROR, s_crlf.getToken(lexer).getType());
182 EXPECT_EQ(Token::UNBALANCED_PAREN, s_crlf.getToken(lexer).getErrorCode());
183 EXPECT_EQ(0, s_crlf.getParenCount(lexer)); // should be reset to 0
184 }
185
TEST_F(MasterLexerStateTest,startToComment)186 TEST_F(MasterLexerStateTest, startToComment) {
187 // Begin with 'start', detect space, then encounter a comment. Skip
188 // the rest of the line, and recognize the new line. Note that the
189 // second ';' is simply ignored.
190 ss << " ;a;\n";
191 ss << ";a;"; // Likewise, but the comment ends with EOF.
192 lexer.pushSource(ss);
193
194 // Initial whitespace (asked for in common_options)
195 EXPECT_EQ(s_null, State::start(lexer, common_options));
196 EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
197 // Comment ending with EOL
198 EXPECT_EQ(s_null, State::start(lexer, common_options));
199 EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
200
201 // Comment ending with EOF
202 EXPECT_EQ(s_null, State::start(lexer, common_options));
203 EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
204 }
205
TEST_F(MasterLexerStateTest,commentAfterParen)206 TEST_F(MasterLexerStateTest, commentAfterParen) {
207 // comment after an opening parenthesis. The code that is tested by
208 // other tests should also ensure that it works correctly, but we
209 // check it explicitly.
210 ss << "( ;this is a comment\na)\n";
211 lexer.pushSource(ss);
212
213 // consume '(', skip comments, consume 'a', then consume ')'
214 EXPECT_EQ(&s_string, State::start(lexer, common_options));
215 s_string.handle(lexer);
216 EXPECT_EQ(s_null, State::start(lexer, common_options));
217 EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
218 }
219
TEST_F(MasterLexerStateTest,crlf)220 TEST_F(MasterLexerStateTest, crlf) {
221 ss << "\r\n"; // case 1
222 ss << "\r "; // case 2
223 ss << "\r;comment\na"; // case 3
224 ss << "\r"; // case 4
225 lexer.pushSource(ss);
226
227 // 1. A sequence of \r, \n is recognized as a single 'end-of-line'
228 EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
229 s_crlf.handle(lexer); // recognize '\n'
230 EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
231 EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
232
233 // 2. Single '\r' (not followed by \n) is recognized as a single
234 // 'end-of-line'. then there will be "initial WS"
235 EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
236 // see ' ', "unget" it
237 s_crlf.handle(lexer);
238 EXPECT_EQ(s_null, State::start(lexer, common_options)); // recognize ' '
239 EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
240
241 // 3. comment between \r and \n
242 EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
243 // skip comments, recognize '\n'
244 s_crlf.handle(lexer);
245 EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
246 EXPECT_EQ(&s_string, State::start(lexer, common_options));
247 s_string.handle(lexer); // skip 'a'
248
249 // 4. \r then EOF
250 EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
251 // see EOF, then "unget" it
252 s_crlf.handle(lexer);
253 EXPECT_EQ(s_null, State::start(lexer, common_options)); // recognize EOF
254 EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
255 }
256
257 // Commonly used check for string related test cases, checking if the given
258 // token has expected values.
259 void
stringTokenCheck(const std::string & expected,const MasterToken & token,bool quoted=false)260 stringTokenCheck(const std::string& expected, const MasterToken& token,
261 bool quoted = false)
262 {
263 EXPECT_EQ(quoted ? Token::QSTRING : Token::STRING, token.getType());
264 EXPECT_EQ(expected, token.getString());
265 const std::string actual(token.getStringRegion().beg,
266 token.getStringRegion().beg +
267 token.getStringRegion().len);
268 EXPECT_EQ(expected, actual);
269
270 // There should be "hidden" nul-terminator after the string data.
271 ASSERT_NE(static_cast<const char*>(NULL), token.getStringRegion().beg);
272 EXPECT_EQ(0, *(token.getStringRegion().beg + token.getStringRegion().len));
273 }
274
TEST_F(MasterLexerStateTest,string)275 TEST_F(MasterLexerStateTest, string) {
276 // Check with simple strings followed by separate characters
277 ss << "followed-by-EOL\n";
278 ss << "followed-by-CR\r";
279 ss << "followed-by-space ";
280 ss << "followed-by-tab\t";
281 ss << "followed-by-comment;this is comment and ignored\n";
282 ss << "followed-by-paren(closing)";
283 ss << "followed-by-EOF";
284 lexer.pushSource(ss);
285
286 EXPECT_EQ(&s_string, State::start(lexer, common_options));
287 s_string.handle(lexer); // recognize str, see \n
288 EXPECT_FALSE(s_string.wasLastEOL(lexer));
289 stringTokenCheck("followed-by-EOL", s_string.getToken(lexer));
290 EXPECT_EQ(s_null, State::start(lexer, common_options)); // skip \n
291
292 EXPECT_EQ(&s_string, State::start(lexer, common_options));
293 s_string.handle(lexer); // recognize str, see \r
294 stringTokenCheck("followed-by-CR", s_string.getToken(lexer));
295 EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // handle \r...
296 s_crlf.handle(lexer); // ...and skip it
297
298 EXPECT_EQ(&s_string, State::start(lexer, common_options));
299 s_string.handle(lexer); // recognize str, see ' '
300 stringTokenCheck("followed-by-space", s_string.getToken(lexer));
301
302 // skip ' ', then recognize the next string
303 EXPECT_EQ(&s_string, State::start(lexer, common_options));
304 s_string.handle(lexer); // recognize str, see \t
305 stringTokenCheck("followed-by-tab", s_string.getToken(lexer));
306
307 // skip \t, then recognize the next string
308 EXPECT_EQ(&s_string, State::start(lexer, common_options));
309 s_string.handle(lexer); // recognize str, see comment
310 stringTokenCheck("followed-by-comment", s_string.getToken(lexer));
311 EXPECT_EQ(s_null, State::start(lexer, common_options)); // skip \n after it
312
313 EXPECT_EQ(&s_string, State::start(lexer, common_options));
314 s_string.handle(lexer); // recognize str, see '('
315 stringTokenCheck("followed-by-paren", s_string.getToken(lexer));
316 EXPECT_EQ(&s_string, State::start(lexer, common_options)); // str in ()
317 s_string.handle(lexer); // recognize the str, see ')'
318 stringTokenCheck("closing", s_string.getToken(lexer));
319
320 EXPECT_EQ(&s_string, State::start(lexer, common_options));
321 s_string.handle(lexer); // recognize str, see EOF
322 stringTokenCheck("followed-by-EOF", s_string.getToken(lexer));
323 }
324
TEST_F(MasterLexerStateTest,stringEscape)325 TEST_F(MasterLexerStateTest, stringEscape) {
326 // some of the separate characters should be considered part of the
327 // string if escaped.
328 ss << "escaped\\ space ";
329 ss << "escaped\\\ttab ";
330 ss << "escaped\\(paren ";
331 ss << "escaped\\)close ";
332 ss << "escaped\\;comment ";
333 ss << "escaped\\\\ backslash "; // second '\' shouldn't escape ' '
334 lexer.pushSource(ss);
335
336 EXPECT_EQ(&s_string, State::start(lexer, common_options));
337 s_string.handle(lexer); // recognize str, see ' ' at end
338 stringTokenCheck("escaped\\ space", s_string.getToken(lexer));
339
340 EXPECT_EQ(&s_string, State::start(lexer, common_options));
341 s_string.handle(lexer); // recognize str, see ' ' at end
342 stringTokenCheck("escaped\\\ttab", s_string.getToken(lexer));
343
344 EXPECT_EQ(&s_string, State::start(lexer, common_options));
345 s_string.handle(lexer); // recognize str, see ' ' at end
346 stringTokenCheck("escaped\\(paren", s_string.getToken(lexer));
347
348 EXPECT_EQ(&s_string, State::start(lexer, common_options));
349 s_string.handle(lexer); // recognize str, see ' ' at end
350 stringTokenCheck("escaped\\)close", s_string.getToken(lexer));
351
352 EXPECT_EQ(&s_string, State::start(lexer, common_options));
353 s_string.handle(lexer); // recognize str, see ' ' at end
354 stringTokenCheck("escaped\\;comment", s_string.getToken(lexer));
355
356 EXPECT_EQ(&s_string, State::start(lexer, common_options));
357 s_string.handle(lexer); // recognize str, see ' ' in mid
358 stringTokenCheck("escaped\\\\", s_string.getToken(lexer));
359
360 // Confirm the word that follows the escaped '\' is correctly recognized.
361 EXPECT_EQ(&s_string, State::start(lexer, common_options));
362 s_string.handle(lexer); // recognize str, see ' ' at end
363 stringTokenCheck("backslash", s_string.getToken(lexer));
364 }
365
TEST_F(MasterLexerStateTest,quotedString)366 TEST_F(MasterLexerStateTest, quotedString) {
367 ss << "\"ignore-quotes\"\n";
368 ss << "\"quoted string\" "; // space is part of the qstring
369 ss << "\"\" "; // empty quoted string
370 // also check other separator characters. note that \r doesn't cause
371 // UNBALANCED_QUOTES. Not sure if it's intentional, but that's how the
372 // BIND 9 version works, so we follow it (it should be too minor to matter
373 // in practice anyway)
374 ss << "\"quoted()\t\rstring\" ";
375 ss << "\"escape\\ in quote\" ";
376 ss << "\"escaped\\\"\" ";
377 ss << "\"escaped backslash\\\\\" ";
378 ss << "\"no;comment\"";
379 lexer.pushSource(ss);
380
381 // by default, '"' is unexpected (when QSTRING is not specified),
382 // and it returns MasterToken::UNEXPECTED_QUOTES.
383 EXPECT_EQ(s_null, State::start(lexer, common_options));
384 EXPECT_EQ(Token::UNEXPECTED_QUOTES, s_string.getToken(lexer).getErrorCode());
385 // Read it as a QSTRING.
386 s_qstring.handle(lexer); // recognize quoted str, see \n
387 stringTokenCheck("ignore-quotes", s_qstring.getToken(lexer), true);
388 EXPECT_EQ(s_null, State::start(lexer, common_options)); // skip \n after it
389 EXPECT_TRUE(s_string.wasLastEOL(lexer));
390
391 // If QSTRING is specified in option, '"' is regarded as a beginning of
392 // a quoted string.
393 const MasterLexer::Options options = common_options | MasterLexer::QSTRING;
394 EXPECT_EQ(&s_qstring, State::start(lexer, options));
395 EXPECT_FALSE(s_string.wasLastEOL(lexer)); // EOL is canceled due to '"'
396 s_qstring.handle(lexer);
397 stringTokenCheck("quoted string", s_string.getToken(lexer), true);
398
399 // Empty string is okay as qstring
400 EXPECT_EQ(&s_qstring, State::start(lexer, options));
401 s_qstring.handle(lexer);
402 stringTokenCheck("", s_string.getToken(lexer), true);
403
404 // Also checks other separator characters within a qstring
405 EXPECT_EQ(&s_qstring, State::start(lexer, options));
406 s_qstring.handle(lexer);
407 stringTokenCheck("quoted()\t\rstring", s_string.getToken(lexer), true);
408
409 // escape character mostly doesn't have any effect in the qstring
410 // processing
411 EXPECT_EQ(&s_qstring, State::start(lexer, options));
412 s_qstring.handle(lexer);
413 stringTokenCheck("escape\\ in quote", s_string.getToken(lexer), true);
414
415 // The only exception is the quotation mark itself. Note that the escape
416 // only works on the quotation mark immediately after it.
417 EXPECT_EQ(&s_qstring, State::start(lexer, options));
418 s_qstring.handle(lexer);
419 stringTokenCheck("escaped\"", s_string.getToken(lexer), true);
420
421 // quoted '\' then '"'. Unlike the previous case '"' shouldn't be
422 // escaped.
423 EXPECT_EQ(&s_qstring, State::start(lexer, options));
424 s_qstring.handle(lexer);
425 stringTokenCheck("escaped backslash\\\\", s_string.getToken(lexer), true);
426
427 // ';' has no meaning in a quoted string (not indicating a comment)
428 EXPECT_EQ(&s_qstring, State::start(lexer, options));
429 s_qstring.handle(lexer);
430 stringTokenCheck("no;comment", s_string.getToken(lexer), true);
431 }
432
TEST_F(MasterLexerStateTest,brokenQuotedString)433 TEST_F(MasterLexerStateTest, brokenQuotedString) {
434 ss << "\"unbalanced-quote\n";
435 ss << "\"quoted\\\n\" ";
436 ss << "\"unclosed quote and EOF";
437 lexer.pushSource(ss);
438
439 // EOL is encountered without closing the quote
440 const MasterLexer::Options options = common_options | MasterLexer::QSTRING;
441 EXPECT_EQ(&s_qstring, State::start(lexer, options));
442 s_qstring.handle(lexer);
443 ASSERT_EQ(Token::ERROR, s_qstring.getToken(lexer).getType());
444 EXPECT_EQ(Token::UNBALANCED_QUOTES,
445 s_qstring.getToken(lexer).getErrorCode());
446 // We can resume after the error from the '\n'
447 EXPECT_EQ(s_null, State::start(lexer, options));
448 EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
449
450 // \n is okay in a quoted string if escaped
451 EXPECT_EQ(&s_qstring, State::start(lexer, options));
452 s_qstring.handle(lexer);
453 stringTokenCheck("quoted\\\n", s_string.getToken(lexer), true);
454
455 // EOF is encountered without closing the quote
456 EXPECT_EQ(&s_qstring, State::start(lexer, options));
457 s_qstring.handle(lexer);
458 ASSERT_EQ(Token::ERROR, s_qstring.getToken(lexer).getType());
459 EXPECT_EQ(Token::UNEXPECTED_END, s_qstring.getToken(lexer).getErrorCode());
460 // If we continue we'll simply see the EOF
461 EXPECT_EQ(s_null, State::start(lexer, options));
462 EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
463 }
464
TEST_F(MasterLexerStateTest,basicNumbers)465 TEST_F(MasterLexerStateTest, basicNumbers) {
466 ss << "0 ";
467 ss << "1 ";
468 ss << "12345 ";
469 ss << "4294967295 "; // 2^32-1
470 ss << "4294967296 "; // Out of range
471 ss << "340282366920938463463374607431768211456 ";
472 // Very much out of range (2^128)
473 ss << "005 "; // Leading zeroes are ignored
474 ss << "42;asdf\n"; // Number with comment
475 ss << "37"; // Simple number again, here to make
476 // sure none of the above messed up
477 // the tokenizer
478 lexer.pushSource(ss);
479
480 // Ask the lexer to recognize numbers as well
481 const MasterLexer::Options options = common_options | MasterLexer::NUMBER;
482
483 EXPECT_EQ(&s_number, State::start(lexer, options));
484 s_number.handle(lexer);
485 EXPECT_EQ(0, s_number.getToken(lexer).getNumber());
486
487 EXPECT_EQ(&s_number, State::start(lexer, options));
488 s_number.handle(lexer);
489 EXPECT_EQ(1, s_number.getToken(lexer).getNumber());
490
491 EXPECT_EQ(&s_number, State::start(lexer, options));
492 s_number.handle(lexer);
493 EXPECT_EQ(12345, s_number.getToken(lexer).getNumber());
494
495 EXPECT_EQ(&s_number, State::start(lexer, options));
496 s_number.handle(lexer);
497 EXPECT_EQ(4294967295u, s_number.getToken(lexer).getNumber());
498
499 EXPECT_EQ(&s_number, State::start(lexer, options));
500 s_number.handle(lexer);
501 EXPECT_EQ(Token::NUMBER_OUT_OF_RANGE,
502 s_number.getToken(lexer).getErrorCode());
503
504 EXPECT_EQ(&s_number, State::start(lexer, options));
505 s_number.handle(lexer);
506 EXPECT_EQ(Token::NUMBER_OUT_OF_RANGE,
507 s_number.getToken(lexer).getErrorCode());
508
509 EXPECT_EQ(&s_number, State::start(lexer, options));
510 s_number.handle(lexer);
511 EXPECT_EQ(5, s_number.getToken(lexer).getNumber());
512
513 EXPECT_EQ(&s_number, State::start(lexer, options));
514 s_number.handle(lexer);
515 EXPECT_EQ(42, s_number.getToken(lexer).getNumber());
516
517 EXPECT_EQ(s_null, State::start(lexer, options));
518 EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
519 EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
520
521 EXPECT_EQ(&s_number, State::start(lexer, options));
522 s_number.handle(lexer);
523 EXPECT_EQ(37, s_number.getToken(lexer).getNumber());
524
525 // If we continue we'll simply see the EOF
526 EXPECT_EQ(s_null, State::start(lexer, options));
527 EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
528 }
529
530 // Test tokens that look like (or start out as) numbers,
531 // but turn out to be strings. Tests include escaped characters.
TEST_F(MasterLexerStateTest,stringNumbers)532 TEST_F(MasterLexerStateTest, stringNumbers) {
533 ss << "123 "; // Should be read as a string if the
534 // NUMBER option is not given
535 ss << "-1 "; // Negative numbers are interpreted
536 // as strings (unsigned integers only)
537 ss << "123abc456 "; // 'Numbers' containing non-digits should
538 // be interpreted as strings
539 ss << "123\\456 "; // Numbers containing escaped digits are
540 // interpreted as strings
541 ss << "3scaped\\ space ";
542 ss << "3scaped\\\ttab ";
543 ss << "3scaped\\(paren ";
544 ss << "3scaped\\)close ";
545 ss << "3scaped\\;comment ";
546 ss << "3scaped\\\\ 8ackslash "; // second '\' shouldn't escape ' '
547
548 lexer.pushSource(ss);
549
550 // Note that common_options does not include MasterLexer::NUMBER,
551 // so the token should be recognized as a string
552 EXPECT_EQ(&s_string, State::start(lexer, common_options));
553 s_string.handle(lexer);
554 stringTokenCheck("123", s_string.getToken(lexer), false);
555
556 // Ask the lexer to recognize numbers as well
557 const MasterLexer::Options options = common_options | MasterLexer::NUMBER;
558
559 EXPECT_EQ(&s_string, State::start(lexer, options));
560 s_string.handle(lexer);
561 stringTokenCheck("-1", s_string.getToken(lexer), false);
562
563 // Starts out as a number, but ends up being a string
564 EXPECT_EQ(&s_number, State::start(lexer, options));
565 s_number.handle(lexer);
566 stringTokenCheck("123abc456", s_number.getToken(lexer), false);
567
568 EXPECT_EQ(&s_number, State::start(lexer, options));
569 s_number.handle(lexer);
570 stringTokenCheck("123\\456", s_number.getToken(lexer), false);
571
572 EXPECT_EQ(&s_number, State::start(lexer, options));
573 s_number.handle(lexer); // recognize str, see ' ' at end
574 stringTokenCheck("3scaped\\ space", s_number.getToken(lexer));
575
576 EXPECT_EQ(&s_number, State::start(lexer, options));
577 s_number.handle(lexer); // recognize str, see ' ' at end
578 stringTokenCheck("3scaped\\\ttab", s_number.getToken(lexer));
579
580 EXPECT_EQ(&s_number, State::start(lexer, options));
581 s_number.handle(lexer); // recognize str, see ' ' at end
582 stringTokenCheck("3scaped\\(paren", s_number.getToken(lexer));
583
584 EXPECT_EQ(&s_number, State::start(lexer, options));
585 s_number.handle(lexer); // recognize str, see ' ' at end
586 stringTokenCheck("3scaped\\)close", s_number.getToken(lexer));
587
588 EXPECT_EQ(&s_number, State::start(lexer, options));
589 s_number.handle(lexer); // recognize str, see ' ' at end
590 stringTokenCheck("3scaped\\;comment", s_number.getToken(lexer));
591
592 EXPECT_EQ(&s_number, State::start(lexer, options));
593 s_number.handle(lexer); // recognize str, see ' ' in mid
594 stringTokenCheck("3scaped\\\\", s_number.getToken(lexer));
595
596 // Confirm the word that follows the escaped '\' is correctly recognized.
597 EXPECT_EQ(&s_number, State::start(lexer, options));
598 s_number.handle(lexer); // recognize str, see ' ' at end
599 stringTokenCheck("8ackslash", s_number.getToken(lexer));
600
601 // If we continue we'll simply see the EOF
602 EXPECT_EQ(s_null, State::start(lexer, options));
603 EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
604 }
605
606 } // end anonymous namespace
607
608