1 /* 2 * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. 3 * Use of this file is governed by the BSD 3-clause license that 4 * can be found in the LICENSE.txt file in the project root. 5 */ 6 7 package org.antlr.v4.test.tool; 8 9 import org.antlr.v4.runtime.atn.ATN; 10 import org.antlr.v4.runtime.atn.ATNSerializer; 11 import org.antlr.v4.tool.DOTGenerator; 12 import org.antlr.v4.tool.Grammar; 13 import org.antlr.v4.tool.LexerGrammar; 14 import org.junit.Before; 15 import org.junit.Test; 16 17 import java.util.Arrays; 18 19 import static org.junit.Assert.assertEquals; 20 21 public class TestATNSerialization extends BaseJavaToolTest { 22 @Before 23 @Override testSetUp()24 public void testSetUp() throws Exception { 25 super.testSetUp(); 26 } 27 testSimpleNoBlock()28 @Test public void testSimpleNoBlock() throws Exception { 29 Grammar g = new Grammar( 30 "parser grammar T;\n"+ 31 "a : A B ;"); 32 String expecting = 33 "max type 2\n" + 34 "0:RULE_START 0\n" + 35 "1:RULE_STOP 0\n" + 36 "2:BASIC 0\n" + 37 "3:BASIC 0\n" + 38 "4:BASIC 0\n" + 39 "5:BASIC 0\n" + 40 "rule 0:0\n" + 41 "0->2 EPSILON 0,0,0\n" + 42 "2->3 ATOM 1,0,0\n" + 43 "3->4 ATOM 2,0,0\n" + 44 "4->1 EPSILON 0,0,0\n"; 45 ATN atn = createATN(g, true); 46 String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames())); 47 assertEquals(expecting, result); 48 } 49 testEOF()50 @Test public void testEOF() throws Exception { 51 Grammar g = new Grammar( 52 "parser grammar T;\n"+ 53 "a : A EOF ;"); 54 String expecting = 55 "max type 1\n" + 56 "0:RULE_START 0\n" + 57 "1:RULE_STOP 0\n" + 58 "2:BASIC 0\n" + 59 "3:BASIC 0\n" + 60 "4:BASIC 0\n" + 61 "5:BASIC 0\n" + 62 "rule 0:0\n" + 63 "0->2 EPSILON 0,0,0\n" + 64 "2->3 ATOM 1,0,0\n" + 65 "3->4 ATOM 0,0,1\n" + 66 "4->1 EPSILON 0,0,0\n"; 67 ATN atn = createATN(g, true); 68 String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames())); 69 assertEquals(expecting, result); 70 } 71 testEOFInSet()72 @Test public void testEOFInSet() throws Exception { 73 Grammar g = new Grammar( 74 "parser grammar T;\n"+ 75 "a : (A|EOF) ;"); 76 String expecting = 77 "max type 1\n" + 78 "0:RULE_START 0\n" + 79 "1:RULE_STOP 0\n" + 80 "2:BASIC 0\n" + 81 "3:BASIC 0\n" + 82 "4:BASIC 0\n" + 83 "rule 0:0\n" + 84 "0:EOF, A..A\n" + 85 "0->2 EPSILON 0,0,0\n" + 86 "2->3 SET 0,0,0\n" + 87 "3->1 EPSILON 0,0,0\n"; 88 ATN atn = createATN(g, true); 89 String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames())); 90 assertEquals(expecting, result); 91 } 92 testNot()93 @Test public void testNot() throws Exception { 94 Grammar g = new Grammar( 95 "parser grammar T;\n"+ 96 "tokens {A, B, C}\n" + 97 "a : ~A ;"); 98 String expecting = 99 "max type 3\n" + 100 "0:RULE_START 0\n" + 101 "1:RULE_STOP 0\n" + 102 "2:BASIC 0\n" + 103 "3:BASIC 0\n" + 104 "4:BASIC 0\n" + 105 "rule 0:0\n" + 106 "0:A..A\n" + 107 "0->2 EPSILON 0,0,0\n" + 108 "2->3 NOT_SET 0,0,0\n" + 109 "3->1 EPSILON 0,0,0\n"; 110 ATN atn = createATN(g, true); 111 DOTGenerator gen = new DOTGenerator(g); 112 String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames())); 113 assertEquals(expecting, result); 114 } 115 testWildcard()116 @Test public void testWildcard() throws Exception { 117 Grammar g = new Grammar( 118 "parser grammar T;\n"+ 119 "tokens {A, B, C}\n" + 120 "a : . ;"); 121 String expecting = 122 "max type 3\n" + 123 "0:RULE_START 0\n" + 124 "1:RULE_STOP 0\n" + 125 "2:BASIC 0\n" + 126 "3:BASIC 0\n" + 127 "4:BASIC 0\n" + 128 "rule 0:0\n" + 129 "0->2 EPSILON 0,0,0\n" + 130 "2->3 WILDCARD 0,0,0\n" + 131 "3->1 EPSILON 0,0,0\n"; 132 ATN atn = createATN(g, true); 133 String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames())); 134 assertEquals(expecting, result); 135 } 136 testPEGAchillesHeel()137 @Test public void testPEGAchillesHeel() throws Exception { 138 Grammar g = new Grammar( 139 "parser grammar T;\n"+ 140 "a : A | A B ;"); 141 String expecting = 142 "max type 2\n" + 143 "0:RULE_START 0\n" + 144 "1:RULE_STOP 0\n" + 145 "2:BASIC 0\n" + 146 "3:BASIC 0\n" + 147 "4:BASIC 0\n" + 148 "5:BLOCK_START 0 6\n" + 149 "6:BLOCK_END 0\n" + 150 "7:BASIC 0\n" + 151 "rule 0:0\n" + 152 "0->5 EPSILON 0,0,0\n" + 153 "2->6 ATOM 1,0,0\n" + 154 "3->4 ATOM 1,0,0\n" + 155 "4->6 ATOM 2,0,0\n" + 156 "5->2 EPSILON 0,0,0\n" + 157 "5->3 EPSILON 0,0,0\n" + 158 "6->1 EPSILON 0,0,0\n" + 159 "0:5\n"; 160 ATN atn = createATN(g, true); 161 String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames())); 162 assertEquals(expecting, result); 163 } 164 test3Alts()165 @Test public void test3Alts() throws Exception { 166 Grammar g = new Grammar( 167 "parser grammar T;\n"+ 168 "a : A | A B | A B C ;"); 169 String expecting = 170 "max type 3\n" + 171 "0:RULE_START 0\n" + 172 "1:RULE_STOP 0\n" + 173 "2:BASIC 0\n" + 174 "3:BASIC 0\n" + 175 "4:BASIC 0\n" + 176 "5:BASIC 0\n" + 177 "6:BASIC 0\n" + 178 "7:BASIC 0\n" + 179 "8:BLOCK_START 0 9\n" + 180 "9:BLOCK_END 0\n" + 181 "10:BASIC 0\n" + 182 "rule 0:0\n" + 183 "0->8 EPSILON 0,0,0\n" + 184 "2->9 ATOM 1,0,0\n" + 185 "3->4 ATOM 1,0,0\n" + 186 "4->9 ATOM 2,0,0\n" + 187 "5->6 ATOM 1,0,0\n" + 188 "6->7 ATOM 2,0,0\n" + 189 "7->9 ATOM 3,0,0\n" + 190 "8->2 EPSILON 0,0,0\n" + 191 "8->3 EPSILON 0,0,0\n" + 192 "8->5 EPSILON 0,0,0\n" + 193 "9->1 EPSILON 0,0,0\n" + 194 "0:8\n"; 195 ATN atn = createATN(g, true); 196 String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames())); 197 assertEquals(expecting, result); 198 } 199 testSimpleLoop()200 @Test public void testSimpleLoop() throws Exception { 201 Grammar g = new Grammar( 202 "parser grammar T;\n"+ 203 "a : A+ B ;"); 204 String expecting = 205 "max type 2\n" + 206 "0:RULE_START 0\n" + 207 "1:RULE_STOP 0\n" + 208 "2:BASIC 0\n" + 209 "3:PLUS_BLOCK_START 0 4\n" + 210 "4:BLOCK_END 0\n" + 211 "5:PLUS_LOOP_BACK 0\n" + 212 "6:LOOP_END 0 5\n" + 213 "7:BASIC 0\n" + 214 "8:BASIC 0\n" + 215 "9:BASIC 0\n" + 216 "rule 0:0\n" + 217 "0->3 EPSILON 0,0,0\n" + 218 "2->4 ATOM 1,0,0\n" + 219 "3->2 EPSILON 0,0,0\n" + 220 "4->5 EPSILON 0,0,0\n" + 221 "5->3 EPSILON 0,0,0\n" + 222 "5->6 EPSILON 0,0,0\n" + 223 "6->7 EPSILON 0,0,0\n" + 224 "7->8 ATOM 2,0,0\n" + 225 "8->1 EPSILON 0,0,0\n" + 226 "0:5\n"; 227 ATN atn = createATN(g, true); 228 String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames())); 229 assertEquals(expecting, result); 230 } 231 testRuleRef()232 @Test public void testRuleRef() throws Exception { 233 Grammar g = new Grammar( 234 "parser grammar T;\n"+ 235 "a : e ;\n" + 236 "e : E ;\n"); 237 String expecting = 238 "max type 1\n" + 239 "0:RULE_START 0\n" + 240 "1:RULE_STOP 0\n" + 241 "2:RULE_START 1\n" + 242 "3:RULE_STOP 1\n" + 243 "4:BASIC 0\n" + 244 "5:BASIC 0\n" + 245 "6:BASIC 1\n" + 246 "7:BASIC 1\n" + 247 "8:BASIC 1\n" + 248 "rule 0:0\n" + 249 "rule 1:2\n" + 250 "0->4 EPSILON 0,0,0\n" + 251 "2->6 EPSILON 0,0,0\n" + 252 "4->5 RULE 2,1,0\n" + 253 "5->1 EPSILON 0,0,0\n" + 254 "6->7 ATOM 1,0,0\n" + 255 "7->3 EPSILON 0,0,0\n"; 256 ATN atn = createATN(g, true); 257 String result = ATNSerializer.getDecoded(atn, Arrays.asList(g.getTokenNames())); 258 assertEquals(expecting, result); 259 } 260 testLexerTwoRules()261 @Test public void testLexerTwoRules() throws Exception { 262 LexerGrammar lg = new LexerGrammar( 263 "lexer grammar L;\n"+ 264 "A : 'a' ;\n" + 265 "B : 'b' ;\n"); 266 String expecting = 267 "max type 2\n" + 268 "0:TOKEN_START -1\n" + 269 "1:RULE_START 0\n" + 270 "2:RULE_STOP 0\n" + 271 "3:RULE_START 1\n" + 272 "4:RULE_STOP 1\n" + 273 "5:BASIC 0\n" + 274 "6:BASIC 0\n" + 275 "7:BASIC 1\n" + 276 "8:BASIC 1\n" + 277 "rule 0:1 1\n" + 278 "rule 1:3 2\n" + 279 "mode 0:0\n" + 280 "0->1 EPSILON 0,0,0\n" + 281 "0->3 EPSILON 0,0,0\n" + 282 "1->5 EPSILON 0,0,0\n" + 283 "3->7 EPSILON 0,0,0\n" + 284 "5->6 ATOM 97,0,0\n" + 285 "6->2 EPSILON 0,0,0\n" + 286 "7->8 ATOM 98,0,0\n" + 287 "8->4 EPSILON 0,0,0\n" + 288 "0:0\n"; 289 ATN atn = createATN(lg, true); 290 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 291 assertEquals(expecting, result); 292 } 293 testLexerUnicodeSMPLiteralSerializedToSet()294 @Test public void testLexerUnicodeSMPLiteralSerializedToSet() throws Exception { 295 LexerGrammar lg = new LexerGrammar( 296 "lexer grammar L;\n"+ 297 "INT : '\\u{1F4A9}' ;"); 298 String expecting = 299 "max type 1\n" + 300 "0:TOKEN_START -1\n" + 301 "1:RULE_START 0\n" + 302 "2:RULE_STOP 0\n" + 303 "3:BASIC 0\n" + 304 "4:BASIC 0\n" + 305 "rule 0:1 1\n" + 306 "mode 0:0\n" + 307 "0:128169..128169\n" + 308 "0->1 EPSILON 0,0,0\n" + 309 "1->3 EPSILON 0,0,0\n" + 310 "3->4 SET 0,0,0\n" + 311 "4->2 EPSILON 0,0,0\n" + 312 "0:0\n"; 313 ATN atn = createATN(lg, true); 314 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 315 assertEquals(expecting, result); 316 } 317 testLexerUnicodeSMPRangeSerializedToSet()318 @Test public void testLexerUnicodeSMPRangeSerializedToSet() throws Exception { 319 LexerGrammar lg = new LexerGrammar( 320 "lexer grammar L;\n"+ 321 "INT : ('a'..'\\u{1F4A9}') ;"); 322 String expecting = 323 "max type 1\n" + 324 "0:TOKEN_START -1\n" + 325 "1:RULE_START 0\n" + 326 "2:RULE_STOP 0\n" + 327 "3:BASIC 0\n" + 328 "4:BASIC 0\n" + 329 "rule 0:1 1\n" + 330 "mode 0:0\n" + 331 "0:'a'..128169\n" + 332 "0->1 EPSILON 0,0,0\n" + 333 "1->3 EPSILON 0,0,0\n" + 334 "3->4 SET 0,0,0\n" + 335 "4->2 EPSILON 0,0,0\n" + 336 "0:0\n"; 337 ATN atn = createATN(lg, true); 338 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 339 assertEquals(expecting, result); 340 } 341 testLexerUnicodeSMPSetSerializedAfterBMPSet()342 @Test public void testLexerUnicodeSMPSetSerializedAfterBMPSet() throws Exception { 343 LexerGrammar lg = new LexerGrammar( 344 "lexer grammar L;\n"+ 345 "SMP : ('\\u{1F4A9}' | '\\u{1F4AF}') ;\n"+ 346 "BMP : ('a' | 'x') ;"); 347 String expecting = 348 "max type 2\n" + 349 "0:TOKEN_START -1\n" + 350 "1:RULE_START 0\n" + 351 "2:RULE_STOP 0\n" + 352 "3:RULE_START 1\n" + 353 "4:RULE_STOP 1\n" + 354 "5:BASIC 0\n" + 355 "6:BASIC 0\n" + 356 "7:BASIC 1\n" + 357 "8:BASIC 1\n" + 358 "rule 0:1 1\n" + 359 "rule 1:3 2\n" + 360 "mode 0:0\n" + 361 "0:'a'..'a', 'x'..'x'\n" + 362 "1:128169..128169, 128175..128175\n" + 363 "0->1 EPSILON 0,0,0\n" + 364 "0->3 EPSILON 0,0,0\n" + 365 "1->5 EPSILON 0,0,0\n" + 366 "3->7 EPSILON 0,0,0\n" + 367 "5->6 SET 1,0,0\n" + 368 "6->2 EPSILON 0,0,0\n" + 369 "7->8 SET 0,0,0\n" + 370 "8->4 EPSILON 0,0,0\n" + 371 "0:0\n"; 372 ATN atn = createATN(lg, true); 373 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 374 assertEquals(expecting, result); 375 } 376 testLexerNotLiteral()377 @Test public void testLexerNotLiteral() throws Exception { 378 LexerGrammar lg = new LexerGrammar( 379 "lexer grammar L;\n"+ 380 "INT : ~'a' ;"); 381 String expecting = 382 "max type 1\n" + 383 "0:TOKEN_START -1\n" + 384 "1:RULE_START 0\n" + 385 "2:RULE_STOP 0\n" + 386 "3:BASIC 0\n" + 387 "4:BASIC 0\n" + 388 "rule 0:1 1\n" + 389 "mode 0:0\n" + 390 "0:'a'..'a'\n" + 391 "0->1 EPSILON 0,0,0\n" + 392 "1->3 EPSILON 0,0,0\n" + 393 "3->4 NOT_SET 0,0,0\n" + 394 "4->2 EPSILON 0,0,0\n" + 395 "0:0\n"; 396 ATN atn = createATN(lg, true); 397 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 398 assertEquals(expecting, result); 399 } 400 testLexerRange()401 @Test public void testLexerRange() throws Exception { 402 LexerGrammar lg = new LexerGrammar( 403 "lexer grammar L;\n"+ 404 "INT : '0'..'9' ;\n"); 405 String expecting = 406 "max type 1\n" + 407 "0:TOKEN_START -1\n" + 408 "1:RULE_START 0\n" + 409 "2:RULE_STOP 0\n" + 410 "3:BASIC 0\n" + 411 "4:BASIC 0\n" + 412 "rule 0:1 1\n" + 413 "mode 0:0\n" + 414 "0->1 EPSILON 0,0,0\n" + 415 "1->3 EPSILON 0,0,0\n" + 416 "3->4 RANGE 48,57,0\n" + 417 "4->2 EPSILON 0,0,0\n" + 418 "0:0\n"; 419 ATN atn = createATN(lg, true); 420 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 421 assertEquals(expecting, result); 422 } 423 testLexerEOF()424 @Test public void testLexerEOF() throws Exception { 425 LexerGrammar lg = new LexerGrammar( 426 "lexer grammar L;\n"+ 427 "INT : 'a' EOF ;\n"); 428 String expecting = 429 "max type 1\n" + 430 "0:TOKEN_START -1\n" + 431 "1:RULE_START 0\n" + 432 "2:RULE_STOP 0\n" + 433 "3:BASIC 0\n" + 434 "4:BASIC 0\n" + 435 "5:BASIC 0\n" + 436 "rule 0:1 1\n" + 437 "mode 0:0\n" + 438 "0->1 EPSILON 0,0,0\n" + 439 "1->3 EPSILON 0,0,0\n" + 440 "3->4 ATOM 97,0,0\n" + 441 "4->5 ATOM 0,0,1\n" + 442 "5->2 EPSILON 0,0,0\n" + 443 "0:0\n"; 444 ATN atn = createATN(lg, true); 445 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 446 assertEquals(expecting, result); 447 } 448 testLexerEOFInSet()449 @Test public void testLexerEOFInSet() throws Exception { 450 LexerGrammar lg = new LexerGrammar( 451 "lexer grammar L;\n"+ 452 "INT : 'a' (EOF|'\\n') ;\n"); 453 String expecting = 454 "max type 1\n" + 455 "0:TOKEN_START -1\n" + 456 "1:RULE_START 0\n" + 457 "2:RULE_STOP 0\n" + 458 "3:BASIC 0\n" + 459 "4:BASIC 0\n" + 460 "5:BLOCK_START 0 6\n" + 461 "6:BLOCK_END 0\n" + 462 "rule 0:1 1\n" + 463 "mode 0:0\n" + 464 "0:EOF, '\\n'..'\\n'\n" + 465 "0->1 EPSILON 0,0,0\n" + 466 "1->3 EPSILON 0,0,0\n" + 467 "3->5 ATOM 97,0,0\n" + 468 "4->6 SET 0,0,0\n" + 469 "5->4 EPSILON 0,0,0\n" + 470 "6->2 EPSILON 0,0,0\n" + 471 "0:0\n" + 472 "1:5\n"; 473 ATN atn = createATN(lg, true); 474 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 475 assertEquals(expecting, result); 476 } 477 testLexerLoops()478 @Test public void testLexerLoops() throws Exception { 479 LexerGrammar lg = new LexerGrammar( 480 "lexer grammar L;\n"+ 481 "INT : '0'..'9'+ ;\n"); 482 String expecting = 483 "max type 1\n" + 484 "0:TOKEN_START -1\n" + 485 "1:RULE_START 0\n" + 486 "2:RULE_STOP 0\n" + 487 "3:BASIC 0\n" + 488 "4:PLUS_BLOCK_START 0 5\n" + 489 "5:BLOCK_END 0\n" + 490 "6:PLUS_LOOP_BACK 0\n" + 491 "7:LOOP_END 0 6\n" + 492 "rule 0:1 1\n" + 493 "mode 0:0\n" + 494 "0->1 EPSILON 0,0,0\n" + 495 "1->4 EPSILON 0,0,0\n" + 496 "3->5 RANGE 48,57,0\n" + 497 "4->3 EPSILON 0,0,0\n" + 498 "5->6 EPSILON 0,0,0\n" + 499 "6->4 EPSILON 0,0,0\n" + 500 "6->7 EPSILON 0,0,0\n" + 501 "7->2 EPSILON 0,0,0\n" + 502 "0:0\n" + 503 "1:6\n"; 504 ATN atn = createATN(lg, true); 505 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 506 assertEquals(expecting, result); 507 } 508 testLexerAction()509 @Test public void testLexerAction() throws Exception { 510 LexerGrammar lg = new LexerGrammar( 511 "lexer grammar L;\n"+ 512 "A : 'a' {a} ;\n" + 513 "B : 'b' ;\n" + 514 "C : 'c' {c} ;\n"); 515 String expecting = 516 "max type 3\n" + 517 "0:TOKEN_START -1\n" + 518 "1:RULE_START 0\n" + 519 "2:RULE_STOP 0\n" + 520 "3:RULE_START 1\n" + 521 "4:RULE_STOP 1\n" + 522 "5:RULE_START 2\n" + 523 "6:RULE_STOP 2\n" + 524 "7:BASIC 0\n" + 525 "8:BASIC 0\n" + 526 "9:BASIC 0\n" + 527 "10:BASIC 1\n" + 528 "11:BASIC 1\n" + 529 "12:BASIC 2\n" + 530 "13:BASIC 2\n" + 531 "14:BASIC 2\n" + 532 "rule 0:1 1\n" + 533 "rule 1:3 2\n" + 534 "rule 2:5 3\n" + 535 "mode 0:0\n" + 536 "0->1 EPSILON 0,0,0\n" + 537 "0->3 EPSILON 0,0,0\n" + 538 "0->5 EPSILON 0,0,0\n" + 539 "1->7 EPSILON 0,0,0\n" + 540 "3->10 EPSILON 0,0,0\n" + 541 "5->12 EPSILON 0,0,0\n" + 542 "7->8 ATOM 97,0,0\n" + 543 "8->9 ACTION 0,0,0\n" + 544 "9->2 EPSILON 0,0,0\n" + 545 "10->11 ATOM 98,0,0\n" + 546 "11->4 EPSILON 0,0,0\n" + 547 "12->13 ATOM 99,0,0\n" + 548 "13->14 ACTION 2,1,0\n" + 549 "14->6 EPSILON 0,0,0\n" + 550 "0:0\n"; 551 ATN atn = createATN(lg, true); 552 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 553 assertEquals(expecting, result); 554 } 555 testLexerNotSet()556 @Test public void testLexerNotSet() throws Exception { 557 LexerGrammar lg = new LexerGrammar( 558 "lexer grammar L;\n"+ 559 "ID : ~('a'|'b')\n ;"); 560 String expecting = 561 "max type 1\n" + 562 "0:TOKEN_START -1\n" + 563 "1:RULE_START 0\n" + 564 "2:RULE_STOP 0\n" + 565 "3:BASIC 0\n" + 566 "4:BASIC 0\n" + 567 "rule 0:1 1\n" + 568 "mode 0:0\n" + 569 "0:'a'..'b'\n" + 570 "0->1 EPSILON 0,0,0\n" + 571 "1->3 EPSILON 0,0,0\n" + 572 "3->4 NOT_SET 0,0,0\n" + 573 "4->2 EPSILON 0,0,0\n" + 574 "0:0\n"; 575 ATN atn = createATN(lg, true); 576 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 577 assertEquals(expecting, result); 578 } 579 testLexerSetWithRange()580 @Test public void testLexerSetWithRange() throws Exception { 581 LexerGrammar lg = new LexerGrammar( 582 "lexer grammar L;\n"+ 583 "ID : ('a'|'b'|'e'|'p'..'t')\n ;"); 584 String expecting = 585 "max type 1\n" + 586 "0:TOKEN_START -1\n" + 587 "1:RULE_START 0\n" + 588 "2:RULE_STOP 0\n" + 589 "3:BASIC 0\n" + 590 "4:BASIC 0\n" + 591 "rule 0:1 1\n" + 592 "mode 0:0\n" + 593 "0:'a'..'b', 'e'..'e', 'p'..'t'\n" + 594 "0->1 EPSILON 0,0,0\n" + 595 "1->3 EPSILON 0,0,0\n" + 596 "3->4 SET 0,0,0\n" + 597 "4->2 EPSILON 0,0,0\n" + 598 "0:0\n"; 599 ATN atn = createATN(lg, true); 600 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 601 assertEquals(expecting, result); 602 } 603 testLexerNotSetWithRange()604 @Test public void testLexerNotSetWithRange() throws Exception { 605 LexerGrammar lg = new LexerGrammar( 606 "lexer grammar L;\n"+ 607 "ID : ~('a'|'b'|'e'|'p'..'t')\n ;"); 608 String expecting = 609 "max type 1\n" + 610 "0:TOKEN_START -1\n" + 611 "1:RULE_START 0\n" + 612 "2:RULE_STOP 0\n" + 613 "3:BASIC 0\n" + 614 "4:BASIC 0\n" + 615 "rule 0:1 1\n" + 616 "mode 0:0\n" + 617 "0:'a'..'b', 'e'..'e', 'p'..'t'\n" + 618 "0->1 EPSILON 0,0,0\n" + 619 "1->3 EPSILON 0,0,0\n" + 620 "3->4 NOT_SET 0,0,0\n" + 621 "4->2 EPSILON 0,0,0\n" + 622 "0:0\n"; 623 ATN atn = createATN(lg, true); 624 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 625 assertEquals(expecting, result); 626 } 627 testLexerUnicodeUnescapedBMPNotSet()628 @Test public void testLexerUnicodeUnescapedBMPNotSet() throws Exception { 629 LexerGrammar lg = new LexerGrammar( 630 "lexer grammar L;\n"+ 631 "ID : ~('\u4E9C'|'\u4E9D')\n ;"); 632 String expecting = 633 "max type 1\n" + 634 "0:TOKEN_START -1\n" + 635 "1:RULE_START 0\n" + 636 "2:RULE_STOP 0\n" + 637 "3:BASIC 0\n" + 638 "4:BASIC 0\n" + 639 "rule 0:1 1\n" + 640 "mode 0:0\n" + 641 "0:'\\u4E9C'..'\\u4E9D'\n" + 642 "0->1 EPSILON 0,0,0\n" + 643 "1->3 EPSILON 0,0,0\n" + 644 "3->4 NOT_SET 0,0,0\n" + 645 "4->2 EPSILON 0,0,0\n" + 646 "0:0\n"; 647 ATN atn = createATN(lg, true); 648 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 649 assertEquals(expecting, result); 650 } 651 testLexerUnicodeUnescapedBMPSetWithRange()652 @Test public void testLexerUnicodeUnescapedBMPSetWithRange() throws Exception { 653 LexerGrammar lg = new LexerGrammar( 654 "lexer grammar L;\n"+ 655 "ID : ('\u4E9C'|'\u4E9D'|'\u6C5F'|'\u305F'..'\u307B')\n ;"); 656 String expecting = 657 "max type 1\n" + 658 "0:TOKEN_START -1\n" + 659 "1:RULE_START 0\n" + 660 "2:RULE_STOP 0\n" + 661 "3:BASIC 0\n" + 662 "4:BASIC 0\n" + 663 "rule 0:1 1\n" + 664 "mode 0:0\n" + 665 "0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" + 666 "0->1 EPSILON 0,0,0\n" + 667 "1->3 EPSILON 0,0,0\n" + 668 "3->4 SET 0,0,0\n" + 669 "4->2 EPSILON 0,0,0\n" + 670 "0:0\n"; 671 ATN atn = createATN(lg, true); 672 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 673 assertEquals(expecting, result); 674 } 675 testLexerUnicodeUnescapedBMPNotSetWithRange()676 @Test public void testLexerUnicodeUnescapedBMPNotSetWithRange() throws Exception { 677 LexerGrammar lg = new LexerGrammar( 678 "lexer grammar L;\n"+ 679 "ID : ~('\u4E9C'|'\u4E9D'|'\u6C5F'|'\u305F'..'\u307B')\n ;"); 680 String expecting = 681 "max type 1\n" + 682 "0:TOKEN_START -1\n" + 683 "1:RULE_START 0\n" + 684 "2:RULE_STOP 0\n" + 685 "3:BASIC 0\n" + 686 "4:BASIC 0\n" + 687 "rule 0:1 1\n" + 688 "mode 0:0\n" + 689 "0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" + 690 "0->1 EPSILON 0,0,0\n" + 691 "1->3 EPSILON 0,0,0\n" + 692 "3->4 NOT_SET 0,0,0\n" + 693 "4->2 EPSILON 0,0,0\n" + 694 "0:0\n"; 695 ATN atn = createATN(lg, true); 696 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 697 assertEquals(expecting, result); 698 } 699 testLexerUnicodeEscapedBMPNotSet()700 @Test public void testLexerUnicodeEscapedBMPNotSet() throws Exception { 701 LexerGrammar lg = new LexerGrammar( 702 "lexer grammar L;\n"+ 703 "ID : ~('\\u4E9C'|'\\u4E9D')\n ;"); 704 String expecting = 705 "max type 1\n" + 706 "0:TOKEN_START -1\n" + 707 "1:RULE_START 0\n" + 708 "2:RULE_STOP 0\n" + 709 "3:BASIC 0\n" + 710 "4:BASIC 0\n" + 711 "rule 0:1 1\n" + 712 "mode 0:0\n" + 713 "0:'\\u4E9C'..'\\u4E9D'\n" + 714 "0->1 EPSILON 0,0,0\n" + 715 "1->3 EPSILON 0,0,0\n" + 716 "3->4 NOT_SET 0,0,0\n" + 717 "4->2 EPSILON 0,0,0\n" + 718 "0:0\n"; 719 ATN atn = createATN(lg, true); 720 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 721 assertEquals(expecting, result); 722 } 723 testLexerUnicodeEscapedBMPSetWithRange()724 @Test public void testLexerUnicodeEscapedBMPSetWithRange() throws Exception { 725 LexerGrammar lg = new LexerGrammar( 726 "lexer grammar L;\n"+ 727 "ID : ('\\u4E9C'|'\\u4E9D'|'\\u6C5F'|'\\u305F'..'\\u307B')\n ;"); 728 String expecting = 729 "max type 1\n" + 730 "0:TOKEN_START -1\n" + 731 "1:RULE_START 0\n" + 732 "2:RULE_STOP 0\n" + 733 "3:BASIC 0\n" + 734 "4:BASIC 0\n" + 735 "rule 0:1 1\n" + 736 "mode 0:0\n" + 737 "0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" + 738 "0->1 EPSILON 0,0,0\n" + 739 "1->3 EPSILON 0,0,0\n" + 740 "3->4 SET 0,0,0\n" + 741 "4->2 EPSILON 0,0,0\n" + 742 "0:0\n"; 743 ATN atn = createATN(lg, true); 744 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 745 assertEquals(expecting, result); 746 } 747 testLexerUnicodeEscapedBMPNotSetWithRange()748 @Test public void testLexerUnicodeEscapedBMPNotSetWithRange() throws Exception { 749 LexerGrammar lg = new LexerGrammar( 750 "lexer grammar L;\n"+ 751 "ID : ~('\\u4E9C'|'\\u4E9D'|'\\u6C5F'|'\\u305F'..'\\u307B')\n ;"); 752 String expecting = 753 "max type 1\n" + 754 "0:TOKEN_START -1\n" + 755 "1:RULE_START 0\n" + 756 "2:RULE_STOP 0\n" + 757 "3:BASIC 0\n" + 758 "4:BASIC 0\n" + 759 "rule 0:1 1\n" + 760 "mode 0:0\n" + 761 "0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" + 762 "0->1 EPSILON 0,0,0\n" + 763 "1->3 EPSILON 0,0,0\n" + 764 "3->4 NOT_SET 0,0,0\n" + 765 "4->2 EPSILON 0,0,0\n" + 766 "0:0\n"; 767 ATN atn = createATN(lg, true); 768 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 769 assertEquals(expecting, result); 770 } 771 testLexerUnicodeEscapedSMPNotSet()772 @Test public void testLexerUnicodeEscapedSMPNotSet() throws Exception { 773 LexerGrammar lg = new LexerGrammar( 774 "lexer grammar L;\n"+ 775 "ID : ~('\\u{1F4A9}'|'\\u{1F4AA}')\n ;"); 776 String expecting = 777 "max type 1\n" + 778 "0:TOKEN_START -1\n" + 779 "1:RULE_START 0\n" + 780 "2:RULE_STOP 0\n" + 781 "3:BASIC 0\n" + 782 "4:BASIC 0\n" + 783 "rule 0:1 1\n" + 784 "mode 0:0\n" + 785 "0:128169..128170\n" + 786 "0->1 EPSILON 0,0,0\n" + 787 "1->3 EPSILON 0,0,0\n" + 788 "3->4 NOT_SET 0,0,0\n" + 789 "4->2 EPSILON 0,0,0\n" + 790 "0:0\n"; 791 ATN atn = createATN(lg, true); 792 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 793 assertEquals(expecting, result); 794 } 795 testLexerUnicodeEscapedSMPSetWithRange()796 @Test public void testLexerUnicodeEscapedSMPSetWithRange() throws Exception { 797 LexerGrammar lg = new LexerGrammar( 798 "lexer grammar L;\n"+ 799 "ID : ('\\u{1F4A9}'|'\\u{1F4AA}'|'\\u{1F441}'|'\\u{1D40F}'..'\\u{1D413}')\n ;"); 800 String expecting = 801 "max type 1\n" + 802 "0:TOKEN_START -1\n" + 803 "1:RULE_START 0\n" + 804 "2:RULE_STOP 0\n" + 805 "3:BASIC 0\n" + 806 "4:BASIC 0\n" + 807 "rule 0:1 1\n" + 808 "mode 0:0\n" + 809 "0:119823..119827, 128065..128065, 128169..128170\n" + 810 "0->1 EPSILON 0,0,0\n" + 811 "1->3 EPSILON 0,0,0\n" + 812 "3->4 SET 0,0,0\n" + 813 "4->2 EPSILON 0,0,0\n" + 814 "0:0\n"; 815 ATN atn = createATN(lg, true); 816 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 817 assertEquals(expecting, result); 818 } 819 testLexerUnicodeEscapedSMPNotSetWithRange()820 @Test public void testLexerUnicodeEscapedSMPNotSetWithRange() throws Exception { 821 LexerGrammar lg = new LexerGrammar( 822 "lexer grammar L;\n"+ 823 "ID : ~('\\u{1F4A9}'|'\\u{1F4AA}'|'\\u{1F441}'|'\\u{1D40F}'..'\\u{1D413}')\n ;"); 824 String expecting = 825 "max type 1\n" + 826 "0:TOKEN_START -1\n" + 827 "1:RULE_START 0\n" + 828 "2:RULE_STOP 0\n" + 829 "3:BASIC 0\n" + 830 "4:BASIC 0\n" + 831 "rule 0:1 1\n" + 832 "mode 0:0\n" + 833 "0:119823..119827, 128065..128065, 128169..128170\n" + 834 "0->1 EPSILON 0,0,0\n" + 835 "1->3 EPSILON 0,0,0\n" + 836 "3->4 NOT_SET 0,0,0\n" + 837 "4->2 EPSILON 0,0,0\n" + 838 "0:0\n"; 839 ATN atn = createATN(lg, true); 840 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 841 assertEquals(expecting, result); 842 } 843 testLexerWildcardWithMode()844 @Test public void testLexerWildcardWithMode() throws Exception { 845 LexerGrammar lg = new LexerGrammar( 846 "lexer grammar L;\n"+ 847 "ID : 'a'..'z'+ ;\n"+ 848 "mode CMT;" + 849 "COMMENT : '*/' {skip(); popMode();} ;\n" + 850 "JUNK : . {more();} ;\n"); 851 String expecting = 852 "max type 3\n" + 853 "0:TOKEN_START -1\n" + 854 "1:TOKEN_START -1\n" + 855 "2:RULE_START 0\n" + 856 "3:RULE_STOP 0\n" + 857 "4:RULE_START 1\n" + 858 "5:RULE_STOP 1\n" + 859 "6:RULE_START 2\n" + 860 "7:RULE_STOP 2\n" + 861 "8:BASIC 0\n" + 862 "9:PLUS_BLOCK_START 0 10\n" + 863 "10:BLOCK_END 0\n" + 864 "11:PLUS_LOOP_BACK 0\n" + 865 "12:LOOP_END 0 11\n" + 866 "13:BASIC 1\n" + 867 "14:BASIC 1\n" + 868 "15:BASIC 1\n" + 869 "16:BASIC 1\n" + 870 "17:BASIC 1\n" + 871 "18:BASIC 2\n" + 872 "19:BASIC 2\n" + 873 "20:BASIC 2\n" + 874 "rule 0:2 1\n" + 875 "rule 1:4 2\n" + 876 "rule 2:6 3\n" + 877 "mode 0:0\n" + 878 "mode 1:1\n" + 879 "0->2 EPSILON 0,0,0\n" + 880 "1->4 EPSILON 0,0,0\n" + 881 "1->6 EPSILON 0,0,0\n" + 882 "2->9 EPSILON 0,0,0\n" + 883 "4->13 EPSILON 0,0,0\n" + 884 "6->18 EPSILON 0,0,0\n" + 885 "8->10 RANGE 97,122,0\n" + 886 "9->8 EPSILON 0,0,0\n" + 887 "10->11 EPSILON 0,0,0\n" + 888 "11->9 EPSILON 0,0,0\n" + 889 "11->12 EPSILON 0,0,0\n" + 890 "12->3 EPSILON 0,0,0\n" + 891 "13->14 ATOM 42,0,0\n" + 892 "14->15 ATOM 47,0,0\n" + 893 "15->16 EPSILON 0,0,0\n" + 894 "16->17 ACTION 1,0,0\n" + 895 "17->5 EPSILON 0,0,0\n" + 896 "18->19 WILDCARD 0,0,0\n" + 897 "19->20 ACTION 2,1,0\n" + 898 "20->7 EPSILON 0,0,0\n" + 899 "0:0\n" + 900 "1:1\n" + 901 "2:11\n"; 902 ATN atn = createATN(lg, true); 903 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 904 assertEquals(expecting, result); 905 } 906 testLexerNotSetWithRange2()907 @Test public void testLexerNotSetWithRange2() throws Exception { 908 LexerGrammar lg = new LexerGrammar( 909 "lexer grammar L;\n"+ 910 "ID : ~('a'|'b') ~('e'|'p'..'t')\n ;"); 911 String expecting = 912 "max type 1\n" + 913 "0:TOKEN_START -1\n" + 914 "1:RULE_START 0\n" + 915 "2:RULE_STOP 0\n" + 916 "3:BASIC 0\n" + 917 "4:BASIC 0\n" + 918 "5:BASIC 0\n" + 919 "rule 0:1 1\n" + 920 "mode 0:0\n" + 921 "0:'a'..'b'\n" + 922 "1:'e'..'e', 'p'..'t'\n" + 923 "0->1 EPSILON 0,0,0\n" + 924 "1->3 EPSILON 0,0,0\n" + 925 "3->4 NOT_SET 0,0,0\n" + 926 "4->5 NOT_SET 1,0,0\n" + 927 "5->2 EPSILON 0,0,0\n" + 928 "0:0\n"; 929 ATN atn = createATN(lg, true); 930 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 931 assertEquals(expecting, result); 932 } 933 testModeInLexer()934 @Test public void testModeInLexer() throws Exception { 935 LexerGrammar lg = new LexerGrammar( 936 "lexer grammar L;\n"+ 937 "A : 'a'\n ;\n" + 938 "B : 'b';\n" + 939 "mode M;\n" + 940 "C : 'c';\n"+ 941 "D : 'd';\n"); 942 String expecting = 943 "max type 4\n" + 944 "0:TOKEN_START -1\n" + 945 "1:TOKEN_START -1\n" + 946 "2:RULE_START 0\n" + 947 "3:RULE_STOP 0\n" + 948 "4:RULE_START 1\n" + 949 "5:RULE_STOP 1\n" + 950 "6:RULE_START 2\n" + 951 "7:RULE_STOP 2\n" + 952 "8:RULE_START 3\n" + 953 "9:RULE_STOP 3\n" + 954 "10:BASIC 0\n" + 955 "11:BASIC 0\n" + 956 "12:BASIC 1\n" + 957 "13:BASIC 1\n" + 958 "14:BASIC 2\n" + 959 "15:BASIC 2\n" + 960 "16:BASIC 3\n" + 961 "17:BASIC 3\n" + 962 "rule 0:2 1\n" + 963 "rule 1:4 2\n" + 964 "rule 2:6 3\n" + 965 "rule 3:8 4\n" + 966 "mode 0:0\n" + 967 "mode 1:1\n" + 968 "0->2 EPSILON 0,0,0\n" + 969 "0->4 EPSILON 0,0,0\n" + 970 "1->6 EPSILON 0,0,0\n" + 971 "1->8 EPSILON 0,0,0\n" + 972 "2->10 EPSILON 0,0,0\n" + 973 "4->12 EPSILON 0,0,0\n" + 974 "6->14 EPSILON 0,0,0\n" + 975 "8->16 EPSILON 0,0,0\n" + 976 "10->11 ATOM 97,0,0\n" + 977 "11->3 EPSILON 0,0,0\n" + 978 "12->13 ATOM 98,0,0\n" + 979 "13->5 EPSILON 0,0,0\n" + 980 "14->15 ATOM 99,0,0\n" + 981 "15->7 EPSILON 0,0,0\n" + 982 "16->17 ATOM 100,0,0\n" + 983 "17->9 EPSILON 0,0,0\n" + 984 "0:0\n" + 985 "1:1\n"; 986 ATN atn = createATN(lg, true); 987 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 988 assertEquals(expecting, result); 989 } 990 test2ModesInLexer()991 @Test public void test2ModesInLexer() throws Exception { 992 LexerGrammar lg = new LexerGrammar( 993 "lexer grammar L;\n"+ 994 "A : 'a'\n ;\n" + 995 "mode M;\n" + 996 "B : 'b';\n" + 997 "mode M2;\n" + 998 "C : 'c';\n"); 999 String expecting = 1000 "max type 3\n" + 1001 "0:TOKEN_START -1\n" + 1002 "1:TOKEN_START -1\n" + 1003 "2:TOKEN_START -1\n" + 1004 "3:RULE_START 0\n" + 1005 "4:RULE_STOP 0\n" + 1006 "5:RULE_START 1\n" + 1007 "6:RULE_STOP 1\n" + 1008 "7:RULE_START 2\n" + 1009 "8:RULE_STOP 2\n" + 1010 "9:BASIC 0\n" + 1011 "10:BASIC 0\n" + 1012 "11:BASIC 1\n" + 1013 "12:BASIC 1\n" + 1014 "13:BASIC 2\n" + 1015 "14:BASIC 2\n" + 1016 "rule 0:3 1\n" + 1017 "rule 1:5 2\n" + 1018 "rule 2:7 3\n" + 1019 "mode 0:0\n" + 1020 "mode 1:1\n" + 1021 "mode 2:2\n" + 1022 "0->3 EPSILON 0,0,0\n" + 1023 "1->5 EPSILON 0,0,0\n" + 1024 "2->7 EPSILON 0,0,0\n" + 1025 "3->9 EPSILON 0,0,0\n" + 1026 "5->11 EPSILON 0,0,0\n" + 1027 "7->13 EPSILON 0,0,0\n" + 1028 "9->10 ATOM 97,0,0\n" + 1029 "10->4 EPSILON 0,0,0\n" + 1030 "11->12 ATOM 98,0,0\n" + 1031 "12->6 EPSILON 0,0,0\n" + 1032 "13->14 ATOM 99,0,0\n" + 1033 "14->8 EPSILON 0,0,0\n" + 1034 "0:0\n" + 1035 "1:1\n" + 1036 "2:2\n"; 1037 ATN atn = createATN(lg, true); 1038 String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); 1039 assertEquals(expecting, result); 1040 } 1041 1042 } 1043