1 /* 2 * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. 3 * Use of this file is governed by the BSD 3-clause license that 4 * can be found in the LICENSE.txt file in the project root. 5 */ 6 7 package org.antlr.v4.test.tool; 8 9 import org.antlr.v4.runtime.ANTLRInputStream; 10 import org.antlr.v4.runtime.CharStream; 11 import org.antlr.v4.runtime.LexerInterpreter; 12 import org.antlr.v4.runtime.Token; 13 import org.antlr.v4.runtime.TokenSource; 14 import org.antlr.v4.runtime.TokenStream; 15 import org.antlr.v4.runtime.UnbufferedTokenStream; 16 import org.antlr.v4.tool.LexerGrammar; 17 import org.junit.Before; 18 import org.junit.Test; 19 20 import java.io.StringReader; 21 import java.util.Arrays; 22 import java.util.Collections; 23 import java.util.List; 24 25 import static org.junit.Assert.assertEquals; 26 27 @SuppressWarnings("unused") 28 public class TestUnbufferedTokenStream extends BaseJavaToolTest { 29 @Before 30 @Override testSetUp()31 public void testSetUp() throws Exception { 32 super.testSetUp(); 33 } 34 testLookahead()35 @Test public void testLookahead() throws Exception { 36 LexerGrammar g = new LexerGrammar( 37 "lexer grammar t;\n"+ 38 "ID : 'a'..'z'+;\n" + 39 "INT : '0'..'9'+;\n" + 40 "SEMI : ';';\n" + 41 "ASSIGN : '=';\n" + 42 "PLUS : '+';\n" + 43 "MULT : '*';\n" + 44 "WS : ' '+;\n"); 45 // Tokens: 012345678901234567 46 // Input: x = 302; 47 CharStream input = new ANTLRInputStream( 48 new StringReader("x = 302;") 49 ); 50 LexerInterpreter lexEngine = g.createLexerInterpreter(input); 51 TokenStream tokens = new UnbufferedTokenStream<Token>(lexEngine); 52 53 assertEquals("x", tokens.LT(1).getText()); 54 assertEquals(" ", tokens.LT(2).getText()); 55 assertEquals("=", tokens.LT(3).getText()); 56 assertEquals(" ", tokens.LT(4).getText()); 57 assertEquals("302", tokens.LT(5).getText()); 58 assertEquals(";", tokens.LT(6).getText()); 59 } 60 testNoBuffering()61 @Test public void testNoBuffering() throws Exception { 62 LexerGrammar g = new LexerGrammar( 63 "lexer grammar t;\n"+ 64 "ID : 'a'..'z'+;\n" + 65 "INT : '0'..'9'+;\n" + 66 "SEMI : ';';\n" + 67 "ASSIGN : '=';\n" + 68 "PLUS : '+';\n" + 69 "MULT : '*';\n" + 70 "WS : ' '+;\n"); 71 // Tokens: 012345678901234567 72 // Input: x = 302; 73 CharStream input = new ANTLRInputStream( 74 new StringReader("x = 302;") 75 ); 76 LexerInterpreter lexEngine = g.createLexerInterpreter(input); 77 TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine); 78 79 assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString()); 80 assertEquals("x", tokens.LT(1).getText()); 81 tokens.consume(); // move to WS 82 assertEquals(" ", tokens.LT(1).getText()); 83 assertEquals("[[@1,1:1=' ',<7>,1:1]]", tokens.getRemainingBuffer().toString()); 84 tokens.consume(); 85 assertEquals("=", tokens.LT(1).getText()); 86 assertEquals("[[@2,2:2='=',<4>,1:2]]", tokens.getRemainingBuffer().toString()); 87 tokens.consume(); 88 assertEquals(" ", tokens.LT(1).getText()); 89 assertEquals("[[@3,3:3=' ',<7>,1:3]]", tokens.getRemainingBuffer().toString()); 90 tokens.consume(); 91 assertEquals("302", tokens.LT(1).getText()); 92 assertEquals("[[@4,4:6='302',<2>,1:4]]", tokens.getRemainingBuffer().toString()); 93 tokens.consume(); 94 assertEquals(";", tokens.LT(1).getText()); 95 assertEquals("[[@5,7:7=';',<3>,1:7]]", tokens.getRemainingBuffer().toString()); 96 } 97 testMarkStart()98 @Test public void testMarkStart() throws Exception { 99 LexerGrammar g = new LexerGrammar( 100 "lexer grammar t;\n"+ 101 "ID : 'a'..'z'+;\n" + 102 "INT : '0'..'9'+;\n" + 103 "SEMI : ';';\n" + 104 "ASSIGN : '=';\n" + 105 "PLUS : '+';\n" + 106 "MULT : '*';\n" + 107 "WS : ' '+;\n"); 108 // Tokens: 012345678901234567 109 // Input: x = 302; 110 CharStream input = new ANTLRInputStream( 111 new StringReader("x = 302;") 112 ); 113 LexerInterpreter lexEngine = g.createLexerInterpreter(input); 114 TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine); 115 116 int m = tokens.mark(); 117 assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString()); 118 assertEquals("x", tokens.LT(1).getText()); 119 tokens.consume(); // consume x 120 assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]]", tokens.getBuffer().toString()); 121 tokens.consume(); // ' ' 122 tokens.consume(); // = 123 tokens.consume(); // ' ' 124 tokens.consume(); // 302 125 tokens.consume(); // ; 126 assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]," + 127 " [@2,2:2='=',<4>,1:2], [@3,3:3=' ',<7>,1:3]," + 128 " [@4,4:6='302',<2>,1:4], [@5,7:7=';',<3>,1:7]," + 129 " [@6,8:7='<EOF>',<-1>,1:8]]", 130 tokens.getBuffer().toString()); 131 } 132 testMarkThenRelease()133 @Test public void testMarkThenRelease() throws Exception { 134 LexerGrammar g = new LexerGrammar( 135 "lexer grammar t;\n"+ 136 "ID : 'a'..'z'+;\n" + 137 "INT : '0'..'9'+;\n" + 138 "SEMI : ';';\n" + 139 "ASSIGN : '=';\n" + 140 "PLUS : '+';\n" + 141 "MULT : '*';\n" + 142 "WS : ' '+;\n"); 143 // Tokens: 012345678901234567 144 // Input: x = 302; 145 CharStream input = new ANTLRInputStream( 146 new StringReader("x = 302 + 1;") 147 ); 148 LexerInterpreter lexEngine = g.createLexerInterpreter(input); 149 TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine); 150 151 int m = tokens.mark(); 152 assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString()); 153 assertEquals("x", tokens.LT(1).getText()); 154 tokens.consume(); // consume x 155 assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]]", tokens.getBuffer().toString()); 156 tokens.consume(); // ' ' 157 tokens.consume(); // = 158 tokens.consume(); // ' ' 159 assertEquals("302", tokens.LT(1).getText()); 160 tokens.release(m); // "x = 302" is in buffer. will kill buffer 161 tokens.consume(); // 302 162 tokens.consume(); // ' ' 163 m = tokens.mark(); // mark at the + 164 assertEquals("+", tokens.LT(1).getText()); 165 tokens.consume(); // '+' 166 tokens.consume(); // ' ' 167 tokens.consume(); // 1 168 tokens.consume(); // ; 169 assertEquals("<EOF>", tokens.LT(1).getText()); 170 // we marked at the +, so that should be the start of the buffer 171 assertEquals("[[@6,8:8='+',<5>,1:8], [@7,9:9=' ',<7>,1:9]," + 172 " [@8,10:10='1',<2>,1:10], [@9,11:11=';',<3>,1:11]," + 173 " [@10,12:11='<EOF>',<-1>,1:12]]", 174 tokens.getBuffer().toString()); 175 tokens.release(m); 176 } 177 178 protected static class TestingUnbufferedTokenStream<T extends Token> extends UnbufferedTokenStream<T> { 179 TestingUnbufferedTokenStream(TokenSource tokenSource)180 public TestingUnbufferedTokenStream(TokenSource tokenSource) { 181 super(tokenSource); 182 } 183 184 /** For testing. What's in moving window into token stream from 185 * current index, LT(1) or tokens[p], to end of buffer? 186 */ getRemainingBuffer()187 protected List<? extends Token> getRemainingBuffer() { 188 if ( n==0 ) { 189 return Collections.emptyList(); 190 } 191 192 return Arrays.asList(tokens).subList(p, n); 193 } 194 195 /** For testing. What's in moving window buffer into data stream. 196 * From 0..p-1 have been consume. 197 */ getBuffer()198 protected List<? extends Token> getBuffer() { 199 if ( n==0 ) { 200 return Collections.emptyList(); 201 } 202 203 return Arrays.asList(tokens).subList(0, n); 204 } 205 206 } 207 } 208