1 /*
2  * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 package org.antlr.v4.test.tool;
8 
9 import org.antlr.v4.runtime.ANTLRInputStream;
10 import org.antlr.v4.runtime.CharStream;
11 import org.antlr.v4.runtime.LexerInterpreter;
12 import org.antlr.v4.runtime.Token;
13 import org.antlr.v4.runtime.TokenSource;
14 import org.antlr.v4.runtime.TokenStream;
15 import org.antlr.v4.runtime.UnbufferedTokenStream;
16 import org.antlr.v4.tool.LexerGrammar;
17 import org.junit.Before;
18 import org.junit.Test;
19 
20 import java.io.StringReader;
21 import java.util.Arrays;
22 import java.util.Collections;
23 import java.util.List;
24 
25 import static org.junit.Assert.assertEquals;
26 
27 @SuppressWarnings("unused")
28 public class TestUnbufferedTokenStream extends BaseJavaToolTest {
29 	@Before
30 	@Override
testSetUp()31 	public void testSetUp() throws Exception {
32 		super.testSetUp();
33 	}
34 
testLookahead()35 	@Test public void testLookahead() throws Exception {
36         LexerGrammar g = new LexerGrammar(
37             "lexer grammar t;\n"+
38             "ID : 'a'..'z'+;\n" +
39             "INT : '0'..'9'+;\n" +
40             "SEMI : ';';\n" +
41             "ASSIGN : '=';\n" +
42             "PLUS : '+';\n" +
43             "MULT : '*';\n" +
44             "WS : ' '+;\n");
45         // Tokens: 012345678901234567
46         // Input:  x = 302;
47         CharStream input = new ANTLRInputStream(
48 			new StringReader("x = 302;")
49 		);
50         LexerInterpreter lexEngine = g.createLexerInterpreter(input);
51         TokenStream tokens = new UnbufferedTokenStream<Token>(lexEngine);
52 
53 		assertEquals("x", tokens.LT(1).getText());
54 		assertEquals(" ", tokens.LT(2).getText());
55 		assertEquals("=", tokens.LT(3).getText());
56 		assertEquals(" ", tokens.LT(4).getText());
57 		assertEquals("302", tokens.LT(5).getText());
58 		assertEquals(";", tokens.LT(6).getText());
59     }
60 
testNoBuffering()61 	@Test public void testNoBuffering() throws Exception {
62         LexerGrammar g = new LexerGrammar(
63             "lexer grammar t;\n"+
64             "ID : 'a'..'z'+;\n" +
65             "INT : '0'..'9'+;\n" +
66             "SEMI : ';';\n" +
67             "ASSIGN : '=';\n" +
68             "PLUS : '+';\n" +
69             "MULT : '*';\n" +
70             "WS : ' '+;\n");
71         // Tokens: 012345678901234567
72         // Input:  x = 302;
73         CharStream input = new ANTLRInputStream(
74 			new StringReader("x = 302;")
75 		);
76         LexerInterpreter lexEngine = g.createLexerInterpreter(input);
77 		TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
78 
79 		assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString());
80 		assertEquals("x", tokens.LT(1).getText());
81 		tokens.consume(); // move to WS
82 		assertEquals(" ", tokens.LT(1).getText());
83 		assertEquals("[[@1,1:1=' ',<7>,1:1]]", tokens.getRemainingBuffer().toString());
84 		tokens.consume();
85 		assertEquals("=", tokens.LT(1).getText());
86 		assertEquals("[[@2,2:2='=',<4>,1:2]]", tokens.getRemainingBuffer().toString());
87 		tokens.consume();
88 		assertEquals(" ", tokens.LT(1).getText());
89 		assertEquals("[[@3,3:3=' ',<7>,1:3]]", tokens.getRemainingBuffer().toString());
90 		tokens.consume();
91 		assertEquals("302", tokens.LT(1).getText());
92 		assertEquals("[[@4,4:6='302',<2>,1:4]]", tokens.getRemainingBuffer().toString());
93 		tokens.consume();
94 		assertEquals(";", tokens.LT(1).getText());
95 		assertEquals("[[@5,7:7=';',<3>,1:7]]", tokens.getRemainingBuffer().toString());
96     }
97 
testMarkStart()98 	@Test public void testMarkStart() throws Exception {
99         LexerGrammar g = new LexerGrammar(
100             "lexer grammar t;\n"+
101             "ID : 'a'..'z'+;\n" +
102             "INT : '0'..'9'+;\n" +
103             "SEMI : ';';\n" +
104             "ASSIGN : '=';\n" +
105             "PLUS : '+';\n" +
106             "MULT : '*';\n" +
107             "WS : ' '+;\n");
108         // Tokens: 012345678901234567
109         // Input:  x = 302;
110         CharStream input = new ANTLRInputStream(
111 			new StringReader("x = 302;")
112 		);
113         LexerInterpreter lexEngine = g.createLexerInterpreter(input);
114 		TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
115 
116 		int m = tokens.mark();
117 		assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString());
118 		assertEquals("x", tokens.LT(1).getText());
119 		tokens.consume(); // consume x
120 		assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]]", tokens.getBuffer().toString());
121 		tokens.consume(); // ' '
122 		tokens.consume(); // =
123 		tokens.consume(); // ' '
124 		tokens.consume(); // 302
125 		tokens.consume(); // ;
126 		assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]," +
127 					 " [@2,2:2='=',<4>,1:2], [@3,3:3=' ',<7>,1:3]," +
128 					 " [@4,4:6='302',<2>,1:4], [@5,7:7=';',<3>,1:7]," +
129 					 " [@6,8:7='<EOF>',<-1>,1:8]]",
130 					 tokens.getBuffer().toString());
131     }
132 
testMarkThenRelease()133 	@Test public void testMarkThenRelease() throws Exception {
134         LexerGrammar g = new LexerGrammar(
135             "lexer grammar t;\n"+
136             "ID : 'a'..'z'+;\n" +
137             "INT : '0'..'9'+;\n" +
138             "SEMI : ';';\n" +
139             "ASSIGN : '=';\n" +
140             "PLUS : '+';\n" +
141             "MULT : '*';\n" +
142             "WS : ' '+;\n");
143         // Tokens: 012345678901234567
144         // Input:  x = 302;
145         CharStream input = new ANTLRInputStream(
146 			new StringReader("x = 302 + 1;")
147 		);
148         LexerInterpreter lexEngine = g.createLexerInterpreter(input);
149 		TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
150 
151 		int m = tokens.mark();
152 		assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString());
153 		assertEquals("x", tokens.LT(1).getText());
154 		tokens.consume(); // consume x
155 		assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]]", tokens.getBuffer().toString());
156 		tokens.consume(); // ' '
157 		tokens.consume(); // =
158 		tokens.consume(); // ' '
159 		assertEquals("302", tokens.LT(1).getText());
160 		tokens.release(m); // "x = 302" is in buffer. will kill buffer
161 		tokens.consume(); // 302
162 		tokens.consume(); // ' '
163 		m = tokens.mark(); // mark at the +
164 		assertEquals("+", tokens.LT(1).getText());
165 		tokens.consume(); // '+'
166 		tokens.consume(); // ' '
167 		tokens.consume(); // 1
168 		tokens.consume(); // ;
169 		assertEquals("<EOF>", tokens.LT(1).getText());
170 		// we marked at the +, so that should be the start of the buffer
171 		assertEquals("[[@6,8:8='+',<5>,1:8], [@7,9:9=' ',<7>,1:9]," +
172 					 " [@8,10:10='1',<2>,1:10], [@9,11:11=';',<3>,1:11]," +
173 					 " [@10,12:11='<EOF>',<-1>,1:12]]",
174 					 tokens.getBuffer().toString());
175 		tokens.release(m);
176     }
177 
178 	protected static class TestingUnbufferedTokenStream<T extends Token> extends UnbufferedTokenStream<T> {
179 
TestingUnbufferedTokenStream(TokenSource tokenSource)180 		public TestingUnbufferedTokenStream(TokenSource tokenSource) {
181 			super(tokenSource);
182 		}
183 
184 		/** For testing.  What's in moving window into token stream from
185 		 *  current index, LT(1) or tokens[p], to end of buffer?
186 		 */
getRemainingBuffer()187 		protected List<? extends Token> getRemainingBuffer() {
188 			if ( n==0 ) {
189 				return Collections.emptyList();
190 			}
191 
192 			return Arrays.asList(tokens).subList(p, n);
193 		}
194 
195 		/** For testing.  What's in moving window buffer into data stream.
196 		 *  From 0..p-1 have been consume.
197 		 */
getBuffer()198 		protected List<? extends Token> getBuffer() {
199 			if ( n==0 ) {
200 				return Collections.emptyList();
201 			}
202 
203 			return Arrays.asList(tokens).subList(0, n);
204 		}
205 
206 	}
207 }
208