1 /*
2  * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
3  * Use of this file is governed by the BSD 3-clause license that
4  * can be found in the LICENSE.txt file in the project root.
5  */
6 
7 package org.antlr.v4.test.tool;
8 
9 import org.antlr.v4.runtime.CharStream;
10 import org.antlr.v4.runtime.CommonToken;
11 import org.antlr.v4.runtime.CommonTokenStream;
12 import org.antlr.v4.runtime.IntStream;
13 import org.antlr.v4.runtime.Lexer;
14 import org.antlr.v4.runtime.Token;
15 import org.antlr.v4.runtime.TokenFactory;
16 import org.antlr.v4.runtime.TokenSource;
17 import org.antlr.v4.runtime.TokenStream;
18 import org.antlr.v4.runtime.WritableToken;
19 import org.junit.Before;
20 import org.junit.Test;
21 
22 import static org.junit.Assert.assertEquals;
23 
24 public class TestCommonTokenStream extends TestBufferedTokenStream {
25 	@Before
26 	@Override
testSetUp()27 	public void testSetUp() throws Exception {
28 		super.testSetUp();
29 	}
30 
31 	@Override
createTokenStream(TokenSource src)32 	protected TokenStream createTokenStream(TokenSource src) {
33 		return new CommonTokenStream(src);
34 	}
35 
testOffChannel()36 	@Test public void testOffChannel() throws Exception {
37         TokenSource lexer = // simulate input " x =34  ;\n"
38             new TokenSource() {
39                 int i = 0;
40                 @SuppressWarnings("serial")
41 				WritableToken[] tokens = {
42                     new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
43                     new CommonToken(1,"x"),
44                     new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
45                     new CommonToken(1,"="),
46                     new CommonToken(1,"34"),
47                     new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
48                     new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},
49                     new CommonToken(1,";"),
50                     new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}},
51                     new CommonToken(Token.EOF,"")
52                 };
53                 @Override
54                 public Token nextToken() {
55                     return tokens[i++];
56                 }
57                 @Override
58                 public String getSourceName() { return "test"; }
59 				@Override
60 				public int getCharPositionInLine() {
61 					return 0;
62 				}
63 				@Override
64 				public int getLine() {
65 					return 0;
66 				}
67 				@Override
68 				public CharStream getInputStream() {
69 					return null;
70 				}
71 
72 				@Override
73 				public void setTokenFactory(TokenFactory<?> factory) {
74 				}
75 
76 				@Override
77 				public TokenFactory<?> getTokenFactory() {
78 					return null;
79 				}
80 			};
81 
82         CommonTokenStream tokens = new CommonTokenStream(lexer);
83 
84         assertEquals("x", tokens.LT(1).getText()); // must skip first off channel token
85         tokens.consume();
86         assertEquals("=", tokens.LT(1).getText());
87         assertEquals("x", tokens.LT(-1).getText());
88 
89         tokens.consume();
90         assertEquals("34", tokens.LT(1).getText());
91         assertEquals("=", tokens.LT(-1).getText());
92 
93         tokens.consume();
94         assertEquals(";", tokens.LT(1).getText());
95         assertEquals("34", tokens.LT(-1).getText());
96 
97         tokens.consume();
98         assertEquals(Token.EOF, tokens.LA(1));
99         assertEquals(";", tokens.LT(-1).getText());
100 
101         assertEquals("34", tokens.LT(-2).getText());
102         assertEquals("=", tokens.LT(-3).getText());
103         assertEquals("x", tokens.LT(-4).getText());
104     }
105 
testFetchOffChannel()106 	@Test public void testFetchOffChannel() throws Exception {
107 		TokenSource lexer = // simulate input " x =34  ; \n"
108 		                    // token indexes   01234 56789
109 			new TokenSource() {
110 				int i = 0;
111 				@SuppressWarnings("serial")
112 				WritableToken[] tokens = {
113 				new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 0
114 				new CommonToken(1,"x"),								// 1
115 				new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},	// 2
116 				new CommonToken(1,"="),								// 3
117 				new CommonToken(1,"34"),							// 4
118 				new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},	// 5
119 				new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 6
120 				new CommonToken(1,";"),								// 7
121 				new CommonToken(1," ")  {{channel = Lexer.HIDDEN;}},// 8
122 				new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}},// 9
123 				new CommonToken(Token.EOF,"")						// 10
124 				};
125 				@Override
126 				public Token nextToken() {
127 					return tokens[i++];
128 				}
129 				@Override
130 				public String getSourceName() { return "test"; }
131 				@Override
132 				public int getCharPositionInLine() {
133 					return 0;
134 				}
135 				@Override
136 				public int getLine() {
137 					return 0;
138 				}
139 				@Override
140 				public CharStream getInputStream() {
141 					return null;
142 				}
143 
144 				@Override
145 				public void setTokenFactory(TokenFactory<?> factory) {
146 				}
147 
148 				@Override
149 				public TokenFactory<?> getTokenFactory() {
150 					return null;
151 				}
152 			};
153 
154 		CommonTokenStream tokens = new CommonTokenStream(lexer);
155 		tokens.fill();
156 		assertEquals(null, tokens.getHiddenTokensToLeft(0));
157 		assertEquals(null, tokens.getHiddenTokensToRight(0));
158 
159 		assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]",
160 					 tokens.getHiddenTokensToLeft(1).toString());
161 		assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]",
162 					 tokens.getHiddenTokensToRight(1).toString());
163 
164 		assertEquals(null, tokens.getHiddenTokensToLeft(2));
165 		assertEquals(null, tokens.getHiddenTokensToRight(2));
166 
167 		assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]",
168 					 tokens.getHiddenTokensToLeft(3).toString());
169 		assertEquals(null, tokens.getHiddenTokensToRight(3));
170 
171 		assertEquals(null, tokens.getHiddenTokensToLeft(4));
172 		assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]",
173 					 tokens.getHiddenTokensToRight(4).toString());
174 
175 		assertEquals(null, tokens.getHiddenTokensToLeft(5));
176 		assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]",
177 					 tokens.getHiddenTokensToRight(5).toString());
178 
179 		assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]",
180 					 tokens.getHiddenTokensToLeft(6).toString());
181 		assertEquals(null, tokens.getHiddenTokensToRight(6));
182 
183 		assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]",
184 					 tokens.getHiddenTokensToLeft(7).toString());
185 		assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]",
186 					 tokens.getHiddenTokensToRight(7).toString());
187 
188 		assertEquals(null, tokens.getHiddenTokensToLeft(8));
189 		assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]",
190 					 tokens.getHiddenTokensToRight(8).toString());
191 
192 		assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]",
193 					 tokens.getHiddenTokensToLeft(9).toString());
194 		assertEquals(null, tokens.getHiddenTokensToRight(9));
195 	}
196 
197 	@Test
testSingleEOF()198 	public void testSingleEOF() throws Exception {
199 		TokenSource lexer = new TokenSource() {
200 
201 			@Override
202 			public Token nextToken() {
203 				return new CommonToken(Token.EOF);
204 			}
205 
206 			@Override
207 			public int getLine() {
208 				return 0;
209 			}
210 
211 			@Override
212 			public int getCharPositionInLine() {
213 				return 0;
214 			}
215 
216 			@Override
217 			public CharStream getInputStream() {
218 				return null;
219 			}
220 
221 			@Override
222 			public String getSourceName() {
223 				return IntStream.UNKNOWN_SOURCE_NAME;
224 			}
225 
226 			@Override
227 			public TokenFactory<?> getTokenFactory() {
228 				throw new UnsupportedOperationException("Not supported yet.");
229 			}
230 
231 			@Override
232 			public void setTokenFactory(TokenFactory<?> factory) {
233 				throw new UnsupportedOperationException("Not supported yet.");
234 			}
235 		};
236 
237 		CommonTokenStream tokens = new CommonTokenStream(lexer);
238 		tokens.fill();
239 
240 		assertEquals(Token.EOF, tokens.LA(1));
241 		assertEquals(0, tokens.index());
242 		assertEquals(1, tokens.size());
243 	}
244 
245 	@Test(expected = IllegalStateException.class)
testCannotConsumeEOF()246 	public void testCannotConsumeEOF() throws Exception {
247 		TokenSource lexer = new TokenSource() {
248 
249 			@Override
250 			public Token nextToken() {
251 				return new CommonToken(Token.EOF);
252 			}
253 
254 			@Override
255 			public int getLine() {
256 				return 0;
257 			}
258 
259 			@Override
260 			public int getCharPositionInLine() {
261 				return 0;
262 			}
263 
264 			@Override
265 			public CharStream getInputStream() {
266 				return null;
267 			}
268 
269 			@Override
270 			public String getSourceName() {
271 				return IntStream.UNKNOWN_SOURCE_NAME;
272 			}
273 
274 			@Override
275 			public TokenFactory<?> getTokenFactory() {
276 				throw new UnsupportedOperationException("Not supported yet.");
277 			}
278 
279 			@Override
280 			public void setTokenFactory(TokenFactory<?> factory) {
281 				throw new UnsupportedOperationException("Not supported yet.");
282 			}
283 		};
284 
285 		CommonTokenStream tokens = new CommonTokenStream(lexer);
286 		tokens.fill();
287 
288 		assertEquals(Token.EOF, tokens.LA(1));
289 		assertEquals(0, tokens.index());
290 		assertEquals(1, tokens.size());
291 		tokens.consume();
292 	}
293 }
294