1 /* 2 * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. 3 * Use of this file is governed by the BSD 3-clause license that 4 * can be found in the LICENSE.txt file in the project root. 5 */ 6 7 package org.antlr.v4.test.tool; 8 9 import org.antlr.v4.runtime.CharStream; 10 import org.antlr.v4.runtime.CommonToken; 11 import org.antlr.v4.runtime.CommonTokenStream; 12 import org.antlr.v4.runtime.IntStream; 13 import org.antlr.v4.runtime.Lexer; 14 import org.antlr.v4.runtime.Token; 15 import org.antlr.v4.runtime.TokenFactory; 16 import org.antlr.v4.runtime.TokenSource; 17 import org.antlr.v4.runtime.TokenStream; 18 import org.antlr.v4.runtime.WritableToken; 19 import org.junit.Before; 20 import org.junit.Test; 21 22 import static org.junit.Assert.assertEquals; 23 24 public class TestCommonTokenStream extends TestBufferedTokenStream { 25 @Before 26 @Override testSetUp()27 public void testSetUp() throws Exception { 28 super.testSetUp(); 29 } 30 31 @Override createTokenStream(TokenSource src)32 protected TokenStream createTokenStream(TokenSource src) { 33 return new CommonTokenStream(src); 34 } 35 testOffChannel()36 @Test public void testOffChannel() throws Exception { 37 TokenSource lexer = // simulate input " x =34 ;\n" 38 new TokenSource() { 39 int i = 0; 40 @SuppressWarnings("serial") 41 WritableToken[] tokens = { 42 new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, 43 new CommonToken(1,"x"), 44 new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, 45 new CommonToken(1,"="), 46 new CommonToken(1,"34"), 47 new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, 48 new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, 49 new CommonToken(1,";"), 50 new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}}, 51 new CommonToken(Token.EOF,"") 52 }; 53 @Override 54 public Token nextToken() { 55 return tokens[i++]; 56 } 57 @Override 58 public String getSourceName() { return "test"; } 59 @Override 60 public int getCharPositionInLine() { 61 return 0; 62 } 63 @Override 64 public int getLine() { 65 return 0; 66 } 67 @Override 68 public CharStream getInputStream() { 69 return null; 70 } 71 72 @Override 73 public void setTokenFactory(TokenFactory<?> factory) { 74 } 75 76 @Override 77 public TokenFactory<?> getTokenFactory() { 78 return null; 79 } 80 }; 81 82 CommonTokenStream tokens = new CommonTokenStream(lexer); 83 84 assertEquals("x", tokens.LT(1).getText()); // must skip first off channel token 85 tokens.consume(); 86 assertEquals("=", tokens.LT(1).getText()); 87 assertEquals("x", tokens.LT(-1).getText()); 88 89 tokens.consume(); 90 assertEquals("34", tokens.LT(1).getText()); 91 assertEquals("=", tokens.LT(-1).getText()); 92 93 tokens.consume(); 94 assertEquals(";", tokens.LT(1).getText()); 95 assertEquals("34", tokens.LT(-1).getText()); 96 97 tokens.consume(); 98 assertEquals(Token.EOF, tokens.LA(1)); 99 assertEquals(";", tokens.LT(-1).getText()); 100 101 assertEquals("34", tokens.LT(-2).getText()); 102 assertEquals("=", tokens.LT(-3).getText()); 103 assertEquals("x", tokens.LT(-4).getText()); 104 } 105 testFetchOffChannel()106 @Test public void testFetchOffChannel() throws Exception { 107 TokenSource lexer = // simulate input " x =34 ; \n" 108 // token indexes 01234 56789 109 new TokenSource() { 110 int i = 0; 111 @SuppressWarnings("serial") 112 WritableToken[] tokens = { 113 new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 0 114 new CommonToken(1,"x"), // 1 115 new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 2 116 new CommonToken(1,"="), // 3 117 new CommonToken(1,"34"), // 4 118 new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 5 119 new CommonToken(1," ") {{channel = Lexer.HIDDEN;}}, // 6 120 new CommonToken(1,";"), // 7 121 new CommonToken(1," ") {{channel = Lexer.HIDDEN;}},// 8 122 new CommonToken(1,"\n") {{channel = Lexer.HIDDEN;}},// 9 123 new CommonToken(Token.EOF,"") // 10 124 }; 125 @Override 126 public Token nextToken() { 127 return tokens[i++]; 128 } 129 @Override 130 public String getSourceName() { return "test"; } 131 @Override 132 public int getCharPositionInLine() { 133 return 0; 134 } 135 @Override 136 public int getLine() { 137 return 0; 138 } 139 @Override 140 public CharStream getInputStream() { 141 return null; 142 } 143 144 @Override 145 public void setTokenFactory(TokenFactory<?> factory) { 146 } 147 148 @Override 149 public TokenFactory<?> getTokenFactory() { 150 return null; 151 } 152 }; 153 154 CommonTokenStream tokens = new CommonTokenStream(lexer); 155 tokens.fill(); 156 assertEquals(null, tokens.getHiddenTokensToLeft(0)); 157 assertEquals(null, tokens.getHiddenTokensToRight(0)); 158 159 assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]", 160 tokens.getHiddenTokensToLeft(1).toString()); 161 assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", 162 tokens.getHiddenTokensToRight(1).toString()); 163 164 assertEquals(null, tokens.getHiddenTokensToLeft(2)); 165 assertEquals(null, tokens.getHiddenTokensToRight(2)); 166 167 assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", 168 tokens.getHiddenTokensToLeft(3).toString()); 169 assertEquals(null, tokens.getHiddenTokensToRight(3)); 170 171 assertEquals(null, tokens.getHiddenTokensToLeft(4)); 172 assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", 173 tokens.getHiddenTokensToRight(4).toString()); 174 175 assertEquals(null, tokens.getHiddenTokensToLeft(5)); 176 assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]", 177 tokens.getHiddenTokensToRight(5).toString()); 178 179 assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]", 180 tokens.getHiddenTokensToLeft(6).toString()); 181 assertEquals(null, tokens.getHiddenTokensToRight(6)); 182 183 assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", 184 tokens.getHiddenTokensToLeft(7).toString()); 185 assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]", 186 tokens.getHiddenTokensToRight(7).toString()); 187 188 assertEquals(null, tokens.getHiddenTokensToLeft(8)); 189 assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]", 190 tokens.getHiddenTokensToRight(8).toString()); 191 192 assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]", 193 tokens.getHiddenTokensToLeft(9).toString()); 194 assertEquals(null, tokens.getHiddenTokensToRight(9)); 195 } 196 197 @Test testSingleEOF()198 public void testSingleEOF() throws Exception { 199 TokenSource lexer = new TokenSource() { 200 201 @Override 202 public Token nextToken() { 203 return new CommonToken(Token.EOF); 204 } 205 206 @Override 207 public int getLine() { 208 return 0; 209 } 210 211 @Override 212 public int getCharPositionInLine() { 213 return 0; 214 } 215 216 @Override 217 public CharStream getInputStream() { 218 return null; 219 } 220 221 @Override 222 public String getSourceName() { 223 return IntStream.UNKNOWN_SOURCE_NAME; 224 } 225 226 @Override 227 public TokenFactory<?> getTokenFactory() { 228 throw new UnsupportedOperationException("Not supported yet."); 229 } 230 231 @Override 232 public void setTokenFactory(TokenFactory<?> factory) { 233 throw new UnsupportedOperationException("Not supported yet."); 234 } 235 }; 236 237 CommonTokenStream tokens = new CommonTokenStream(lexer); 238 tokens.fill(); 239 240 assertEquals(Token.EOF, tokens.LA(1)); 241 assertEquals(0, tokens.index()); 242 assertEquals(1, tokens.size()); 243 } 244 245 @Test(expected = IllegalStateException.class) testCannotConsumeEOF()246 public void testCannotConsumeEOF() throws Exception { 247 TokenSource lexer = new TokenSource() { 248 249 @Override 250 public Token nextToken() { 251 return new CommonToken(Token.EOF); 252 } 253 254 @Override 255 public int getLine() { 256 return 0; 257 } 258 259 @Override 260 public int getCharPositionInLine() { 261 return 0; 262 } 263 264 @Override 265 public CharStream getInputStream() { 266 return null; 267 } 268 269 @Override 270 public String getSourceName() { 271 return IntStream.UNKNOWN_SOURCE_NAME; 272 } 273 274 @Override 275 public TokenFactory<?> getTokenFactory() { 276 throw new UnsupportedOperationException("Not supported yet."); 277 } 278 279 @Override 280 public void setTokenFactory(TokenFactory<?> factory) { 281 throw new UnsupportedOperationException("Not supported yet."); 282 } 283 }; 284 285 CommonTokenStream tokens = new CommonTokenStream(lexer); 286 tokens.fill(); 287 288 assertEquals(Token.EOF, tokens.LA(1)); 289 assertEquals(0, tokens.index()); 290 assertEquals(1, tokens.size()); 291 tokens.consume(); 292 } 293 } 294