1 /* 2 * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26 package com.sun.tools.javac.parser; 27 28 import java.util.HashMap; 29 import java.util.Locale; 30 import java.util.function.Predicate; 31 import java.util.Map; 32 33 import com.sun.tools.javac.api.Formattable; 34 import com.sun.tools.javac.api.Messages; 35 import com.sun.tools.javac.parser.Tokens.Token.Tag; 36 import com.sun.tools.javac.util.*; 37 38 /** A class that defines codes/utilities for Java source tokens 39 * returned from lexical analysis. 40 * 41 * <p><b>This is NOT part of any supported API. 42 * If you write code that depends on this, you do so at your own risk. 43 * This code and its internal interfaces are subject to change or 44 * deletion without notice.</b> 45 */ 46 public class Tokens { 47 48 private final Names names; 49 50 /** 51 * Keyword array. Maps name indices to Token. 52 */ 53 private Map<String, TokenKind> keywords = new HashMap<>(); 54 55 public static final Context.Key<Tokens> tokensKey = new Context.Key<>(); 56 instance(Context context)57 public static Tokens instance(Context context) { 58 Tokens instance = context.get(tokensKey); 59 if (instance == null) 60 instance = new Tokens(context); 61 return instance; 62 } 63 Tokens(Context context)64 protected Tokens(Context context) { 65 context.put(tokensKey, this); 66 names = Names.instance(context); 67 for (TokenKind t : TokenKind.values()) { 68 if (t.name != null) { 69 names.fromString(t.name); 70 keywords.put(t.name, t); 71 } 72 } 73 } 74 75 /** 76 * Create a new token given a name; if the name corresponds to a token name, 77 * a new token of the corresponding kind is returned; otherwise, an 78 * identifier token is returned. 79 */ lookupKind(Name name)80 TokenKind lookupKind(Name name) { 81 TokenKind t = keywords.get(name.toString()); 82 return (t != null) ? t : TokenKind.IDENTIFIER; 83 } 84 lookupKind(String name)85 TokenKind lookupKind(String name) { 86 TokenKind t = keywords.get(name); 87 return (t != null) ? t : TokenKind.IDENTIFIER; 88 } 89 90 /** 91 * This enum defines all tokens used by the javac scanner. A token is 92 * optionally associated with a name. 93 */ 94 public enum TokenKind implements Formattable, Predicate<TokenKind> { 95 EOF(), 96 ERROR(), 97 IDENTIFIER(Tag.NAMED), 98 ABSTRACT("abstract"), 99 ASSERT("assert", Tag.NAMED), 100 BOOLEAN("boolean", Tag.NAMED), 101 BREAK("break"), 102 BYTE("byte", Tag.NAMED), 103 CASE("case"), 104 CATCH("catch"), 105 CHAR("char", Tag.NAMED), 106 CLASS("class"), 107 CONST("const"), 108 CONTINUE("continue"), 109 DEFAULT("default"), 110 DO("do"), 111 DOUBLE("double", Tag.NAMED), 112 ELSE("else"), 113 ENUM("enum", Tag.NAMED), 114 EXTENDS("extends"), 115 FINAL("final"), 116 FINALLY("finally"), 117 FLOAT("float", Tag.NAMED), 118 FOR("for"), 119 GOTO("goto"), 120 IF("if"), 121 IMPLEMENTS("implements"), 122 IMPORT("import"), 123 INSTANCEOF("instanceof"), 124 INT("int", Tag.NAMED), 125 INTERFACE("interface"), 126 LONG("long", Tag.NAMED), 127 NATIVE("native"), 128 NEW("new"), 129 PACKAGE("package"), 130 PRIVATE("private"), 131 PROTECTED("protected"), 132 PUBLIC("public"), 133 RETURN("return"), 134 SHORT("short", Tag.NAMED), 135 STATIC("static"), 136 STRICTFP("strictfp"), 137 SUPER("super", Tag.NAMED), 138 SWITCH("switch"), 139 SYNCHRONIZED("synchronized"), 140 THIS("this", Tag.NAMED), 141 THROW("throw"), 142 THROWS("throws"), 143 TRANSIENT("transient"), 144 TRY("try"), 145 VOID("void", Tag.NAMED), 146 VOLATILE("volatile"), 147 WHILE("while"), 148 INTLITERAL(Tag.NUMERIC), 149 LONGLITERAL(Tag.NUMERIC), 150 FLOATLITERAL(Tag.NUMERIC), 151 DOUBLELITERAL(Tag.NUMERIC), 152 CHARLITERAL(Tag.NUMERIC), 153 STRINGLITERAL(Tag.STRING), 154 TRUE("true", Tag.NAMED), 155 FALSE("false", Tag.NAMED), 156 NULL("null", Tag.NAMED), 157 UNDERSCORE("_", Tag.NAMED), 158 ARROW("->"), 159 COLCOL("::"), 160 LPAREN("("), 161 RPAREN(")"), 162 LBRACE("{"), 163 RBRACE("}"), 164 LBRACKET("["), 165 RBRACKET("]"), 166 SEMI(";"), 167 COMMA(","), 168 DOT("."), 169 ELLIPSIS("..."), 170 EQ("="), 171 GT(">"), 172 LT("<"), 173 BANG("!"), 174 TILDE("~"), 175 QUES("?"), 176 COLON(":"), 177 EQEQ("=="), 178 LTEQ("<="), 179 GTEQ(">="), 180 BANGEQ("!="), 181 AMPAMP("&&"), 182 BARBAR("||"), 183 PLUSPLUS("++"), 184 SUBSUB("--"), 185 PLUS("+"), 186 SUB("-"), 187 STAR("*"), 188 SLASH("/"), 189 AMP("&"), 190 BAR("|"), 191 CARET("^"), 192 PERCENT("%"), 193 LTLT("<<"), 194 GTGT(">>"), 195 GTGTGT(">>>"), 196 PLUSEQ("+="), 197 SUBEQ("-="), 198 STAREQ("*="), 199 SLASHEQ("/="), 200 AMPEQ("&="), 201 BAREQ("|="), 202 CARETEQ("^="), 203 PERCENTEQ("%="), 204 LTLTEQ("<<="), 205 GTGTEQ(">>="), 206 GTGTGTEQ(">>>="), 207 MONKEYS_AT("@"), 208 CUSTOM; 209 210 public final String name; 211 public final Tag tag; 212 TokenKind()213 TokenKind() { 214 this(null, Tag.DEFAULT); 215 } 216 TokenKind(String name)217 TokenKind(String name) { 218 this(name, Tag.DEFAULT); 219 } 220 TokenKind(Tag tag)221 TokenKind(Tag tag) { 222 this(null, tag); 223 } 224 TokenKind(String name, Tag tag)225 TokenKind(String name, Tag tag) { 226 this.name = name; 227 this.tag = tag; 228 } 229 toString()230 public String toString() { 231 switch (this) { 232 case IDENTIFIER: 233 return "token.identifier"; 234 case CHARLITERAL: 235 return "token.character"; 236 case STRINGLITERAL: 237 return "token.string"; 238 case INTLITERAL: 239 return "token.integer"; 240 case LONGLITERAL: 241 return "token.long-integer"; 242 case FLOATLITERAL: 243 return "token.float"; 244 case DOUBLELITERAL: 245 return "token.double"; 246 case ERROR: 247 return "token.bad-symbol"; 248 case EOF: 249 return "token.end-of-input"; 250 case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN: 251 case LBRACKET: case RBRACKET: case LBRACE: case RBRACE: 252 return "'" + name + "'"; 253 default: 254 return name; 255 } 256 } 257 getKind()258 public String getKind() { 259 return "Token"; 260 } 261 toString(Locale locale, Messages messages)262 public String toString(Locale locale, Messages messages) { 263 return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString()); 264 } 265 266 @Override test(TokenKind that)267 public boolean test(TokenKind that) { 268 return this == that; 269 } 270 } 271 272 public interface Comment { 273 274 enum CommentStyle { 275 LINE, 276 BLOCK, 277 JAVADOC, 278 } 279 getText()280 String getText(); getSourcePos(int index)281 int getSourcePos(int index); getStyle()282 CommentStyle getStyle(); isDeprecated()283 boolean isDeprecated(); 284 } 285 286 /** 287 * This is the class representing a javac token. Each token has several fields 288 * that are set by the javac lexer (i.e. start/end position, string value, etc). 289 */ 290 public static class Token { 291 292 /** tags constants **/ 293 enum Tag { 294 DEFAULT, 295 NAMED, 296 STRING, 297 NUMERIC; 298 } 299 300 /** The token kind */ 301 public final TokenKind kind; 302 303 /** The start position of this token */ 304 public final int pos; 305 306 /** The end position of this token */ 307 public final int endPos; 308 309 /** Comment reader associated with this token */ 310 public final List<Comment> comments; 311 Token(TokenKind kind, int pos, int endPos, List<Comment> comments)312 Token(TokenKind kind, int pos, int endPos, List<Comment> comments) { 313 this.kind = kind; 314 this.pos = pos; 315 this.endPos = endPos; 316 this.comments = comments; 317 checkKind(); 318 } 319 split(Tokens tokens)320 Token[] split(Tokens tokens) { 321 if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) { 322 throw new AssertionError("Cant split" + kind); 323 } 324 325 TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1)); 326 TokenKind t2 = tokens.lookupKind(kind.name.substring(1)); 327 328 if (t1 == null || t2 == null) { 329 throw new AssertionError("Cant split - bad subtokens"); 330 } 331 return new Token[] { 332 new Token(t1, pos, pos + t1.name.length(), comments), 333 new Token(t2, pos + t1.name.length(), endPos, null) 334 }; 335 } 336 checkKind()337 protected void checkKind() { 338 if (kind.tag != Tag.DEFAULT) { 339 throw new AssertionError("Bad token kind - expected " + Tag.DEFAULT); 340 } 341 } 342 name()343 public Name name() { 344 throw new UnsupportedOperationException(); 345 } 346 stringVal()347 public String stringVal() { 348 throw new UnsupportedOperationException(); 349 } 350 radix()351 public int radix() { 352 throw new UnsupportedOperationException(); 353 } 354 355 /** 356 * Preserve classic semantics - if multiple javadocs are found on the token 357 * the last one is returned 358 */ comment(Comment.CommentStyle style)359 public Comment comment(Comment.CommentStyle style) { 360 List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC); 361 return comments.isEmpty() ? 362 null : 363 comments.head; 364 } 365 366 /** 367 * Preserve classic semantics - deprecated should be set if at least one 368 * javadoc comment attached to this token contains the '@deprecated' string 369 */ deprecatedFlag()370 public boolean deprecatedFlag() { 371 for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) { 372 if (c.isDeprecated()) { 373 return true; 374 } 375 } 376 return false; 377 } 378 getComments(Comment.CommentStyle style)379 private List<Comment> getComments(Comment.CommentStyle style) { 380 if (comments == null) { 381 return List.nil(); 382 } else { 383 ListBuffer<Comment> buf = new ListBuffer<>(); 384 for (Comment c : comments) { 385 if (c.getStyle() == style) { 386 buf.add(c); 387 } 388 } 389 return buf.toList(); 390 } 391 } 392 } 393 394 static final class NamedToken extends Token { 395 /** The name of this token */ 396 public final Name name; 397 NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments)398 public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) { 399 super(kind, pos, endPos, comments); 400 this.name = name; 401 } 402 checkKind()403 protected void checkKind() { 404 if (kind.tag != Tag.NAMED) { 405 throw new AssertionError("Bad token kind - expected " + Tag.NAMED); 406 } 407 } 408 409 @Override name()410 public Name name() { 411 return name; 412 } 413 } 414 415 static class StringToken extends Token { 416 /** The string value of this token */ 417 public final String stringVal; 418 StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments)419 public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) { 420 super(kind, pos, endPos, comments); 421 this.stringVal = stringVal; 422 } 423 checkKind()424 protected void checkKind() { 425 if (kind.tag != Tag.STRING) { 426 throw new AssertionError("Bad token kind - expected " + Tag.STRING); 427 } 428 } 429 430 @Override stringVal()431 public String stringVal() { 432 return stringVal; 433 } 434 } 435 436 static final class NumericToken extends StringToken { 437 /** The 'radix' value of this token */ 438 public final int radix; 439 NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments)440 public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) { 441 super(kind, pos, endPos, stringVal, comments); 442 this.radix = radix; 443 } 444 checkKind()445 protected void checkKind() { 446 if (kind.tag != Tag.NUMERIC) { 447 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC); 448 } 449 } 450 451 @Override radix()452 public int radix() { 453 return radix; 454 } 455 } 456 457 public static final Token DUMMY = 458 new Token(TokenKind.ERROR, 0, 0, null); 459 } 460