1 /*
2  * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4  *
5  * This code is free software; you can redistribute it and/or modify it
6  * under the terms of the GNU General Public License version 2 only, as
7  * published by the Free Software Foundation.  Oracle designates this
8  * particular file as subject to the "Classpath" exception as provided
9  * by Oracle in the LICENSE file that accompanied this code.
10  *
11  * This code is distributed in the hope that it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14  * version 2 for more details (a copy is included in the LICENSE file that
15  * accompanied this code).
16  *
17  * You should have received a copy of the GNU General Public License version
18  * 2 along with this work; if not, write to the Free Software Foundation,
19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20  *
21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22  * or visit www.oracle.com if you need additional information or have any
23  * questions.
24  */
25 
26 package com.sun.tools.javac.parser;
27 
28 import java.util.HashMap;
29 import java.util.Locale;
30 import java.util.function.Predicate;
31 import java.util.Map;
32 
33 import com.sun.tools.javac.api.Formattable;
34 import com.sun.tools.javac.api.Messages;
35 import com.sun.tools.javac.parser.Tokens.Token.Tag;
36 import com.sun.tools.javac.util.*;
37 
38 /** A class that defines codes/utilities for Java source tokens
39  *  returned from lexical analysis.
40  *
41  *  <p><b>This is NOT part of any supported API.
42  *  If you write code that depends on this, you do so at your own risk.
43  *  This code and its internal interfaces are subject to change or
44  *  deletion without notice.</b>
45  */
46 public class Tokens {
47 
48     private final Names names;
49 
50     /**
51      * Keyword array. Maps name indices to Token.
52      */
53     private Map<String, TokenKind> keywords = new HashMap<>();
54 
55     public static final Context.Key<Tokens> tokensKey = new Context.Key<>();
56 
instance(Context context)57     public static Tokens instance(Context context) {
58         Tokens instance = context.get(tokensKey);
59         if (instance == null)
60             instance = new Tokens(context);
61         return instance;
62     }
63 
Tokens(Context context)64     protected Tokens(Context context) {
65         context.put(tokensKey, this);
66         names = Names.instance(context);
67         for (TokenKind t : TokenKind.values()) {
68             if (t.name != null) {
69                 names.fromString(t.name);
70                 keywords.put(t.name, t);
71             }
72         }
73     }
74 
75     /**
76      * Create a new token given a name; if the name corresponds to a token name,
77      * a new token of the corresponding kind is returned; otherwise, an
78      * identifier token is returned.
79      */
lookupKind(Name name)80     TokenKind lookupKind(Name name) {
81         TokenKind t = keywords.get(name.toString());
82         return (t != null) ? t : TokenKind.IDENTIFIER;
83     }
84 
lookupKind(String name)85     TokenKind lookupKind(String name) {
86         TokenKind t = keywords.get(name);
87         return (t != null) ? t : TokenKind.IDENTIFIER;
88     }
89 
90     /**
91      * This enum defines all tokens used by the javac scanner. A token is
92      * optionally associated with a name.
93      */
94     public enum TokenKind implements Formattable, Predicate<TokenKind> {
95         EOF(),
96         ERROR(),
97         IDENTIFIER(Tag.NAMED),
98         ABSTRACT("abstract"),
99         ASSERT("assert", Tag.NAMED),
100         BOOLEAN("boolean", Tag.NAMED),
101         BREAK("break"),
102         BYTE("byte", Tag.NAMED),
103         CASE("case"),
104         CATCH("catch"),
105         CHAR("char", Tag.NAMED),
106         CLASS("class"),
107         CONST("const"),
108         CONTINUE("continue"),
109         DEFAULT("default"),
110         DO("do"),
111         DOUBLE("double", Tag.NAMED),
112         ELSE("else"),
113         ENUM("enum", Tag.NAMED),
114         EXTENDS("extends"),
115         FINAL("final"),
116         FINALLY("finally"),
117         FLOAT("float", Tag.NAMED),
118         FOR("for"),
119         GOTO("goto"),
120         IF("if"),
121         IMPLEMENTS("implements"),
122         IMPORT("import"),
123         INSTANCEOF("instanceof"),
124         INT("int", Tag.NAMED),
125         INTERFACE("interface"),
126         LONG("long", Tag.NAMED),
127         NATIVE("native"),
128         NEW("new"),
129         PACKAGE("package"),
130         PRIVATE("private"),
131         PROTECTED("protected"),
132         PUBLIC("public"),
133         RETURN("return"),
134         SHORT("short", Tag.NAMED),
135         STATIC("static"),
136         STRICTFP("strictfp"),
137         SUPER("super", Tag.NAMED),
138         SWITCH("switch"),
139         SYNCHRONIZED("synchronized"),
140         THIS("this", Tag.NAMED),
141         THROW("throw"),
142         THROWS("throws"),
143         TRANSIENT("transient"),
144         TRY("try"),
145         VOID("void", Tag.NAMED),
146         VOLATILE("volatile"),
147         WHILE("while"),
148         INTLITERAL(Tag.NUMERIC),
149         LONGLITERAL(Tag.NUMERIC),
150         FLOATLITERAL(Tag.NUMERIC),
151         DOUBLELITERAL(Tag.NUMERIC),
152         CHARLITERAL(Tag.NUMERIC),
153         STRINGLITERAL(Tag.STRING),
154         TRUE("true", Tag.NAMED),
155         FALSE("false", Tag.NAMED),
156         NULL("null", Tag.NAMED),
157         UNDERSCORE("_", Tag.NAMED),
158         ARROW("->"),
159         COLCOL("::"),
160         LPAREN("("),
161         RPAREN(")"),
162         LBRACE("{"),
163         RBRACE("}"),
164         LBRACKET("["),
165         RBRACKET("]"),
166         SEMI(";"),
167         COMMA(","),
168         DOT("."),
169         ELLIPSIS("..."),
170         EQ("="),
171         GT(">"),
172         LT("<"),
173         BANG("!"),
174         TILDE("~"),
175         QUES("?"),
176         COLON(":"),
177         EQEQ("=="),
178         LTEQ("<="),
179         GTEQ(">="),
180         BANGEQ("!="),
181         AMPAMP("&&"),
182         BARBAR("||"),
183         PLUSPLUS("++"),
184         SUBSUB("--"),
185         PLUS("+"),
186         SUB("-"),
187         STAR("*"),
188         SLASH("/"),
189         AMP("&"),
190         BAR("|"),
191         CARET("^"),
192         PERCENT("%"),
193         LTLT("<<"),
194         GTGT(">>"),
195         GTGTGT(">>>"),
196         PLUSEQ("+="),
197         SUBEQ("-="),
198         STAREQ("*="),
199         SLASHEQ("/="),
200         AMPEQ("&="),
201         BAREQ("|="),
202         CARETEQ("^="),
203         PERCENTEQ("%="),
204         LTLTEQ("<<="),
205         GTGTEQ(">>="),
206         GTGTGTEQ(">>>="),
207         MONKEYS_AT("@"),
208         CUSTOM;
209 
210         public final String name;
211         public final Tag tag;
212 
TokenKind()213         TokenKind() {
214             this(null, Tag.DEFAULT);
215         }
216 
TokenKind(String name)217         TokenKind(String name) {
218             this(name, Tag.DEFAULT);
219         }
220 
TokenKind(Tag tag)221         TokenKind(Tag tag) {
222             this(null, tag);
223         }
224 
TokenKind(String name, Tag tag)225         TokenKind(String name, Tag tag) {
226             this.name = name;
227             this.tag = tag;
228         }
229 
toString()230         public String toString() {
231             switch (this) {
232             case IDENTIFIER:
233                 return "token.identifier";
234             case CHARLITERAL:
235                 return "token.character";
236             case STRINGLITERAL:
237                 return "token.string";
238             case INTLITERAL:
239                 return "token.integer";
240             case LONGLITERAL:
241                 return "token.long-integer";
242             case FLOATLITERAL:
243                 return "token.float";
244             case DOUBLELITERAL:
245                 return "token.double";
246             case ERROR:
247                 return "token.bad-symbol";
248             case EOF:
249                 return "token.end-of-input";
250             case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
251             case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
252                 return "'" + name + "'";
253             default:
254                 return name;
255             }
256         }
257 
getKind()258         public String getKind() {
259             return "Token";
260         }
261 
toString(Locale locale, Messages messages)262         public String toString(Locale locale, Messages messages) {
263             return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
264         }
265 
266         @Override
test(TokenKind that)267         public boolean test(TokenKind that) {
268             return this == that;
269         }
270     }
271 
272     public interface Comment {
273 
274         enum CommentStyle {
275             LINE,
276             BLOCK,
277             JAVADOC,
278         }
279 
getText()280         String getText();
getSourcePos(int index)281         int getSourcePos(int index);
getStyle()282         CommentStyle getStyle();
isDeprecated()283         boolean isDeprecated();
284     }
285 
286     /**
287      * This is the class representing a javac token. Each token has several fields
288      * that are set by the javac lexer (i.e. start/end position, string value, etc).
289      */
290     public static class Token {
291 
292         /** tags constants **/
293         enum Tag {
294             DEFAULT,
295             NAMED,
296             STRING,
297             NUMERIC;
298         }
299 
300         /** The token kind */
301         public final TokenKind kind;
302 
303         /** The start position of this token */
304         public final int pos;
305 
306         /** The end position of this token */
307         public final int endPos;
308 
309         /** Comment reader associated with this token */
310         public final List<Comment> comments;
311 
Token(TokenKind kind, int pos, int endPos, List<Comment> comments)312         Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
313             this.kind = kind;
314             this.pos = pos;
315             this.endPos = endPos;
316             this.comments = comments;
317             checkKind();
318         }
319 
split(Tokens tokens)320         Token[] split(Tokens tokens) {
321             if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
322                 throw new AssertionError("Cant split" + kind);
323             }
324 
325             TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
326             TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
327 
328             if (t1 == null || t2 == null) {
329                 throw new AssertionError("Cant split - bad subtokens");
330             }
331             return new Token[] {
332                 new Token(t1, pos, pos + t1.name.length(), comments),
333                 new Token(t2, pos + t1.name.length(), endPos, null)
334             };
335         }
336 
checkKind()337         protected void checkKind() {
338             if (kind.tag != Tag.DEFAULT) {
339                 throw new AssertionError("Bad token kind - expected " + Tag.DEFAULT);
340             }
341         }
342 
name()343         public Name name() {
344             throw new UnsupportedOperationException();
345         }
346 
stringVal()347         public String stringVal() {
348             throw new UnsupportedOperationException();
349         }
350 
radix()351         public int radix() {
352             throw new UnsupportedOperationException();
353         }
354 
355         /**
356          * Preserve classic semantics - if multiple javadocs are found on the token
357          * the last one is returned
358          */
comment(Comment.CommentStyle style)359         public Comment comment(Comment.CommentStyle style) {
360             List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);
361             return comments.isEmpty() ?
362                     null :
363                     comments.head;
364         }
365 
366         /**
367          * Preserve classic semantics - deprecated should be set if at least one
368          * javadoc comment attached to this token contains the '@deprecated' string
369          */
deprecatedFlag()370         public boolean deprecatedFlag() {
371             for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {
372                 if (c.isDeprecated()) {
373                     return true;
374                 }
375             }
376             return false;
377         }
378 
getComments(Comment.CommentStyle style)379         private List<Comment> getComments(Comment.CommentStyle style) {
380             if (comments == null) {
381                 return List.nil();
382             } else {
383                 ListBuffer<Comment> buf = new ListBuffer<>();
384                 for (Comment c : comments) {
385                     if (c.getStyle() == style) {
386                         buf.add(c);
387                     }
388                 }
389                 return buf.toList();
390             }
391         }
392     }
393 
394     static final class NamedToken extends Token {
395         /** The name of this token */
396         public final Name name;
397 
NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments)398         public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {
399             super(kind, pos, endPos, comments);
400             this.name = name;
401         }
402 
checkKind()403         protected void checkKind() {
404             if (kind.tag != Tag.NAMED) {
405                 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
406             }
407         }
408 
409         @Override
name()410         public Name name() {
411             return name;
412         }
413     }
414 
415     static class StringToken extends Token {
416         /** The string value of this token */
417         public final String stringVal;
418 
StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments)419         public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
420             super(kind, pos, endPos, comments);
421             this.stringVal = stringVal;
422         }
423 
checkKind()424         protected void checkKind() {
425             if (kind.tag != Tag.STRING) {
426                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
427             }
428         }
429 
430         @Override
stringVal()431         public String stringVal() {
432             return stringVal;
433         }
434     }
435 
436     static final class NumericToken extends StringToken {
437         /** The 'radix' value of this token */
438         public final int radix;
439 
NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments)440         public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
441             super(kind, pos, endPos, stringVal, comments);
442             this.radix = radix;
443         }
444 
checkKind()445         protected void checkKind() {
446             if (kind.tag != Tag.NUMERIC) {
447                 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
448             }
449         }
450 
451         @Override
radix()452         public int radix() {
453             return radix;
454         }
455     }
456 
457     public static final Token DUMMY =
458                 new Token(TokenKind.ERROR, 0, 0, null);
459 }
460