1 /*******************************************************************************
2 * Copyright (c) 2000, 2020 IBM Corporation and others.
3 *
4 * This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License 2.0
6 * which accompanies this distribution, and is available at
7 * https://www.eclipse.org/legal/epl-2.0/
8 *
9 * SPDX-License-Identifier: EPL-2.0
10 *
11 * Contributors:
12 * IBM Corporation - initial API and implementation
13 * Stephan Herrmann - Contribution for bug 186342 - [compiler][null] Using annotations for null checking
14 * Jesper S Moller -. Contribution for bug 400830: [1.8][formatter] Code formatter for Java 8
15 *******************************************************************************/
16 package org.eclipse.jdt.internal.compiler.parser;
17
18 import java.util.ArrayList;
19 import java.util.List;
20
21 import org.eclipse.jdt.core.compiler.CharOperation;
22 import org.eclipse.jdt.core.compiler.InvalidInputException;
23 import org.eclipse.jdt.internal.compiler.CompilationResult;
24 import org.eclipse.jdt.internal.compiler.DefaultErrorHandlingPolicies;
25 import org.eclipse.jdt.internal.compiler.ast.Statement;
26 import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants;
27 import org.eclipse.jdt.internal.compiler.impl.CompilerOptions;
28 import org.eclipse.jdt.internal.compiler.problem.DefaultProblemFactory;
29 import org.eclipse.jdt.internal.compiler.problem.ProblemReporter;
30 import org.eclipse.jdt.internal.compiler.util.Util;
31
32 /**
33 * IMPORTANT NOTE: Internal Scanner implementation. It is mirrored in
34 * org.eclipse.jdt.core.compiler public package where it is API.
35 * The mirror implementation is using the backward compatible ITerminalSymbols constant
36 * definitions (stable with 2.0), whereas the internal implementation uses TerminalTokens
37 * which constant values reflect the latest parser generation state.
38 */
39 public class Scanner implements TerminalTokens {
40
41 //public int newIdentCount = 0;
42
43 /* APIs ares
44 - getNextToken() which return the current type of the token
45 (this value is not memorized by the scanner)
46 - getCurrentTokenSource() which provides with the token "REAL" source
47 (aka all unicode have been transformed into a correct char)
48 - sourceStart gives the position into the stream
49 - currentPosition-1 gives the sourceEnd position into the stream
50 */
51 public long sourceLevel;
52 public long complianceLevel;
53
54 // 1.4 feature
55 public boolean useAssertAsAnIndentifier = false;
56 //flag indicating if processed source contains occurrences of keyword assert
57 public boolean containsAssertKeyword = false;
58 public boolean previewEnabled;
59
60 // 1.5 feature
61 public boolean useEnumAsAnIndentifier = false;
62
63 public boolean recordLineSeparator = false;
64 public char currentCharacter;
65 public int startPosition;
66 public int currentPosition;
67 public int initialPosition, eofPosition;
68 // after this position eof are generated instead of real token from the source
69
70 public boolean skipComments = false;
71 public boolean tokenizeComments = false;
72 public boolean tokenizeWhiteSpace = false;
73
74 //source should be viewed as a window (aka a part)
75 //of a entire very large stream
76 public char source[];
77
78 //unicode support
79 public char[] withoutUnicodeBuffer;
80 public int withoutUnicodePtr; //when == 0 ==> no unicode in the current token
81 public boolean unicodeAsBackSlash = false;
82
83 public boolean scanningFloatLiteral = false;
84
85 //support for /** comments
86 public final static int COMMENT_ARRAYS_SIZE = 30;
87 public int[] commentStops = new int[COMMENT_ARRAYS_SIZE];
88 public int[] commentStarts = new int[COMMENT_ARRAYS_SIZE];
89 public int[] commentTagStarts = new int[COMMENT_ARRAYS_SIZE];
90 public int commentPtr = -1; // no comment test with commentPtr value -1
91 public int lastCommentLinePosition = -1;
92
93 // task tag support
94 public char[][] foundTaskTags = null;
95 public char[][] foundTaskMessages;
96 public char[][] foundTaskPriorities = null;
97 public int[][] foundTaskPositions;
98 public int foundTaskCount = 0;
99 public char[][] taskTags = null;
100 public char[][] taskPriorities = null;
101 public boolean isTaskCaseSensitive = true;
102
103 //diet parsing support - jump over some method body when requested
104 public boolean diet = false;
105
106 //support for the poor-line-debuggers ....
107 //remember the position of the cr/lf
108 public int[] lineEnds = new int[250];
109 public int linePtr = -1;
110 public boolean wasAcr = false;
111
112 public boolean fakeInModule = false;
113 boolean inCase = false;
114 /* package */ int yieldColons = -1;
115 boolean breakPreviewAllowed = false;
116 /**
117 * The current context of the scanner w.r.t restricted keywords
118 *
119 */
120 enum ScanContext {
121 EXPECTING_KEYWORD, EXPECTING_IDENTIFIER, AFTER_REQUIRES, INACTIVE
122 }
123 protected ScanContext scanContext = null;
124 protected boolean insideModuleInfo = false;
125 public static final String END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$
126
127 public static final String INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$
128 public static final String INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$
129 public static final String INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant"; //$NON-NLS-1$
130 public static final String INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$
131 public static final String INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$
132 public static final String INVALID_TEXTBLOCK = "Invalid_Textblock"; //$NON-NLS-1$
133 public static final String INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$
134 public static final String INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$
135 public static final String INVALID_LOW_SURROGATE = "Invalid_Low_Surrogate"; //$NON-NLS-1$
136 public static final String INVALID_HIGH_SURROGATE = "Invalid_High_Surrogate"; //$NON-NLS-1$
137
138 public static final String NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$
139 public static final String UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$
140 public static final String UNTERMINATED_TEXT_BLOCK = "Unterminated_Text_Block"; //$NON-NLS-1$
141 public static final String UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$
142 public static final String INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$
143 public static final String INVALID_DIGIT = "Invalid_Digit"; //$NON-NLS-1$
144 private static final int[] EMPTY_LINE_ENDS = Util.EMPTY_INT_ARRAY;
145
146 public static final String INVALID_BINARY = "Invalid_Binary_Literal"; //$NON-NLS-1$
147 public static final String BINARY_LITERAL_NOT_BELOW_17 = "Binary_Literal_Not_Below_17"; //$NON-NLS-1$
148 public static final String ILLEGAL_HEXA_LITERAL = "Illegal_Hexa_Literal"; //$NON-NLS-1$
149 public static final String INVALID_UNDERSCORE = "Invalid_Underscore"; //$NON-NLS-1$
150 public static final String UNDERSCORES_IN_LITERALS_NOT_BELOW_17 = "Underscores_In_Literals_Not_Below_17"; //$NON-NLS-1$
151
152 //----------------optimized identifier managment------------------
153 static final char[] charArray_a = new char[] {'a'},
154 charArray_b = new char[] {'b'},
155 charArray_c = new char[] {'c'},
156 charArray_d = new char[] {'d'},
157 charArray_e = new char[] {'e'},
158 charArray_f = new char[] {'f'},
159 charArray_g = new char[] {'g'},
160 charArray_h = new char[] {'h'},
161 charArray_i = new char[] {'i'},
162 charArray_j = new char[] {'j'},
163 charArray_k = new char[] {'k'},
164 charArray_l = new char[] {'l'},
165 charArray_m = new char[] {'m'},
166 charArray_n = new char[] {'n'},
167 charArray_o = new char[] {'o'},
168 charArray_p = new char[] {'p'},
169 charArray_q = new char[] {'q'},
170 charArray_r = new char[] {'r'},
171 charArray_s = new char[] {'s'},
172 charArray_t = new char[] {'t'},
173 charArray_u = new char[] {'u'},
174 charArray_v = new char[] {'v'},
175 charArray_w = new char[] {'w'},
176 charArray_x = new char[] {'x'},
177 charArray_y = new char[] {'y'},
178 charArray_z = new char[] {'z'};
179
180 static final char[] initCharArray =
181 new char[] {'\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000'};
182 static final int TableSize = 30, InternalTableSize = 6; //30*6 =210 entries
183
184 public static final int OptimizedLength = 7;
185 public /*static*/ final char[][][][] charArray_length =
186 new char[OptimizedLength][TableSize][InternalTableSize][];
187 // support for detecting non-externalized string literals
188 public static final char[] TAG_PREFIX= "//$NON-NLS-".toCharArray(); //$NON-NLS-1$
189 public static final int TAG_PREFIX_LENGTH= TAG_PREFIX.length;
190 public static final char TAG_POSTFIX= '$';
191 public static final int TAG_POSTFIX_LENGTH= 1;
192
193 // support for complaining on uninterned type comparisons.
194 public static final char[] IDENTITY_COMPARISON_TAG = "//$IDENTITY-COMPARISON$".toCharArray(); //$NON-NLS-1$
195 public boolean [] validIdentityComparisonLines;
196 public boolean checkUninternedIdentityComparison;
197
198 private NLSTag[] nlsTags = null;
199 protected int nlsTagsPtr;
200 public boolean checkNonExternalizedStringLiterals;
201
202 protected int lastPosition;
203
204 // generic support
205 public boolean returnOnlyGreater = false;
206
207 /*static*/ {
208 for (int i = 0; i < 6; i++) {
209 for (int j = 0; j < TableSize; j++) {
210 for (int k = 0; k < InternalTableSize; k++) {
211 this.charArray_length[i][j][k] = initCharArray;
212 }
213 }
214 }
215 }
216 /*static*/ int newEntry2 = 0,
217 newEntry3 = 0,
218 newEntry4 = 0,
219 newEntry5 = 0,
220 newEntry6 = 0;
221 public boolean insideRecovery = false;
222 int lookBack[] = new int[2]; // fall back to spring forward.
223 protected int nextToken = TokenNameNotAToken; // allows for one token push back, only the most recent token can be reliably ungotten.
224 private VanguardScanner vanguardScanner;
225 private VanguardParser vanguardParser;
226 ConflictedParser activeParser = null;
227 private boolean consumingEllipsisAnnotations = false;
228
229 public static final int RoundBracket = 0;
230 public static final int SquareBracket = 1;
231 public static final int CurlyBracket = 2;
232 public static final int BracketKinds = 3;
233
234 // extended unicode support
235 public static final int LOW_SURROGATE_MIN_VALUE = 0xDC00;
236 public static final int HIGH_SURROGATE_MIN_VALUE = 0xD800;
237 public static final int HIGH_SURROGATE_MAX_VALUE = 0xDBFF;
238 public static final int LOW_SURROGATE_MAX_VALUE = 0xDFFF;
239
240 // text block support - 13
241 /* package */ int rawStart = -1;
242
Scanner()243 public Scanner() {
244 this(false /*comment*/, false /*whitespace*/, false /*nls*/, ClassFileConstants.JDK1_3 /*sourceLevel*/, null/*taskTag*/, null/*taskPriorities*/, true /*taskCaseSensitive*/);
245 }
246
Scanner( boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, long sourceLevel, long complianceLevel, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive, boolean isPreviewEnabled)247 public Scanner(
248 boolean tokenizeComments,
249 boolean tokenizeWhiteSpace,
250 boolean checkNonExternalizedStringLiterals,
251 long sourceLevel,
252 long complianceLevel,
253 char[][] taskTags,
254 char[][] taskPriorities,
255 boolean isTaskCaseSensitive,
256 boolean isPreviewEnabled) {
257
258 this.eofPosition = Integer.MAX_VALUE;
259 this.tokenizeComments = tokenizeComments;
260 this.tokenizeWhiteSpace = tokenizeWhiteSpace;
261 this.sourceLevel = sourceLevel;
262 this.lookBack[0] = this.lookBack[1] = this.nextToken = TokenNameNotAToken;
263 this.consumingEllipsisAnnotations = false;
264 this.complianceLevel = complianceLevel;
265 this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
266 this.previewEnabled = isPreviewEnabled;
267 if (taskTags != null) {
268 int taskTagsLength = taskTags.length;
269 int length = taskTagsLength;
270 if (taskPriorities != null) {
271 int taskPrioritiesLength = taskPriorities.length;
272 if (taskPrioritiesLength != taskTagsLength) {
273 if (taskPrioritiesLength > taskTagsLength) {
274 System.arraycopy(taskPriorities, 0, (taskPriorities = new char[taskTagsLength][]), 0, taskTagsLength);
275 } else {
276 System.arraycopy(taskTags, 0, (taskTags = new char[taskPrioritiesLength][]), 0, taskPrioritiesLength);
277 length = taskPrioritiesLength;
278 }
279 }
280 int[] initialIndexes = new int[length];
281 for (int i = 0; i < length; i++) {
282 initialIndexes[i] = i;
283 }
284 Util.reverseQuickSort(taskTags, 0, length - 1, initialIndexes);
285 char[][] temp = new char[length][];
286 for (int i = 0; i < length; i++) {
287 temp[i] = taskPriorities[initialIndexes[i]];
288 }
289 this.taskPriorities = temp;
290 } else {
291 Util.reverseQuickSort(taskTags, 0, length - 1);
292 }
293 this.taskTags = taskTags;
294 this.isTaskCaseSensitive = isTaskCaseSensitive;
295 }
296 }
297
Scanner( boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, long sourceLevel, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive, boolean isPreviewEnabled)298 public Scanner(
299 boolean tokenizeComments,
300 boolean tokenizeWhiteSpace,
301 boolean checkNonExternalizedStringLiterals,
302 long sourceLevel,
303 char[][] taskTags,
304 char[][] taskPriorities,
305 boolean isTaskCaseSensitive,
306 boolean isPreviewEnabled) {
307
308 this(
309 tokenizeComments,
310 tokenizeWhiteSpace,
311 checkNonExternalizedStringLiterals,
312 sourceLevel,
313 sourceLevel,
314 taskTags,
315 taskPriorities,
316 isTaskCaseSensitive,
317 isPreviewEnabled);
318 }
319
Scanner( boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, long sourceLevel, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive)320 public Scanner(
321 boolean tokenizeComments,
322 boolean tokenizeWhiteSpace,
323 boolean checkNonExternalizedStringLiterals,
324 long sourceLevel,
325 char[][] taskTags,
326 char[][] taskPriorities,
327 boolean isTaskCaseSensitive) {
328
329 this(
330 tokenizeComments,
331 tokenizeWhiteSpace,
332 checkNonExternalizedStringLiterals,
333 sourceLevel,
334 sourceLevel,
335 taskTags,
336 taskPriorities,
337 isTaskCaseSensitive,
338 false);
339 }
atEnd()340 public final boolean atEnd() {
341 // This code is not relevant if source is
342 // Only a part of the real stream input
343
344 return this.eofPosition <= this.currentPosition;
345 }
346
347 // chech presence of task: tags
348 // TODO (frederic) see if we need to take unicode characters into account...
checkTaskTag(int commentStart, int commentEnd)349 public void checkTaskTag(int commentStart, int commentEnd) throws InvalidInputException {
350 char[] src = this.source;
351
352 // only look for newer task: tags
353 if (this.foundTaskCount > 0
354 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
355 return;
356 }
357 int foundTaskIndex = this.foundTaskCount;
358 char previous = src[commentStart+1]; // should be '*' or '/'
359 for (
360 int i = commentStart + 2; i < commentEnd && i < this.eofPosition; i++) {
361 char[] tag = null;
362 char[] priority = null;
363 // check for tag occurrence only if not ambiguous with javadoc tag
364 if (previous != '@') {
365 nextTag : for (int itag = 0; itag < this.taskTags.length; itag++) {
366 tag = this.taskTags[itag];
367 int tagLength = tag.length;
368 if (tagLength == 0) continue nextTag;
369
370 // ensure tag is not leaded with letter if tag starts with a letter
371 if (ScannerHelper.isJavaIdentifierStart(this.complianceLevel, tag[0])) {
372 if (ScannerHelper.isJavaIdentifierPart(this.complianceLevel, previous)) {
373 continue nextTag;
374 }
375 }
376
377 for (int t = 0; t < tagLength; t++) {
378 char sc, tc;
379 int x = i+t;
380 if (x >= this.eofPosition || x >= commentEnd) continue nextTag;
381 // case sensitive check
382 if ((sc = src[i + t]) != (tc = tag[t])) {
383 // case insensitive check
384 if (this.isTaskCaseSensitive || (ScannerHelper.toLowerCase(sc) != ScannerHelper.toLowerCase(tc))) {
385 continue nextTag;
386 }
387 }
388 }
389 // ensure tag is not followed with letter if tag finishes with a letter
390 if (i+tagLength < commentEnd && ScannerHelper.isJavaIdentifierPart(this.complianceLevel, src[i+tagLength-1])) {
391 if (ScannerHelper.isJavaIdentifierPart(this.complianceLevel, src[i + tagLength]))
392 continue nextTag;
393 }
394 if (this.foundTaskTags == null) {
395 this.foundTaskTags = new char[5][];
396 this.foundTaskMessages = new char[5][];
397 this.foundTaskPriorities = new char[5][];
398 this.foundTaskPositions = new int[5][];
399 } else if (this.foundTaskCount == this.foundTaskTags.length) {
400 System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
401 System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
402 System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
403 System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0, this.foundTaskCount);
404 }
405
406 priority = this.taskPriorities != null && itag < this.taskPriorities.length
407 ? this.taskPriorities[itag]
408 : null;
409
410 this.foundTaskTags[this.foundTaskCount] = tag;
411 this.foundTaskPriorities[this.foundTaskCount] = priority;
412 this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
413 this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
414 this.foundTaskCount++;
415 i += tagLength - 1; // will be incremented when looping
416 break nextTag;
417 }
418 }
419 previous = src[i];
420 }
421 boolean containsEmptyTask = false;
422 for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
423 // retrieve message start and end positions
424 int msgStart = this.foundTaskPositions[i][0] + this.foundTaskTags[i].length;
425 int max_value = i + 1 < this.foundTaskCount
426 ? this.foundTaskPositions[i + 1][0] - 1
427 : commentEnd - 1;
428 // at most beginning of next task
429 if (max_value < msgStart) {
430 max_value = msgStart; // would only occur if tag is before EOF.
431 }
432 int end = -1;
433 char c;
434 for (int j = msgStart; j < max_value; j++) {
435 if ((c = src[j]) == '\n' || c == '\r') {
436 end = j - 1;
437 break;
438 }
439 }
440 if (end == -1) {
441 for (int j = max_value; j > msgStart; j--) {
442 if ((c = src[j]) == '*') {
443 end = j - 1;
444 break;
445 }
446 }
447 if (end == -1)
448 end = max_value;
449 }
450 if (msgStart == end) {
451 // if the description is empty, we might want to see if two tags are not sharing the same message
452 // see https://bugs.eclipse.org/bugs/show_bug.cgi?id=110797
453 containsEmptyTask = true;
454 continue;
455 }
456 // trim the message
457 // we don't trim the beginning of the message to be able to show it after the task tag
458 while (CharOperation.isWhitespace(src[end]) && msgStart <= end)
459 end--;
460 // update the end position of the task
461 this.foundTaskPositions[i][1] = end;
462 // get the message source
463 final int messageLength = end - msgStart + 1;
464 char[] message = new char[messageLength];
465 System.arraycopy(src, msgStart, message, 0, messageLength);
466 this.foundTaskMessages[i] = message;
467 }
468 if (containsEmptyTask) {
469 for (int i = foundTaskIndex, max = this.foundTaskCount; i < max; i++) {
470 if (this.foundTaskMessages[i].length == 0) {
471 loop: for (int j = i + 1; j < max; j++) {
472 if (this.foundTaskMessages[j].length != 0) {
473 this.foundTaskMessages[i] = this.foundTaskMessages[j];
474 this.foundTaskPositions[i][1] = this.foundTaskPositions[j][1];
475 break loop;
476 }
477 }
478 }
479 }
480 }
481 }
482
483 public char[] getCurrentIdentifierSource() {
484 //return the token REAL source (aka unicodes are precomputed)
485 if (this.withoutUnicodePtr != 0) {
486 //0 is used as a fast test flag so the real first char is in position 1
487 char[] result = new char[this.withoutUnicodePtr];
488 System.arraycopy(
489 this.withoutUnicodeBuffer,
490 1,
491 result,
492 0,
493 this.withoutUnicodePtr);
494 return result;
495 }
496 int length = this.currentPosition - this.startPosition;
497 if (length == this.eofPosition) return this.source;
498 switch (length) { // see OptimizedLength
499 case 1 :
500 return optimizedCurrentTokenSource1();
501 case 2 :
502 return optimizedCurrentTokenSource2();
503 case 3 :
504 return optimizedCurrentTokenSource3();
505 case 4 :
506 return optimizedCurrentTokenSource4();
507 case 5 :
508 return optimizedCurrentTokenSource5();
509 case 6 :
510 return optimizedCurrentTokenSource6();
511 }
512 char[] result = new char[length];
513 System.arraycopy(this.source, this.startPosition, result, 0, length);
514 return result;
515 }
516 public int getCurrentTokenEndPosition(){
517 return this.currentPosition - 1;
518 }
519 public char[] getCurrentTokenSource() {
520 // Return the token REAL source (aka unicodes are precomputed)
521
522 char[] result;
523 if (this.withoutUnicodePtr != 0)
524 // 0 is used as a fast test flag so the real first char is in position 1
525 System.arraycopy(
526 this.withoutUnicodeBuffer,
527 1,
528 result = new char[this.withoutUnicodePtr],
529 0,
530 this.withoutUnicodePtr);
531 else {
532 int length;
533 System.arraycopy(
534 this.source,
535 this.startPosition,
536 result = new char[length = this.currentPosition - this.startPosition],
537 0,
538 length);
539 }
540 return result;
541 }
542 public final String getCurrentTokenString() {
543 // Return current token as a string
544
545 if (this.withoutUnicodePtr != 0) {
546 // 0 is used as a fast test flag so the real first char is in position 1
547 return new String(
548 this.withoutUnicodeBuffer,
549 1,
550 this.withoutUnicodePtr);
551 }
552 return new String(
553 this.source,
554 this.startPosition,
555 this.currentPosition - this.startPosition);
556 }
557 public char[] getCurrentTokenSourceString() {
558 //return the token REAL source (aka unicodes are precomputed).
559 //REMOVE the two " that are at the beginning and the end.
560
561 char[] result;
562 if (this.withoutUnicodePtr != 0)
563 //0 is used as a fast test flag so the real first char is in position 1
564 System.arraycopy(this.withoutUnicodeBuffer, 2,
565 //2 is 1 (real start) + 1 (to jump over the ")
566 result = new char[this.withoutUnicodePtr - 2], 0, this.withoutUnicodePtr - 2);
567 else {
568 int length;
569 System.arraycopy(
570 this.source,
571 this.startPosition + 1,
572 result = new char[length = this.currentPosition - this.startPosition - 2],
573 0,
574 length);
575 }
576 return result;
577 }
578 protected final boolean scanForTextBlockBeginning() {
579 try {
580 // Don't change the position and current character unless we are certain
581 // to be dealing with a text block. For producing all errors like before
582 // in case of a valid """ but missing \r or \n, just return false and not
583 // throw any error.
584 int temp = this.currentPosition;
585 if ((this.source[temp++] == '\"' && this.source[temp++] == '\"')) {
586 char c = this.source[temp++];
587 while (ScannerHelper.isWhitespace(c)) {
588 switch (c) {
589 case 10 : /* \ u000a: LINE FEED */
590 this.currentCharacter = c;
591 this.currentPosition = temp;
592 return true;
593 default:
594 break;
595 }
596 c = this.source[temp++];
597 }
598 }
599 } catch(IndexOutOfBoundsException e) {
600 //let it return false;
601 }
602 return false;
603 }
604 protected final boolean scanForTextBlockClose() throws InvalidInputException {
605 try {
606 if (this.source[this.currentPosition] == '\"' && this.source[this.currentPosition + 1] == '\"') {
607 return true;
608 }
609 } catch(IndexOutOfBoundsException e) {
610 //let it return false;
611 }
612 return false;
613 }
614 public char[] getCurrentTextBlock() {
615 // 1. Normalize, i.e. convert all CR CRLF to LF
616 char[] all;
617 if (this.withoutUnicodePtr != 0) {
618 all = CharOperation.subarray(this.withoutUnicodeBuffer, this.rawStart + 1, this.withoutUnicodePtr + 1 );
619 } else {
620 all = CharOperation.subarray(this.source, this.startPosition + this.rawStart, this.currentPosition - 3);
621 if (all == null) {
622 all = new char[0];
623 }
624 }
625 all = normalize(all);
626 // 2. Split into lines. Consider both \n and \r as line separators
627 char[][] lines = CharOperation.splitOn('\n', all);
628 int size = lines.length;
629 List<char[]> list = new ArrayList<>(lines.length);
630 for(int i = 0; i < lines.length; i++) {
631 char[] line = lines[i];
632 if (i + 1 == size && line.length == 0) {
633 list.add(line);
634 break;
635 }
636 char[][] sub = CharOperation.splitOn('\r', line);
637 if (sub.length == 0) {
638 list.add(line);
639 } else {
640 for (char[] cs : sub) {
641 list.add(cs);
642 }
643 }
644 }
645 size = list.size();
646 lines = list.toArray(new char[size][]);
647
648 // 3. Handle incidental white space
649 // 3.1. Split into lines and identify determining lines
650 int prefix = -1;
651 for(int i = 0; i < size; i++) {
652 char[] line = lines[i];
653 boolean blank = true;
654 int whitespaces = 0;
655 for (char c : line) {
656 if (blank) {
657 if (ScannerHelper.isWhitespace(c)) {
658 whitespaces++;
659 } else {
660 blank = false;
661 }
662 }
663 }
664 // The last line with closing delimiter is part of the
665 // determining line list even if empty
666 if (!blank || (i+1 == size)) {
667 if (prefix < 0 || whitespaces < prefix) {
668 prefix = whitespaces;
669 }
670 }
671 }
672 // 3.2. Remove the common white space prefix
673 // 4. Handle escape sequences that are not already done in getNextToken0()
674 if (prefix == -1)
675 prefix = 0;
676 StringBuilder result = new StringBuilder();
677 boolean newLine = false;
678 for(int i = 0; i < lines.length; i++) {
679 char[] l = lines[i];
680 // Remove the common prefix from each line
681 // And remove all trailing whitespace
682 // Finally append the \n at the end of the line (except the last line)
683 int length = l.length;
684 int trail = length;
685 for(;trail > 0;) {
686 if (!ScannerHelper.isWhitespace(l[trail-1])) {
687 break;
688 }
689 trail--;
690 }
691 if (i >= (size -1)) {
692 if (newLine) result.append('\n');
693 if (trail < prefix)
694 continue;
695 newLine = getLineContent(result, l, prefix, trail-1, false, true);
696 } else {
697 if (i > 0 && newLine)
698 result.append('\n');
699 if (trail <= prefix) {
700 newLine = true;
701 } else {
702 boolean merge = length > 0 && l[length - 1] == '\\';
703 newLine = getLineContent(result, l, prefix, trail-1, merge, false);
704 }
705 }
706 }
707 // get rid of all the cached values
708 this.rawStart = -1;
709 return result.toString().toCharArray();
710 }
711 private char[] normalize(char[] content) {
712 StringBuilder result = new StringBuilder();
713 boolean isCR = false;
714 for (char c : content) {
715 switch (c) {
716 case '\r':
717 result.append(c);
718 isCR = true;
719 break;
720 case '\n':
721 if (!isCR) {
722 result.append(c);
723 }
724 isCR = false;
725 break;
726 default:
727 result.append(c);
728 isCR = false;
729 break;
730 }
731 }
732 return result.toString().toCharArray();
733 }
734 // This method is for handling the left over escaped characters during the first
735 // scanning (scanForStringLiteral). Admittedly this goes over the text block
736 // content again char by char, but this is required in order to correctly
737 // treat all the white space and line endings
738 private boolean getLineContent(StringBuilder result, char[] line, int start, int end, boolean merge, boolean lastLine) {
739 int lastPointer = 0;
740 for(int i = start; i < end; i++) {
741 char c = line[i];
742 if (c == '\\') {
743 if (i < end) {
744 if (lastPointer == i) {
745 lastPointer = i+1;
746 } else {
747 result.append(CharOperation.subarray(line, lastPointer == 0 ? start : lastPointer+1, i));
748 }
749 switch (line[++i]) {
750 case '\\' :
751 result.append('\\');
752 if (i == end)
753 merge = false;
754 //i = lastPointer;
755 lastPointer = i;
756 break;
757 case 's' :
758 result.append(' ');
759 lastPointer = i;
760 break;
761 case 'n' :
762 result.append('\n');
763 lastPointer = i;
764 break;
765 case 'r' :
766 result.append('\r');
767 lastPointer = i;
768 break;
769 case 'f' :
770 result.append('\f');
771 lastPointer = i;
772 break;
773 default :
774 // Direct copy from scanEscapeCharacter
775 int pos = i;
776 char ch = line[pos];
777 int number = ScannerHelper.getHexadecimalValue(ch);
778 if (number >= 0 && number <= 7) {
779 boolean zeroToThreeNot = number > 3;
780 try {
781 if (ScannerHelper.isDigit(ch = line[++pos])) {
782 int digit = ScannerHelper.getHexadecimalValue(ch);
783 if (digit >= 0 && digit <= 7) {
784 number = (number * 8) + digit;
785 if (ScannerHelper.isDigit(ch = line[++pos])) {
786 if (zeroToThreeNot) {
787 // has read \NotZeroToThree OctalDigit Digit --> ignore last character
788 } else {
789 digit = ScannerHelper.getHexadecimalValue(ch);
790 if (digit >= 0 && digit <= 7){ // has read \ZeroToThree OctalDigit OctalDigit
791 number = (number * 8) + digit;
792 } else {
793 // has read \ZeroToThree OctalDigit NonOctalDigit --> ignore last character
794 }
795 }
796 } else {
797 // has read \OctalDigit NonDigit--> ignore last character
798 }
799 } else {
800 // has read \OctalDigit NonOctalDigit--> ignore last character
801 }
802 } else {
803 // has read \OctalDigit --> ignore last character
804 }
805 } catch (InvalidInputException e) {
806 // Unlikely as this has already been processed in scanForStringLiteral()
807 }
808 if (number < 255) {
809 ch = (char) number;
810 //replaceEscapedChar(result, line, start, end, i, lastPointer, ch);
811 }
812 result.append(ch);
813 lastPointer = i = pos -1;
814 } else {
815 // Dealing with just '\'
816 result.append(c);
817 lastPointer = --i;
818 }
819 }
820 }
821 }
822 }
823 end = merge ? end : end >= line.length ? end : end + 1;
824 char[] chars = lastPointer == 0 ?
825 CharOperation.subarray(line, start, end) :
826 CharOperation.subarray(line, lastPointer + 1, end);
827 // The below check is because CharOperation.subarray tend to return null when the
828 // boundaries produce a zero sized char[]
829 if (chars != null)
830 result.append(chars);
831 return (!merge && !lastLine);
832 }
833 public final String getCurrentStringLiteral() {
834 //return the token REAL source (aka unicodes are precomputed).
835 //REMOVE the two " that are at the beginning and the end.
836
837 if (this.withoutUnicodePtr != 0)
838 //0 is used as a fast test flag so the real first char is in position 1
839 //2 is 1 (real start) + 1 (to jump over the ")
840 return new String(this.withoutUnicodeBuffer, 2, this.withoutUnicodePtr - 2);
841 else {
842 return new String(this.source, this.startPosition + 1, this.currentPosition - this.startPosition - 2);
843 }
844 }
845 public final char[] getRawTokenSource() {
846 int length = this.currentPosition - this.startPosition;
847 char[] tokenSource = new char[length];
848 System.arraycopy(this.source, this.startPosition, tokenSource, 0, length);
849 return tokenSource;
850 }
851
852 public final char[] getRawTokenSourceEnd() {
853 int length = this.eofPosition - this.currentPosition - 1;
854 char[] sourceEnd = new char[length];
855 System.arraycopy(this.source, this.currentPosition, sourceEnd, 0, length);
856 return sourceEnd;
857 }
858
859 public int getCurrentTokenStartPosition(){
860 return this.startPosition;
861 }
862 /*
863 * Search the source position corresponding to the end of a given line number
864 *
865 * Line numbers are 1-based, and relative to the scanner initialPosition.
866 * Character positions are 0-based.
867 *
868 * In case the given line number is inconsistent, answers -1.
869 */
870 public final int getLineEnd(int lineNumber) {
871
872 if (this.lineEnds == null || this.linePtr == -1)
873 return -1;
874 if (lineNumber > this.lineEnds.length+1)
875 return -1;
876 if (lineNumber <= 0)
877 return -1;
878 if (lineNumber == this.lineEnds.length + 1)
879 return this.eofPosition;
880 return this.lineEnds[lineNumber-1]; // next line start one character behind the lineEnd of the previous line
881 }
882
883 public final int[] getLineEnds() {
884 //return a bounded copy of this.lineEnds
885 if (this.linePtr == -1) {
886 return EMPTY_LINE_ENDS;
887 }
888 int[] copy;
889 System.arraycopy(this.lineEnds, 0, copy = new int[this.linePtr + 1], 0, this.linePtr + 1);
890 return copy;
891 }
892
893 /**
894 * Search the source position corresponding to the beginning of a given line number
895 *
896 * Line numbers are 1-based, and relative to the scanner initialPosition.
897 * Character positions are 0-based.
898 *
899 * e.g. getLineStart(1) --> 0 indicates that the first line starts at character 0.
900 *
901 * In case the given line number is inconsistent, answers -1.
902 *
903 * @param lineNumber int
904 * @return int
905 */
906 public final int getLineStart(int lineNumber) {
907
908 if (this.lineEnds == null || this.linePtr == -1)
909 return -1;
910 if (lineNumber > this.lineEnds.length + 1)
911 return -1;
912 if (lineNumber <= 0)
913 return -1;
914
915 if (lineNumber == 1)
916 return this.initialPosition;
917 return this.lineEnds[lineNumber-2]+1; // next line start one character behind the lineEnd of the previous line
918 }
919 public final int getNextChar() {
920 try {
921 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
922 && (this.source[this.currentPosition] == 'u')) {
923 getNextUnicodeChar();
924 } else {
925 this.unicodeAsBackSlash = false;
926 if (this.withoutUnicodePtr != 0) {
927 unicodeStore();
928 }
929 }
930 return this.currentCharacter;
931 } catch(IndexOutOfBoundsException | InvalidInputException e) {
932 return -1;
933 }
934 }
935 public final int getNextCharWithBoundChecks() {
936 if (this.currentPosition >= this.eofPosition) {
937 return -1;
938 }
939 this.currentCharacter = this.source[this.currentPosition++];
940 if (this.currentPosition >= this.eofPosition) {
941 this.unicodeAsBackSlash = false;
942 if (this.withoutUnicodePtr != 0) {
943 unicodeStore();
944 }
945 return this.currentCharacter;
946 }
947 if (this.currentCharacter == '\\' && this.source[this.currentPosition] == 'u') {
948 try {
949 getNextUnicodeChar();
950 } catch (InvalidInputException e) {
951 return -1;
952 }
953 } else {
954 this.unicodeAsBackSlash = false;
955 if (this.withoutUnicodePtr != 0) {
956 unicodeStore();
957 }
958 }
959 return this.currentCharacter;
960 }
961 public final boolean getNextChar(char testedChar) {
962 //BOOLEAN
963 //handle the case of unicode.
964 //when a unicode appears then we must use a buffer that holds char internal values
965 //At the end of this method currentCharacter holds the new visited char
966 //and currentPosition points right next after it
967 //Both previous lines are true if the currentCharacter is == to the testedChar
968 //On false, no side effect has occured.
969
970 //ALL getNextChar.... ARE OPTIMIZED COPIES
971
972 if (this.currentPosition >= this.eofPosition) { // handle the obvious case upfront
973 this.unicodeAsBackSlash = false;
974 return false;
975 }
976
977 int temp = this.currentPosition;
978 try {
979 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
980 && (this.source[this.currentPosition] == 'u')) {
981 getNextUnicodeChar();
982 if (this.currentCharacter != testedChar) {
983 this.currentPosition = temp;
984 this.withoutUnicodePtr--;
985 return false;
986 }
987 return true;
988 } //-------------end unicode traitement--------------
989 else {
990 if (this.currentCharacter != testedChar) {
991 this.currentPosition = temp;
992 return false;
993 }
994 this.unicodeAsBackSlash = false;
995 if (this.withoutUnicodePtr != 0)
996 unicodeStore();
997 return true;
998 }
999 } catch(IndexOutOfBoundsException | InvalidInputException e) {
1000 this.unicodeAsBackSlash = false;
1001 this.currentPosition = temp;
1002 return false;
1003 }
1004 }
1005 public final int getNextChar(char testedChar1, char testedChar2) {
1006 //INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others
1007 //test can be done with (x==0) for the first and (x>0) for the second
1008 //handle the case of unicode.
1009 //when a unicode appears then we must use a buffer that holds char internal values
1010 //At the end of this method currentCharacter holds the new visited char
1011 //and currentPosition points right next after it
1012 //Both previous lines are true if the currentCharacter is == to the testedChar1/2
1013 //On false, no side effect has occured.
1014
1015 //ALL getNextChar.... ARE OPTIMIZED COPIES
1016 if (this.currentPosition >= this.eofPosition) // handle the obvious case upfront
1017 return -1;
1018
1019 int temp = this.currentPosition;
1020 try {
1021 int result;
1022 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1023 && (this.source[this.currentPosition] == 'u')) {
1024 getNextUnicodeChar();
1025 if (this.currentCharacter == testedChar1) {
1026 result = 0;
1027 } else if (this.currentCharacter == testedChar2) {
1028 result = 1;
1029 } else {
1030 this.currentPosition = temp;
1031 this.withoutUnicodePtr--;
1032 result = -1;
1033 }
1034 return result;
1035 } else {
1036 if (this.currentCharacter == testedChar1) {
1037 result = 0;
1038 } else if (this.currentCharacter == testedChar2) {
1039 result = 1;
1040 } else {
1041 this.currentPosition = temp;
1042 return -1;
1043 }
1044
1045 if (this.withoutUnicodePtr != 0)
1046 unicodeStore();
1047 return result;
1048 }
1049 } catch(IndexOutOfBoundsException | InvalidInputException e) {
1050 this.currentPosition = temp;
1051 return -1;
1052 }
1053 }
1054 /*
1055 * This method consumes digits as well as underscores if underscores are located between digits
1056 * @throws InvalidInputException if underscores are not located between digits or if underscores are used in source < 1.7
1057 */
1058 private final void consumeDigits(int radix) throws InvalidInputException {
1059 consumeDigits(radix, false);
1060 }
1061 /*
1062 * This method consumes digits as well as underscores if underscores are located between digits
1063 * @throws InvalidInputException if underscores are not located between digits or if underscores are used in source < 1.7
1064 */
1065 private final void consumeDigits(int radix, boolean expectingDigitFirst) throws InvalidInputException {
1066 final int USING_UNDERSCORE = 1;
1067 final int INVALID_POSITION = 2;
1068 switch(consumeDigits0(radix, USING_UNDERSCORE, INVALID_POSITION, expectingDigitFirst)) {
1069 case USING_UNDERSCORE :
1070 if (this.sourceLevel < ClassFileConstants.JDK1_7) {
1071 throw new InvalidInputException(UNDERSCORES_IN_LITERALS_NOT_BELOW_17);
1072 }
1073 break;
1074 case INVALID_POSITION :
1075 if (this.sourceLevel < ClassFileConstants.JDK1_7) {
1076 throw new InvalidInputException(UNDERSCORES_IN_LITERALS_NOT_BELOW_17);
1077 }
1078 throw new InvalidInputException(INVALID_UNDERSCORE);
1079 }
1080 }
1081 private final int consumeDigits0(int radix, int usingUnderscore, int invalidPosition, boolean expectingDigitFirst) throws InvalidInputException {
1082 int kind = 0;
1083 if (getNextChar('_')) {
1084 if (expectingDigitFirst) {
1085 return invalidPosition;
1086 }
1087 kind = usingUnderscore;
1088 while (getNextChar('_')) {/*empty */}
1089 }
1090 if (getNextCharAsDigit(radix)) {
1091 // continue to read digits or underscore
1092 while (getNextCharAsDigit(radix)) {/*empty */}
1093 int kind2 = consumeDigits0(radix, usingUnderscore, invalidPosition, false);
1094 if (kind2 == 0) {
1095 return kind;
1096 }
1097 return kind2;
1098 }
1099 if (kind == usingUnderscore) return invalidPosition;
1100 return kind;
1101 }
1102 public final boolean getNextCharAsDigit() throws InvalidInputException {
1103 //BOOLEAN
1104 //handle the case of unicode.
1105 //when a unicode appears then we must use a buffer that holds char internal values
1106 //At the end of this method currentCharacter holds the new visited char
1107 //and currentPosition points right next after it
1108 //Both previous lines are true if the currentCharacter is a digit
1109 //On false, no side effect has occured.
1110
1111 //ALL getNextChar.... ARE OPTIMIZED COPIES
1112 if (this.currentPosition >= this.eofPosition) // handle the obvious case upfront
1113 return false;
1114
1115 int temp = this.currentPosition;
1116 try {
1117 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1118 && (this.source[this.currentPosition] == 'u')) {
1119 getNextUnicodeChar();
1120 if (!ScannerHelper.isDigit(this.currentCharacter)) {
1121 this.currentPosition = temp;
1122 this.withoutUnicodePtr--;
1123 return false;
1124 }
1125 return true;
1126 } else {
1127 if (!ScannerHelper.isDigit(this.currentCharacter)) {
1128 this.currentPosition = temp;
1129 return false;
1130 }
1131 if (this.withoutUnicodePtr != 0)
1132 unicodeStore();
1133 return true;
1134 }
1135 } catch(IndexOutOfBoundsException | InvalidInputException e) {
1136 this.currentPosition = temp;
1137 return false;
1138 }
1139 }
1140 public final boolean getNextCharAsDigit(int radix) {
1141 //BOOLEAN
1142 //handle the case of unicode.
1143 //when a unicode appears then we must use a buffer that holds char internal values
1144 //At the end of this method currentCharacter holds the new visited char
1145 //and currentPosition points right next after it
1146 //Both previous lines are true if the currentCharacter is a digit base on radix
1147 //On false, no side effect has occured.
1148
1149 //ALL getNextChar.... ARE OPTIMIZED COPIES
1150 if (this.currentPosition >= this.eofPosition) // handle the obvious case upfront
1151 return false;
1152
1153 int temp = this.currentPosition;
1154 try {
1155 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1156 && (this.source[this.currentPosition] == 'u')) {
1157 getNextUnicodeChar();
1158 if (ScannerHelper.digit(this.currentCharacter, radix) == -1) {
1159 this.currentPosition = temp;
1160 this.withoutUnicodePtr--;
1161 return false;
1162 }
1163 return true;
1164 } else {
1165 if (ScannerHelper.digit(this.currentCharacter, radix) == -1) {
1166 this.currentPosition = temp;
1167 return false;
1168 }
1169 if (this.withoutUnicodePtr != 0)
1170 unicodeStore();
1171 return true;
1172 }
1173 } catch(IndexOutOfBoundsException | InvalidInputException e) {
1174 this.currentPosition = temp;
1175 return false;
1176 }
1177 }
1178 public boolean getNextCharAsJavaIdentifierPartWithBoundCheck() {
1179 //BOOLEAN
1180 //handle the case of unicode.
1181 //when a unicode appears then we must use a buffer that holds char internal values
1182 //At the end of this method currentCharacter holds the new visited char
1183 //and currentPosition points right next after it
1184 //Both previous lines are true if the currentCharacter is a JavaIdentifierPart
1185 //On false, no side effect has occured.
1186
1187 //ALL getNextChar.... ARE OPTIMIZED COPIES
1188 int pos = this.currentPosition;
1189 if (pos >= this.eofPosition) // handle the obvious case upfront
1190 return false;
1191
1192 int temp2 = this.withoutUnicodePtr;
1193 try {
1194 boolean unicode = false;
1195 this.currentCharacter = this.source[this.currentPosition++];
1196 if (this.currentPosition < this.eofPosition) {
1197 if (this.currentCharacter == '\\' && this.source[this.currentPosition] == 'u') {
1198 getNextUnicodeChar();
1199 unicode = true;
1200 }
1201 }
1202 char c = this.currentCharacter;
1203 boolean isJavaIdentifierPart = false;
1204 if (c >= HIGH_SURROGATE_MIN_VALUE && c <= HIGH_SURROGATE_MAX_VALUE) {
1205 if (this.complianceLevel < ClassFileConstants.JDK1_5) {
1206 this.currentPosition = pos;
1207 this.withoutUnicodePtr = temp2;
1208 return false;
1209 }
1210 // Unicode 4 detection
1211 char low = (char) getNextCharWithBoundChecks();
1212 if (low < LOW_SURROGATE_MIN_VALUE || low > LOW_SURROGATE_MAX_VALUE) {
1213 // illegal low surrogate
1214 this.currentPosition = pos;
1215 this.withoutUnicodePtr = temp2;
1216 return false;
1217 }
1218 isJavaIdentifierPart = ScannerHelper.isJavaIdentifierPart(this.complianceLevel, c, low);
1219 }
1220 else if (c >= LOW_SURROGATE_MIN_VALUE && c <= LOW_SURROGATE_MAX_VALUE) {
1221 this.currentPosition = pos;
1222 this.withoutUnicodePtr = temp2;
1223 return false;
1224 } else {
1225 isJavaIdentifierPart = ScannerHelper.isJavaIdentifierPart(this.complianceLevel, c);
1226 }
1227 if (unicode) {
1228 if (!isJavaIdentifierPart) {
1229 this.currentPosition = pos;
1230 this.withoutUnicodePtr = temp2;
1231 return false;
1232 }
1233 return true;
1234 } else {
1235 if (!isJavaIdentifierPart) {
1236 this.currentPosition = pos;
1237 return false;
1238 }
1239
1240 if (this.withoutUnicodePtr != 0)
1241 unicodeStore();
1242 return true;
1243 }
1244 } catch(InvalidInputException e) {
1245 this.currentPosition = pos;
1246 this.withoutUnicodePtr = temp2;
1247 return false;
1248 }
1249 }
1250 public boolean getNextCharAsJavaIdentifierPart() {
1251 //BOOLEAN
1252 //handle the case of unicode.
1253 //when a unicode appears then we must use a buffer that holds char internal values
1254 //At the end of this method currentCharacter holds the new visited char
1255 //and currentPosition points right next after it
1256 //Both previous lines are true if the currentCharacter is a JavaIdentifierPart
1257 //On false, no side effect has occured.
1258
1259 //ALL getNextChar.... ARE OPTIMIZED COPIES
1260 int pos;
1261 if ((pos = this.currentPosition) >= this.eofPosition) // handle the obvious case upfront
1262 return false;
1263
1264 int temp2 = this.withoutUnicodePtr;
1265 try {
1266 boolean unicode = false;
1267 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1268 && (this.source[this.currentPosition] == 'u')) {
1269 getNextUnicodeChar();
1270 unicode = true;
1271 }
1272 char c = this.currentCharacter;
1273 boolean isJavaIdentifierPart = false;
1274 if (c >= HIGH_SURROGATE_MIN_VALUE && c <= HIGH_SURROGATE_MAX_VALUE) {
1275 if (this.complianceLevel < ClassFileConstants.JDK1_5) {
1276 this.currentPosition = pos;
1277 this.withoutUnicodePtr = temp2;
1278 return false;
1279 }
1280 // Unicode 4 detection
1281 char low = (char) getNextChar();
1282 if (low < LOW_SURROGATE_MIN_VALUE || low > LOW_SURROGATE_MAX_VALUE) {
1283 // illegal low surrogate
1284 this.currentPosition = pos;
1285 this.withoutUnicodePtr = temp2;
1286 return false;
1287 }
1288 isJavaIdentifierPart = ScannerHelper.isJavaIdentifierPart(this.complianceLevel, c, low);
1289 }
1290 else if (c >= LOW_SURROGATE_MIN_VALUE && c <= LOW_SURROGATE_MAX_VALUE) {
1291 this.currentPosition = pos;
1292 this.withoutUnicodePtr = temp2;
1293 return false;
1294 } else {
1295 isJavaIdentifierPart = ScannerHelper.isJavaIdentifierPart(this.complianceLevel, c);
1296 }
1297 if (unicode) {
1298 if (!isJavaIdentifierPart) {
1299 this.currentPosition = pos;
1300 this.withoutUnicodePtr = temp2;
1301 return false;
1302 }
1303 return true;
1304 } else {
1305 if (!isJavaIdentifierPart) {
1306 this.currentPosition = pos;
1307 return false;
1308 }
1309
1310 if (this.withoutUnicodePtr != 0)
1311 unicodeStore();
1312 return true;
1313 }
1314 } catch(IndexOutOfBoundsException | InvalidInputException e) {
1315 this.currentPosition = pos;
1316 this.withoutUnicodePtr = temp2;
1317 return false;
1318 }
1319 }
1320 /*
1321 * External API in JavaConventions.
1322 * This is used to optimize the case where the scanner is used to scan a single identifier.
1323 * In this case, the AIOOBE is slower to handle than a bound check
1324 */
1325 public int scanIdentifier() throws InvalidInputException {
1326 int whiteStart = 0;
1327 while (true) { //loop for jumping over comments
1328 this.withoutUnicodePtr = 0;
1329 //start with a new token (even comment written with unicode )
1330 // ---------Consume white space and handles startPosition---------
1331 whiteStart = this.currentPosition;
1332 boolean isWhiteSpace, hasWhiteSpaces = false;
1333 int offset;
1334 int unicodePtr;
1335 boolean checkIfUnicode = false;
1336 do {
1337 unicodePtr = this.withoutUnicodePtr;
1338 offset = this.currentPosition;
1339 this.startPosition = this.currentPosition;
1340 if (this.currentPosition < this.eofPosition) {
1341 this.currentCharacter = this.source[this.currentPosition++];
1342 checkIfUnicode = this.currentPosition < this.eofPosition
1343 && this.currentCharacter == '\\'
1344 && this.source[this.currentPosition] == 'u';
1345 } else if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) {
1346 // reposition scanner in case we are interested by spaces as tokens
1347 this.currentPosition--;
1348 this.startPosition = whiteStart;
1349 return TokenNameWHITESPACE;
1350 } else {
1351 return TokenNameEOF;
1352 }
1353 if (checkIfUnicode) {
1354 isWhiteSpace = jumpOverUnicodeWhiteSpace();
1355 offset = this.currentPosition - offset;
1356 } else {
1357 offset = this.currentPosition - offset;
1358 // inline version of:
1359 //isWhiteSpace =
1360 // (this.currentCharacter == ' ') || ScannerHelper.isWhitespace(this.currentCharacter);
1361 switch (this.currentCharacter) {
1362 case 10 : /* \ u000a: LINE FEED */
1363 case 12 : /* \ u000c: FORM FEED */
1364 case 13 : /* \ u000d: CARRIAGE RETURN */
1365 case 32 : /* \ u0020: SPACE */
1366 case 9 : /* \ u0009: HORIZONTAL TABULATION */
1367 isWhiteSpace = true;
1368 break;
1369 default :
1370 isWhiteSpace = false;
1371 }
1372 }
1373 if (isWhiteSpace) {
1374 hasWhiteSpaces = true;
1375 }
1376 } while (isWhiteSpace);
1377 if (hasWhiteSpaces) {
1378 if (this.tokenizeWhiteSpace) {
1379 // reposition scanner in case we are interested by spaces as tokens
1380 this.currentPosition-=offset;
1381 this.startPosition = whiteStart;
1382 if (checkIfUnicode) {
1383 this.withoutUnicodePtr = unicodePtr;
1384 }
1385 return TokenNameWHITESPACE;
1386 } else if (checkIfUnicode) {
1387 this.withoutUnicodePtr = 0;
1388 unicodeStore();
1389 } else {
1390 this.withoutUnicodePtr = 0;
1391 }
1392 }
1393 char c = this.currentCharacter;
1394 if (c < ScannerHelper.MAX_OBVIOUS) {
1395 if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_IDENT_START) != 0) {
1396 return scanIdentifierOrKeywordWithBoundCheck();
1397 }
1398 return TokenNameERROR;
1399 }
1400 boolean isJavaIdStart;
1401 if (c >= HIGH_SURROGATE_MIN_VALUE && c <= HIGH_SURROGATE_MAX_VALUE) {
1402 if (this.complianceLevel < ClassFileConstants.JDK1_5) {
1403 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1404 }
1405 // Unicode 4 detection
1406 char low = (char) getNextCharWithBoundChecks();
1407 if (low < LOW_SURROGATE_MIN_VALUE || low > LOW_SURROGATE_MAX_VALUE) {
1408 // illegal low surrogate
1409 throw new InvalidInputException(INVALID_LOW_SURROGATE);
1410 }
1411 isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c, low);
1412 } else if (c >= LOW_SURROGATE_MIN_VALUE && c <= LOW_SURROGATE_MAX_VALUE) {
1413 if (this.complianceLevel < ClassFileConstants.JDK1_5) {
1414 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1415 }
1416 throw new InvalidInputException(INVALID_HIGH_SURROGATE);
1417 } else {
1418 // optimized case already checked
1419 isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c);
1420 }
1421 if (isJavaIdStart)
1422 return scanIdentifierOrKeywordWithBoundCheck();
1423 return TokenNameERROR;
1424 }
1425 }
1426 public void ungetToken(int unambiguousToken) {
1427 if (this.nextToken != TokenNameNotAToken) {
1428 throw new ArrayIndexOutOfBoundsException("Single cell array overflow"); //$NON-NLS-1$
1429 }
1430 this.nextToken = unambiguousToken;
1431 }
1432 private void updateCase(int token) {
1433 if (token == TokenNamecase) {
1434 this.inCase = true;
1435 this.breakPreviewAllowed = true;
1436 }
1437 if (token == TokenNameCOLON || token == TokenNameARROW)
1438 this.inCase = false;
1439 }
1440 public int getNextToken() throws InvalidInputException {
1441
1442 int token;
1443 if (this.nextToken != TokenNameNotAToken) {
1444 token = this.nextToken;
1445 this.nextToken = TokenNameNotAToken;
1446 return token; // presumed to be unambiguous.
1447 }
1448 if (this.scanContext == null) { // init lazily, since isInModuleDeclaration needs the parser to be known
1449 this.scanContext = isInModuleDeclaration() ? ScanContext.EXPECTING_KEYWORD : ScanContext.INACTIVE;
1450 }
1451 token = getNextToken0();
1452 if (areRestrictedModuleKeywordsActive()) {
1453 if (isRestrictedKeyword(token))
1454 token = disambiguatedRestrictedKeyword(token);
1455 updateScanContext(token);
1456 }
1457 if (this.activeParser == null) { // anybody interested in the grammatical structure of the program should have registered.
1458 return token;
1459 }
1460 if (token == TokenNameLPAREN || token == TokenNameLESS || token == TokenNameAT || token == TokenNameARROW) {
1461 token = disambiguatedToken(token);
1462 } else if (token == TokenNameELLIPSIS) {
1463 this.consumingEllipsisAnnotations = false;
1464 }
1465 this.lookBack[0] = this.lookBack[1];
1466 this.lookBack[1] = token;
1467 updateCase(token);
1468 return token;
1469 }
1470 protected int getNextToken0() throws InvalidInputException {
1471 this.wasAcr = false;
1472 if (this.diet) {
1473 jumpOverMethodBody();
1474 this.diet = false;
1475 return this.currentPosition > this.eofPosition ? TokenNameEOF : TokenNameRBRACE;
1476 }
1477 int whiteStart = 0;
1478 try {
1479 while (true) { //loop for jumping over comments
1480 this.withoutUnicodePtr = 0;
1481 //start with a new token (even comment written with unicode )
1482
1483 // ---------Consume white space and handles startPosition---------
1484 whiteStart = this.currentPosition;
1485 boolean isWhiteSpace, hasWhiteSpaces = false;
1486 int offset;
1487 int unicodePtr;
1488 boolean checkIfUnicode = false;
1489 do {
1490 unicodePtr = this.withoutUnicodePtr;
1491 offset = this.currentPosition;
1492 this.startPosition = this.currentPosition;
1493 try {
1494 checkIfUnicode = ((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1495 && (this.source[this.currentPosition] == 'u');
1496 } catch(IndexOutOfBoundsException e) {
1497 if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) {
1498 // reposition scanner in case we are interested by spaces as tokens
1499 this.currentPosition--;
1500 this.startPosition = whiteStart;
1501 return TokenNameWHITESPACE;
1502 }
1503 if (this.currentPosition > this.eofPosition)
1504 return TokenNameEOF;
1505 }
1506 if (this.currentPosition > this.eofPosition) {
1507 if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) {
1508 this.currentPosition--;
1509 // reposition scanner in case we are interested by spaces as tokens
1510 this.startPosition = whiteStart;
1511 return TokenNameWHITESPACE;
1512 }
1513 return TokenNameEOF;
1514 }
1515 if (checkIfUnicode) {
1516 isWhiteSpace = jumpOverUnicodeWhiteSpace();
1517 offset = this.currentPosition - offset;
1518 } else {
1519 offset = this.currentPosition - offset;
1520 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1521 if (this.recordLineSeparator) {
1522 pushLineSeparator();
1523 }
1524 }
1525 // inline version of:
1526 //isWhiteSpace =
1527 // (this.currentCharacter == ' ') || ScannerHelper.isWhitespace(this.currentCharacter);
1528 switch (this.currentCharacter) {
1529 case 10 : /* \ u000a: LINE FEED */
1530 case 12 : /* \ u000c: FORM FEED */
1531 case 13 : /* \ u000d: CARRIAGE RETURN */
1532 case 32 : /* \ u0020: SPACE */
1533 case 9 : /* \ u0009: HORIZONTAL TABULATION */
1534 isWhiteSpace = true;
1535 break;
1536 default :
1537 isWhiteSpace = false;
1538 }
1539 }
1540 if (isWhiteSpace) {
1541 hasWhiteSpaces = true;
1542 }
1543 } while (isWhiteSpace);
1544 if (hasWhiteSpaces) {
1545 if (this.tokenizeWhiteSpace) {
1546 // reposition scanner in case we are interested by spaces as tokens
1547 this.currentPosition-=offset;
1548 this.startPosition = whiteStart;
1549 if (checkIfUnicode) {
1550 this.withoutUnicodePtr = unicodePtr;
1551 }
1552 return TokenNameWHITESPACE;
1553 } else if (checkIfUnicode) {
1554 this.withoutUnicodePtr = 0;
1555 unicodeStore();
1556 } else {
1557 this.withoutUnicodePtr = 0;
1558 }
1559 }
1560 // ---------Identify the next token-------------
1561 switch (this.currentCharacter) {
1562 case '@' :
1563 /* if (this.sourceLevel >= ClassFileConstants.JDK1_5) {
1564 return TokenNameAT;
1565 } else {
1566 return TokenNameERROR;
1567 }*/
1568 return TokenNameAT;
1569 case '(' :
1570 return TokenNameLPAREN;
1571 case ')' :
1572 return TokenNameRPAREN;
1573 case '{' :
1574 return TokenNameLBRACE;
1575 case '}' :
1576 return TokenNameRBRACE;
1577 case '[' :
1578 return TokenNameLBRACKET;
1579 case ']' :
1580 return TokenNameRBRACKET;
1581 case ';' :
1582 return TokenNameSEMICOLON;
1583 case ',' :
1584 return TokenNameCOMMA;
1585 case '.' :
1586 if (getNextCharAsDigit()) {
1587 return scanNumber(true);
1588 }
1589 int temp = this.currentPosition;
1590 if (getNextChar('.')) {
1591 if (getNextChar('.')) {
1592 return TokenNameELLIPSIS;
1593 } else {
1594 this.currentPosition = temp;
1595 return TokenNameDOT;
1596 }
1597 } else {
1598 this.currentPosition = temp;
1599 return TokenNameDOT;
1600 }
1601 case '+' :
1602 {
1603 int test;
1604 if ((test = getNextChar('+', '=')) == 0)
1605 return TokenNamePLUS_PLUS;
1606 if (test > 0)
1607 return TokenNamePLUS_EQUAL;
1608 return TokenNamePLUS;
1609 }
1610 case '-' :
1611 {
1612 int test;
1613 if ((test = getNextChar('-', '=')) == 0)
1614 return TokenNameMINUS_MINUS;
1615 if (test > 0)
1616 return TokenNameMINUS_EQUAL;
1617 if (getNextChar('>'))
1618 return TokenNameARROW;
1619 return TokenNameMINUS;
1620 }
1621 case '~' :
1622 return TokenNameTWIDDLE;
1623 case '!' :
1624 if (getNextChar('='))
1625 return TokenNameNOT_EQUAL;
1626 return TokenNameNOT;
1627 case '*' :
1628 if (getNextChar('='))
1629 return TokenNameMULTIPLY_EQUAL;
1630 return TokenNameMULTIPLY;
1631 case '%' :
1632 if (getNextChar('='))
1633 return TokenNameREMAINDER_EQUAL;
1634 return TokenNameREMAINDER;
1635 case '<' :
1636 {
1637 int test;
1638 if ((test = getNextChar('=', '<')) == 0)
1639 return TokenNameLESS_EQUAL;
1640 if (test > 0) {
1641 if (getNextChar('='))
1642 return TokenNameLEFT_SHIFT_EQUAL;
1643 return TokenNameLEFT_SHIFT;
1644 }
1645 return TokenNameLESS;
1646 }
1647 case '>' :
1648 {
1649 int test;
1650 if (this.returnOnlyGreater) {
1651 return TokenNameGREATER;
1652 }
1653 if ((test = getNextChar('=', '>')) == 0)
1654 return TokenNameGREATER_EQUAL;
1655 if (test > 0) {
1656 if ((test = getNextChar('=', '>')) == 0)
1657 return TokenNameRIGHT_SHIFT_EQUAL;
1658 if (test > 0) {
1659 if (getNextChar('='))
1660 return TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL;
1661 return TokenNameUNSIGNED_RIGHT_SHIFT;
1662 }
1663 return TokenNameRIGHT_SHIFT;
1664 }
1665 return TokenNameGREATER;
1666 }
1667 case '=' :
1668 if (getNextChar('='))
1669 return TokenNameEQUAL_EQUAL;
1670 return TokenNameEQUAL;
1671 case '&' :
1672 {
1673 int test;
1674 if ((test = getNextChar('&', '=')) == 0)
1675 return TokenNameAND_AND;
1676 if (test > 0)
1677 return TokenNameAND_EQUAL;
1678 return TokenNameAND;
1679 }
1680 case '|' :
1681 {
1682 int test;
1683 if ((test = getNextChar('|', '=')) == 0)
1684 return TokenNameOR_OR;
1685 if (test > 0)
1686 return TokenNameOR_EQUAL;
1687 return TokenNameOR;
1688 }
1689 case '^' :
1690 if (getNextChar('='))
1691 return TokenNameXOR_EQUAL;
1692 return TokenNameXOR;
1693 case '?' :
1694 return TokenNameQUESTION;
1695 case ':' :
1696 if (getNextChar(':'))
1697 return TokenNameCOLON_COLON;
1698 ++this.yieldColons;
1699 return TokenNameCOLON;
1700 case '\'' :
1701 {
1702 int test;
1703 if ((test = getNextChar('\n', '\r')) == 0) {
1704 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1705 }
1706 if (test > 0) {
1707 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1708 for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
1709 if (this.currentPosition + lookAhead == this.eofPosition)
1710 break;
1711 if (this.source[this.currentPosition + lookAhead] == '\n')
1712 break;
1713 if (this.source[this.currentPosition + lookAhead] == '\'') {
1714 this.currentPosition += lookAhead + 1;
1715 break;
1716 }
1717 }
1718 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1719 }
1720 }
1721 if (getNextChar('\'')) {
1722 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1723 for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
1724 if (this.currentPosition + lookAhead == this.eofPosition)
1725 break;
1726 if (this.source[this.currentPosition + lookAhead] == '\n')
1727 break;
1728 if (this.source[this.currentPosition + lookAhead] == '\'') {
1729 this.currentPosition += lookAhead + 1;
1730 break;
1731 }
1732 }
1733 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1734 }
1735 if (getNextChar('\\')) {
1736 if (this.unicodeAsBackSlash) {
1737 // consume next character
1738 this.unicodeAsBackSlash = false;
1739 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) {
1740 getNextUnicodeChar();
1741 } else {
1742 if (this.withoutUnicodePtr != 0) {
1743 unicodeStore();
1744 }
1745 }
1746 } else {
1747 this.currentCharacter = this.source[this.currentPosition++];
1748 }
1749 scanEscapeCharacter();
1750 } else { // consume next character
1751 this.unicodeAsBackSlash = false;
1752 checkIfUnicode = false;
1753 try {
1754 checkIfUnicode = ((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1755 && (this.source[this.currentPosition] == 'u');
1756 } catch(IndexOutOfBoundsException e) {
1757 this.currentPosition--;
1758 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1759 }
1760 if (checkIfUnicode) {
1761 getNextUnicodeChar();
1762 } else {
1763 if (this.withoutUnicodePtr != 0) {
1764 unicodeStore();
1765 }
1766 }
1767 }
1768 if (getNextChar('\''))
1769 return TokenNameCharacterLiteral;
1770 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1771 for (int lookAhead = 0; lookAhead < 20; lookAhead++) {
1772 if (this.currentPosition + lookAhead == this.eofPosition)
1773 break;
1774 if (this.source[this.currentPosition + lookAhead] == '\n')
1775 break;
1776 if (this.source[this.currentPosition + lookAhead] == '\'') {
1777 this.currentPosition += lookAhead + 1;
1778 break;
1779 }
1780 }
1781 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1782 case '"' :
1783 return scanForStringLiteral();
1784 case '/' :
1785 if (!this.skipComments) {
1786 int test = getNextChar('/', '*');
1787 if (test == 0) { //line comment
1788 this.lastCommentLinePosition = this.currentPosition;
1789 try { //get the next char
1790 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1791 && (this.source[this.currentPosition] == 'u')) {
1792 getNextUnicodeChar();
1793 }
1794
1795 //handle the \\u case manually into comment
1796 if (this.currentCharacter == '\\') {
1797 if (this.source[this.currentPosition] == '\\')
1798 this.currentPosition++;
1799 } //jump over the \\
1800 boolean isUnicode = false;
1801 while (this.currentCharacter != '\r' && this.currentCharacter != '\n') {
1802 if (this.currentPosition >= this.eofPosition) {
1803 this.lastCommentLinePosition = this.currentPosition;
1804 this.currentPosition ++;
1805 // this avoids duplicating the code in the catch(IndexOutOfBoundsException e)
1806 throw new IndexOutOfBoundsException();
1807 }
1808 this.lastCommentLinePosition = this.currentPosition;
1809 //get the next char
1810 isUnicode = false;
1811 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1812 && (this.source[this.currentPosition] == 'u')) {
1813 getNextUnicodeChar();
1814 isUnicode = true;
1815 }
1816 //handle the \\u case manually into comment
1817 if (this.currentCharacter == '\\') {
1818 if (this.source[this.currentPosition] == '\\')
1819 this.currentPosition++;
1820 } //jump over the \\
1821 }
1822 /*
1823 * We need to completely consume the line break
1824 */
1825 if (this.currentCharacter == '\r'
1826 && this.eofPosition > this.currentPosition) {
1827 if (this.source[this.currentPosition] == '\n') {
1828 this.currentPosition++;
1829 this.currentCharacter = '\n';
1830 } else if ((this.source[this.currentPosition] == '\\')
1831 && (this.source[this.currentPosition + 1] == 'u')) {
1832 getNextUnicodeChar();
1833 isUnicode = true;
1834 }
1835 }
1836 recordComment(TokenNameCOMMENT_LINE);
1837 if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
1838 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1839 if ((this.checkNonExternalizedStringLiterals || this.checkUninternedIdentityComparison) &&
1840 this.lastPosition < this.currentPosition) {
1841 parseTags();
1842 }
1843 if (this.recordLineSeparator) {
1844 if (isUnicode) {
1845 pushUnicodeLineSeparator();
1846 } else {
1847 pushLineSeparator();
1848 }
1849 }
1850 }
1851 if (this.tokenizeComments) {
1852 return TokenNameCOMMENT_LINE;
1853 }
1854 } catch (IndexOutOfBoundsException e) {
1855 this.currentPosition--;
1856 recordComment(TokenNameCOMMENT_LINE);
1857 if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
1858 if ((this.checkNonExternalizedStringLiterals || this.checkUninternedIdentityComparison) &&
1859 this.lastPosition < this.currentPosition) {
1860 parseTags();
1861 }
1862 if (this.tokenizeComments) {
1863 return TokenNameCOMMENT_LINE;
1864 } else {
1865 this.currentPosition++;
1866 }
1867 }
1868 break;
1869 }
1870 if (test > 0) { //traditional and javadoc comment
1871 try { //get the next char
1872 boolean isJavadoc = false, star = false;
1873 boolean isUnicode = false;
1874 int previous;
1875 // consume next character
1876 this.unicodeAsBackSlash = false;
1877 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1878 && (this.source[this.currentPosition] == 'u')) {
1879 getNextUnicodeChar();
1880 isUnicode = true;
1881 } else {
1882 isUnicode = false;
1883 if (this.withoutUnicodePtr != 0) {
1884 unicodeStore();
1885 }
1886 }
1887
1888 if (this.currentCharacter == '*') {
1889 isJavadoc = true;
1890 star = true;
1891 }
1892 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1893 if (this.recordLineSeparator) {
1894 if (isUnicode) {
1895 pushUnicodeLineSeparator();
1896 } else {
1897 pushLineSeparator();
1898 }
1899 }
1900 }
1901 isUnicode = false;
1902 previous = this.currentPosition;
1903 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1904 && (this.source[this.currentPosition] == 'u')) {
1905 //-------------unicode traitement ------------
1906 getNextUnicodeChar();
1907 isUnicode = true;
1908 } else {
1909 isUnicode = false;
1910 }
1911 //handle the \\u case manually into comment
1912 if (this.currentCharacter == '\\') {
1913 if (this.source[this.currentPosition] == '\\')
1914 this.currentPosition++; //jump over the \\
1915 }
1916 // empty comment is not a javadoc /**/
1917 if (this.currentCharacter == '/') {
1918 isJavadoc = false;
1919 }
1920 //loop until end of comment */
1921 int firstTag = 0;
1922 while ((this.currentCharacter != '/') || (!star)) {
1923 if (this.currentPosition >= this.eofPosition) {
1924 throw new InvalidInputException(UNTERMINATED_COMMENT);
1925 }
1926 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
1927 if (this.recordLineSeparator) {
1928 if (isUnicode) {
1929 pushUnicodeLineSeparator();
1930 } else {
1931 pushLineSeparator();
1932 }
1933 }
1934 }
1935 switch (this.currentCharacter) {
1936 case '*':
1937 star = true;
1938 break;
1939 case '@':
1940 if (firstTag == 0 && this.isFirstTag()) {
1941 firstTag = previous;
1942 }
1943 //$FALL-THROUGH$ default case to set star to false
1944 default:
1945 star = false;
1946 }
1947 //get next char
1948 previous = this.currentPosition;
1949 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
1950 && (this.source[this.currentPosition] == 'u')) {
1951 //-------------unicode traitement ------------
1952 getNextUnicodeChar();
1953 isUnicode = true;
1954 } else {
1955 isUnicode = false;
1956 }
1957 //handle the \\u case manually into comment
1958 if (this.currentCharacter == '\\') {
1959 if (this.source[this.currentPosition] == '\\')
1960 this.currentPosition++;
1961 } //jump over the \\
1962 }
1963 int token = isJavadoc ? TokenNameCOMMENT_JAVADOC : TokenNameCOMMENT_BLOCK;
1964 recordComment(token);
1965 this.commentTagStarts[this.commentPtr] = firstTag;
1966 if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
1967 if (this.tokenizeComments) {
1968 /*
1969 if (isJavadoc)
1970 return TokenNameCOMMENT_JAVADOC;
1971 return TokenNameCOMMENT_BLOCK;
1972 */
1973 return token;
1974 }
1975 } catch (IndexOutOfBoundsException e) {
1976 this.currentPosition--;
1977 throw new InvalidInputException(UNTERMINATED_COMMENT);
1978 }
1979 break;
1980 }
1981 }
1982 if (getNextChar('='))
1983 return TokenNameDIVIDE_EQUAL;
1984 return TokenNameDIVIDE;
1985 case '\u001a' :
1986 if (atEnd())
1987 return TokenNameEOF;
1988 //the atEnd may not be <currentPosition == source.length> if source is only some part of a real (external) stream
1989 throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
1990 default :
1991 char c = this.currentCharacter;
1992 if (c < ScannerHelper.MAX_OBVIOUS) {
1993 if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_IDENT_START) != 0) {
1994 return scanIdentifierOrKeyword();
1995 } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_DIGIT) != 0) {
1996 return scanNumber(false);
1997 } else {
1998 return TokenNameERROR;
1999 }
2000 }
2001 boolean isJavaIdStart;
2002 if (c >= HIGH_SURROGATE_MIN_VALUE && c <= HIGH_SURROGATE_MAX_VALUE) {
2003 if (this.complianceLevel < ClassFileConstants.JDK1_5) {
2004 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2005 }
2006 // Unicode 4 detection
2007 char low = (char) getNextChar();
2008 if (low < LOW_SURROGATE_MIN_VALUE || low > LOW_SURROGATE_MAX_VALUE) {
2009 // illegal low surrogate
2010 throw new InvalidInputException(INVALID_LOW_SURROGATE);
2011 }
2012 isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c, low);
2013 }
2014 else if (c >= LOW_SURROGATE_MIN_VALUE && c <= LOW_SURROGATE_MAX_VALUE) {
2015 if (this.complianceLevel < ClassFileConstants.JDK1_5) {
2016 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2017 }
2018 throw new InvalidInputException(INVALID_HIGH_SURROGATE);
2019 } else {
2020 // optimized case already checked
2021 isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c);
2022 }
2023 if (isJavaIdStart)
2024 return scanIdentifierOrKeyword();
2025 if (ScannerHelper.isDigit(this.currentCharacter)) {
2026 return scanNumber(false);
2027 }
2028 return TokenNameERROR;
2029 }
2030 }
2031 } //-----------------end switch while try--------------------
2032 catch (IndexOutOfBoundsException e) {
2033 if (this.tokenizeWhiteSpace && (whiteStart != this.currentPosition - 1)) {
2034 // reposition scanner in case we are interested by spaces as tokens
2035 this.currentPosition--;
2036 this.startPosition = whiteStart;
2037 return TokenNameWHITESPACE;
2038 }
2039 }
2040 return TokenNameEOF;
2041 }
2042 private int scanForStringLiteral() throws InvalidInputException {
2043 boolean isTextBlock = false;
2044 int lastQuotePos = 0;
2045
2046 // consume next character
2047 this.unicodeAsBackSlash = false;
2048 boolean isUnicode = false;
2049 isTextBlock = scanForTextBlockBeginning();
2050 if (isTextBlock) {
2051 try {
2052 this.rawStart = this.currentPosition - this.startPosition;
2053 int terminators = 0;
2054 while (this.currentPosition <= this.eofPosition) {
2055 if (this.currentCharacter == '"') {
2056 lastQuotePos = this.currentPosition;
2057 // look for text block delimiter
2058 if (scanForTextBlockClose()) {
2059 // Account for just the snippet being passed around
2060 // If already at the EOF, bail out.
2061 if (this.currentPosition + 2 < this.source.length && this.source[this.currentPosition + 2] == '"') {
2062 terminators++;
2063 if (terminators > 2)
2064 throw new InvalidInputException(UNTERMINATED_TEXT_BLOCK);
2065 } else {
2066 this.currentPosition += 2;
2067 return TerminalTokens.TokenNameTextBlock;
2068 }
2069 }
2070 if (this.withoutUnicodePtr != 0) {
2071 unicodeStore();
2072 }
2073 } else {
2074 terminators = 0;
2075 }
2076 outer: if (this.currentCharacter == '\\') {
2077 switch(this.source[this.currentPosition]) {
2078 case 'n' :
2079 case 'r' :
2080 case 'f' :
2081 break outer;
2082 case '\n' :
2083 case '\r' :
2084 this.currentCharacter = '\\';
2085 this.currentPosition++;
2086 break;
2087 case '\\' :
2088 this.currentPosition++;
2089 break;
2090 default :
2091 if (this.unicodeAsBackSlash) {
2092 this.withoutUnicodePtr--;
2093 // consume next character
2094 if (this.currentPosition >= this.eofPosition) {
2095 break;
2096 }
2097 this.unicodeAsBackSlash = false;
2098 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2099 && (this.source[this.currentPosition] == 'u')) {
2100 getNextUnicodeChar();
2101 isUnicode = true;
2102 this.withoutUnicodePtr--;
2103 } else {
2104 isUnicode = false;
2105 }
2106 } else {
2107 if (this.withoutUnicodePtr == 0) {
2108 unicodeInitializeBuffer(this.currentPosition - this.startPosition);
2109 }
2110 this.withoutUnicodePtr --;
2111 this.currentCharacter = this.source[this.currentPosition++];
2112 }
2113 int oldPos = this.currentPosition - 1;
2114 scanEscapeCharacter();
2115 switch (this.currentCharacter) {
2116 case ' ':
2117 if (this.withoutUnicodePtr == 0) {
2118 unicodeInitializeBuffer(this.currentPosition - this.startPosition);
2119 }
2120 // Kludge, retain the '\' and also
2121 // when scanEscapeCharacter reads space in form of \040 and
2122 // set the next character to 's'
2123 // so, we get an escaped scape, i.e. \s, which will later be
2124 // replaced by space
2125 unicodeStore('\\');
2126 this.currentCharacter = 's';
2127 break;
2128 case '\r':
2129 case '\n':
2130 if (this.withoutUnicodePtr == 0) {
2131 unicodeInitializeBuffer(this.currentPosition - this.startPosition);
2132 }
2133 unicodeStore('\\');
2134 this.currentPosition = oldPos;
2135 this.currentCharacter = this.source[this.currentPosition];
2136 break outer;
2137
2138 }
2139 }
2140 if (this.withoutUnicodePtr != 0) {
2141 unicodeStore();
2142 }
2143 }
2144 // consume next character
2145 this.unicodeAsBackSlash = false;
2146 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2147 && (this.source[this.currentPosition] == 'u')) {
2148 getNextUnicodeChar();
2149 isUnicode = true;
2150 } else {
2151 isUnicode = false;
2152 if (this.currentCharacter == '"'/* || skipWhitespace*/)
2153 continue;
2154 if (this.withoutUnicodePtr != 0) {
2155 unicodeStore();
2156 }
2157 }
2158 }
2159 if (lastQuotePos > 0)
2160 this.currentPosition = lastQuotePos;
2161 this.currentPosition = (lastQuotePos > 0) ? lastQuotePos : this.startPosition + this.rawStart;
2162 throw new InvalidInputException(UNTERMINATED_TEXT_BLOCK);
2163 } catch (IndexOutOfBoundsException e) {
2164 this.currentPosition = (lastQuotePos > 0) ? lastQuotePos : this.startPosition + this.rawStart;
2165 throw new InvalidInputException(UNTERMINATED_TEXT_BLOCK);
2166 }
2167 } else {
2168 try {
2169 // consume next character
2170 this.unicodeAsBackSlash = false;
2171 isUnicode = false;
2172 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2173 && (this.source[this.currentPosition] == 'u')) {
2174 getNextUnicodeChar();
2175 isUnicode = true;
2176 } else {
2177 if (this.withoutUnicodePtr != 0) {
2178 unicodeStore();
2179 }
2180 }
2181
2182 while (this.currentCharacter != '"') {
2183 if (this.currentPosition >= this.eofPosition) {
2184 throw new InvalidInputException(UNTERMINATED_STRING);
2185 }
2186 /**** \r and \n are not valid in string literals ****/
2187 if ((this.currentCharacter == '\n') || (this.currentCharacter == '\r')) {
2188 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
2189 if (isUnicode) {
2190 int start = this.currentPosition;
2191 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
2192 if (this.currentPosition >= this.eofPosition) {
2193 this.currentPosition = start;
2194 break;
2195 }
2196 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) {
2197 isUnicode = true;
2198 getNextUnicodeChar();
2199 } else {
2200 isUnicode = false;
2201 }
2202 if (!isUnicode && this.currentCharacter == '\n') {
2203 this.currentPosition--; // set current position on new line character
2204 break;
2205 }
2206 if (this.currentCharacter == '\"') {
2207 throw new InvalidInputException(INVALID_CHAR_IN_STRING);
2208 }
2209 }
2210 } else {
2211 this.currentPosition--; // set current position on new line character
2212 }
2213 throw new InvalidInputException(INVALID_CHAR_IN_STRING);
2214 }
2215 if (this.currentCharacter == '\\') {
2216 if (this.unicodeAsBackSlash) {
2217 this.withoutUnicodePtr--;
2218 // consume next character
2219 this.unicodeAsBackSlash = false;
2220 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) {
2221 getNextUnicodeChar();
2222 isUnicode = true;
2223 this.withoutUnicodePtr--;
2224 } else {
2225 isUnicode = false;
2226 }
2227 } else {
2228 if (this.withoutUnicodePtr == 0) {
2229 unicodeInitializeBuffer(this.currentPosition - this.startPosition);
2230 }
2231 this.withoutUnicodePtr --;
2232 this.currentCharacter = this.source[this.currentPosition++];
2233 }
2234 // we need to compute the escape character in a separate buffer
2235 scanEscapeCharacter();
2236 if (this.withoutUnicodePtr != 0) {
2237 unicodeStore();
2238 }
2239 }
2240 // consume next character
2241 this.unicodeAsBackSlash = false;
2242 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2243 && (this.source[this.currentPosition] == 'u')) {
2244 getNextUnicodeChar();
2245 isUnicode = true;
2246 } else {
2247 isUnicode = false;
2248 if (this.withoutUnicodePtr != 0) {
2249 unicodeStore();
2250 }
2251 }
2252
2253 }
2254 } catch (IndexOutOfBoundsException e) {
2255 this.currentPosition--;
2256 throw new InvalidInputException(UNTERMINATED_STRING);
2257 } catch (InvalidInputException e) {
2258 if (e.getMessage().equals(INVALID_ESCAPE)) {
2259 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
2260 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
2261 if (this.currentPosition + lookAhead == this.eofPosition)
2262 break;
2263 if (this.source[this.currentPosition + lookAhead] == '\n')
2264 break;
2265 if (this.source[this.currentPosition + lookAhead] == '\"') {
2266 this.currentPosition += lookAhead + 1;
2267 break;
2268 }
2269 }
2270
2271 }
2272 throw e; // rethrow
2273 }
2274 return TokenNameStringLiteral;
2275 }
2276 }
2277 public void getNextUnicodeChar()
2278 throws InvalidInputException {
2279 //VOID
2280 //handle the case of unicode.
2281 //when a unicode appears then we must use a buffer that holds char internal values
2282 //At the end of this method currentCharacter holds the new visited char
2283 //and currentPosition points right next after it
2284
2285 //ALL getNextChar.... ARE OPTIMIZED COPIES
2286 int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
2287 this.currentPosition++;
2288 if (this.currentPosition < this.eofPosition) {
2289 while (this.source[this.currentPosition] == 'u') {
2290 this.currentPosition++;
2291 if (this.currentPosition >= this.eofPosition) {
2292 this.currentPosition--;
2293 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2294 }
2295 unicodeSize++;
2296 }
2297 } else {
2298 this.currentPosition--;
2299 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2300 }
2301
2302 if ((this.currentPosition + 4) > this.eofPosition) {
2303 this.currentPosition += (this.eofPosition - this.currentPosition);
2304 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2305 }
2306 if ((c1 = ScannerHelper.getHexadecimalValue(this.source[this.currentPosition++])) > 15
2307 || c1 < 0
2308 || (c2 = ScannerHelper.getHexadecimalValue(this.source[this.currentPosition++])) > 15
2309 || c2 < 0
2310 || (c3 = ScannerHelper.getHexadecimalValue(this.source[this.currentPosition++])) > 15
2311 || c3 < 0
2312 || (c4 = ScannerHelper.getHexadecimalValue(this.source[this.currentPosition++])) > 15
2313 || c4 < 0){
2314 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2315 }
2316 this.currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2317 //need the unicode buffer
2318 if (this.withoutUnicodePtr == 0) {
2319 //buffer all the entries that have been left aside....
2320 unicodeInitializeBuffer(this.currentPosition - unicodeSize - this.startPosition);
2321 }
2322 //fill the buffer with the char
2323 unicodeStore();
2324 this.unicodeAsBackSlash = this.currentCharacter == '\\';
2325 }
2326 public NLSTag[] getNLSTags() {
2327 final int length = this.nlsTagsPtr;
2328 if (length != 0) {
2329 NLSTag[] result = new NLSTag[length];
2330 System.arraycopy(this.nlsTags, 0, result, 0, length);
2331 this.nlsTagsPtr = 0;
2332 return result;
2333 }
2334 return null;
2335 }
2336 public boolean[] getIdentityComparisonLines() {
2337 boolean [] retVal = this.validIdentityComparisonLines;
2338 this.validIdentityComparisonLines = null;
2339 return retVal;
2340 }
2341 public char[] getSource(){
2342 return this.source;
2343 }
2344 protected boolean isFirstTag() {
2345 return true;
2346 }
2347 public final void jumpOverMethodBody() {
2348
2349 this.wasAcr = false;
2350 int found = 1;
2351 try {
2352 while (true) { //loop for jumping over comments
2353 this.withoutUnicodePtr = 0;
2354 // ---------Consume white space and handles startPosition---------
2355 boolean isWhiteSpace;
2356 do {
2357 this.startPosition = this.currentPosition;
2358 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2359 && (this.source[this.currentPosition] == 'u')) {
2360 isWhiteSpace = jumpOverUnicodeWhiteSpace();
2361 } else {
2362 if (this.recordLineSeparator
2363 && ((this.currentCharacter == '\r') || (this.currentCharacter == '\n'))) {
2364 pushLineSeparator();
2365 }
2366 isWhiteSpace = CharOperation.isWhitespace(this.currentCharacter);
2367 }
2368 } while (isWhiteSpace);
2369
2370 // -------consume token until } is found---------
2371 NextToken: switch (this.currentCharacter) {
2372 case '{' :
2373 found++;
2374 break NextToken;
2375 case '}' :
2376 found--;
2377 if (found == 0)
2378 return;
2379 break NextToken;
2380 case '\'' :
2381 {
2382 boolean test;
2383 test = getNextChar('\\');
2384 if (test) {
2385 try {
2386 if (this.unicodeAsBackSlash) {
2387 // consume next character
2388 this.unicodeAsBackSlash = false;
2389 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) {
2390 getNextUnicodeChar();
2391 } else {
2392 if (this.withoutUnicodePtr != 0) {
2393 unicodeStore();
2394 }
2395 }
2396 } else {
2397 this.currentCharacter = this.source[this.currentPosition++];
2398 }
2399 scanEscapeCharacter();
2400 } catch (InvalidInputException ex) {
2401 // ignore
2402 }
2403 } else {
2404 try { // consume next character
2405 this.unicodeAsBackSlash = false;
2406 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2407 && (this.source[this.currentPosition] == 'u')) {
2408 getNextUnicodeChar();
2409 } else {
2410 if (this.withoutUnicodePtr != 0) {
2411 unicodeStore();
2412 }
2413 }
2414 } catch (InvalidInputException ex) {
2415 // ignore
2416 }
2417 }
2418 getNextChar('\'');
2419 break NextToken;
2420 }
2421 case '"' :
2422 boolean isTextBlock = false;
2423 int firstClosingBrace = 0;
2424 try {
2425 try { // consume next character
2426 isTextBlock = scanForTextBlockBeginning();
2427 if (!isTextBlock) {
2428 this.unicodeAsBackSlash = false;
2429 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2430 && (this.source[this.currentPosition] == 'u')) {
2431 getNextUnicodeChar();
2432 } else {
2433 if (this.withoutUnicodePtr != 0) {
2434 unicodeStore();
2435 }
2436 }
2437 }
2438 } catch (InvalidInputException ex) {
2439 // ignore
2440 }
2441
2442 Inner: while (this.currentPosition <= this.eofPosition) {
2443 if (isTextBlock) {
2444 switch (this.currentCharacter) {
2445 case '"':
2446 // look for text block delimiter
2447 if (scanForTextBlockClose()) {
2448 this.currentPosition += 2;
2449 this.currentCharacter = this.source[this.currentPosition];
2450 isTextBlock = false;
2451 break Inner;
2452 }
2453 break;
2454 case '}':
2455 if (firstClosingBrace == 0)
2456 firstClosingBrace = this.currentPosition;
2457 break;
2458 case '\r' :
2459 if (this.source[this.currentPosition] == '\n')
2460 this.currentPosition++;
2461 //$FALL-THROUGH$
2462 case '\n' :
2463 pushLineSeparator();
2464 //$FALL-THROUGH$
2465 default:
2466 if (this.currentCharacter == '\\' && this.source[this.currentPosition++] == '"') {
2467 this.currentPosition++;
2468 }
2469 this.currentCharacter = this.source[this.currentPosition++];
2470 continue Inner;
2471 }
2472 } else if (this.currentCharacter == '"') {
2473 break Inner;
2474 }
2475 if (this.currentCharacter == '\r'){
2476 if (this.source[this.currentPosition] == '\n') this.currentPosition++;
2477 break NextToken; // the string cannot go further that the line
2478 }
2479 if (this.currentCharacter == '\n'){
2480 break; // the string cannot go further that the line
2481 }
2482 if (this.currentCharacter == '\\') {
2483 try {
2484 if (this.unicodeAsBackSlash) {
2485 // consume next character
2486 this.unicodeAsBackSlash = false;
2487 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\') && (this.source[this.currentPosition] == 'u')) {
2488 getNextUnicodeChar();
2489 } else {
2490 if (this.withoutUnicodePtr != 0) {
2491 unicodeStore();
2492 }
2493 }
2494 } else {
2495 this.currentCharacter = this.source[this.currentPosition++];
2496 }
2497 scanEscapeCharacter();
2498 } catch (InvalidInputException ex) {
2499 // ignore
2500 }
2501 }
2502 try { // consume next character
2503 this.unicodeAsBackSlash = false;
2504 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2505 && (this.source[this.currentPosition] == 'u')) {
2506 getNextUnicodeChar();
2507 } else {
2508 if (this.withoutUnicodePtr != 0) {
2509 unicodeStore();
2510 }
2511 }
2512 } catch (InvalidInputException ex) {
2513 // ignore
2514 }
2515 }
2516 } catch (IndexOutOfBoundsException e) {
2517 if(isTextBlock) {
2518 // Pull it back to the first closing brace after the beginning
2519 // of the unclosed text block and let recovery take over.
2520 if (firstClosingBrace > 0) {
2521 this.currentPosition = firstClosingBrace - 1;
2522 }
2523 }
2524 }
2525 break NextToken;
2526 case '/' :
2527 {
2528 int test;
2529 if ((test = getNextChar('/', '*')) == 0) { //line comment
2530 try {
2531 this.lastCommentLinePosition = this.currentPosition;
2532 //get the next char
2533 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2534 && (this.source[this.currentPosition] == 'u')) {
2535 getNextUnicodeChar();
2536 }
2537 //handle the \\u case manually into comment
2538 if (this.currentCharacter == '\\') {
2539 if (this.source[this.currentPosition] == '\\')
2540 this.currentPosition++;
2541 } //jump over the \\
2542 boolean isUnicode = false;
2543 while (this.currentCharacter != '\r' && this.currentCharacter != '\n') {
2544 if (this.currentPosition >= this.eofPosition) {
2545 this.lastCommentLinePosition = this.currentPosition;
2546 this.currentPosition ++;
2547 // this avoids duplicating the code inside the catch(IndexOutOfBoundsException e) below
2548 throw new IndexOutOfBoundsException();
2549 }
2550 this.lastCommentLinePosition = this.currentPosition;
2551 //get the next char
2552 isUnicode = false;
2553 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2554 && (this.source[this.currentPosition] == 'u')) {
2555 isUnicode = true;
2556 getNextUnicodeChar();
2557 }
2558 //handle the \\u case manually into comment
2559 if (this.currentCharacter == '\\') {
2560 if (this.source[this.currentPosition] == '\\')
2561 this.currentPosition++;
2562 } //jump over the \\
2563 }
2564 /*
2565 * We need to completely consume the line break
2566 */
2567 if (this.currentCharacter == '\r'
2568 && this.eofPosition > this.currentPosition) {
2569 if (this.source[this.currentPosition] == '\n') {
2570 this.currentPosition++;
2571 this.currentCharacter = '\n';
2572 } else if ((this.source[this.currentPosition] == '\\')
2573 && (this.source[this.currentPosition + 1] == 'u')) {
2574 isUnicode = true;
2575 getNextUnicodeChar();
2576 }
2577 }
2578 recordComment(TokenNameCOMMENT_LINE);
2579 if (this.recordLineSeparator
2580 && ((this.currentCharacter == '\r') || (this.currentCharacter == '\n'))) {
2581 if ((this.checkNonExternalizedStringLiterals || this.checkUninternedIdentityComparison) &&
2582 this.lastPosition < this.currentPosition) {
2583 parseTags();
2584 }
2585 if (this.recordLineSeparator) {
2586 if (isUnicode) {
2587 pushUnicodeLineSeparator();
2588 } else {
2589 pushLineSeparator();
2590 }
2591 }
2592 }
2593 } catch (IndexOutOfBoundsException e) {
2594 //an eof will then be generated
2595 this.currentPosition--;
2596 recordComment(TokenNameCOMMENT_LINE);
2597 if ((this.checkNonExternalizedStringLiterals || this.checkUninternedIdentityComparison) &&
2598 this.lastPosition < this.currentPosition) {
2599 parseTags();
2600 }
2601 if (!this.tokenizeComments) {
2602 this.currentPosition++;
2603 }
2604 }
2605 break NextToken;
2606 }
2607 if (test > 0) { //traditional and javadoc comment
2608 boolean isJavadoc = false;
2609 try { //get the next char
2610 boolean star = false;
2611 int previous;
2612 boolean isUnicode = false;
2613 // consume next character
2614 this.unicodeAsBackSlash = false;
2615 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2616 && (this.source[this.currentPosition] == 'u')) {
2617 getNextUnicodeChar();
2618 isUnicode = true;
2619 } else {
2620 isUnicode = false;
2621 if (this.withoutUnicodePtr != 0) {
2622 unicodeStore();
2623 }
2624 }
2625
2626 if (this.currentCharacter == '*') {
2627 isJavadoc = true;
2628 star = true;
2629 }
2630 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
2631 if (this.recordLineSeparator) {
2632 if (isUnicode) {
2633 pushUnicodeLineSeparator();
2634 } else {
2635 pushLineSeparator();
2636 }
2637 }
2638 }
2639 isUnicode = false;
2640 previous = this.currentPosition;
2641 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2642 && (this.source[this.currentPosition] == 'u')) {
2643 getNextUnicodeChar();
2644 isUnicode = true;
2645 } else {
2646 isUnicode = false;
2647 }
2648 //handle the \\u case manually into comment
2649 if (this.currentCharacter == '\\') {
2650 if (this.source[this.currentPosition] == '\\')
2651 this.currentPosition++; //jump over the \\
2652 }
2653 // empty comment is not a javadoc /**/
2654 if (this.currentCharacter == '/') {
2655 isJavadoc = false;
2656 }
2657 //loop until end of comment */
2658 int firstTag = 0;
2659 while ((this.currentCharacter != '/') || (!star)) {
2660 if (this.currentPosition >= this.eofPosition) {
2661 return;
2662 }
2663 if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
2664 if (this.recordLineSeparator) {
2665 if (isUnicode) {
2666 pushUnicodeLineSeparator();
2667 } else {
2668 pushLineSeparator();
2669 }
2670 }
2671 }
2672 switch (this.currentCharacter) {
2673 case '*':
2674 star = true;
2675 break;
2676 case '@':
2677 if (firstTag == 0 && this.isFirstTag()) {
2678 firstTag = previous;
2679 }
2680 //$FALL-THROUGH$ default case to set star to false
2681 default:
2682 star = false;
2683 }
2684 //get next char
2685 previous = this.currentPosition;
2686 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
2687 && (this.source[this.currentPosition] == 'u')) {
2688 getNextUnicodeChar();
2689 isUnicode = true;
2690 } else {
2691 isUnicode = false;
2692 }
2693 //handle the \\u case manually into comment
2694 if (this.currentCharacter == '\\') {
2695 if (this.source[this.currentPosition] == '\\')
2696 this.currentPosition++;
2697 } //jump over the \\
2698 }
2699 recordComment(isJavadoc ? TokenNameCOMMENT_JAVADOC : TokenNameCOMMENT_BLOCK);
2700 this.commentTagStarts[this.commentPtr] = firstTag;
2701 } catch (IndexOutOfBoundsException e) {
2702 return;
2703 }
2704 break NextToken;
2705 }
2706 break NextToken;
2707 }
2708
2709 default :
2710 try {
2711 char c = this.currentCharacter;
2712 if (c < ScannerHelper.MAX_OBVIOUS) {
2713 if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_IDENT_START) != 0) {
2714 scanIdentifierOrKeyword();
2715 break NextToken;
2716 } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & ScannerHelper.C_DIGIT) != 0) {
2717 scanNumber(false);
2718 break NextToken;
2719 } else {
2720 break NextToken;
2721 }
2722 }
2723 boolean isJavaIdStart;
2724 if (c >= HIGH_SURROGATE_MIN_VALUE && c <= HIGH_SURROGATE_MAX_VALUE) {
2725 if (this.complianceLevel < ClassFileConstants.JDK1_5) {
2726 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2727 }
2728 // Unicode 4 detection
2729 char low = (char) getNextChar();
2730 if (low < LOW_SURROGATE_MIN_VALUE || low > LOW_SURROGATE_MAX_VALUE) {
2731 // illegal low surrogate
2732 break NextToken;
2733 }
2734 isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c, low);
2735 } else if (c >= LOW_SURROGATE_MIN_VALUE && c <= LOW_SURROGATE_MAX_VALUE) {
2736 break NextToken;
2737 } else {
2738 // optimized case already checked
2739 isJavaIdStart = ScannerHelper.isJavaIdentifierStart(this.complianceLevel, c);
2740 }
2741 if (isJavaIdStart) {
2742 scanIdentifierOrKeyword();
2743 break NextToken;
2744 }
2745 // if (ScannerHelper.isDigit(this.currentCharacter)) {
2746 // scanNumber(false);
2747 // break NextToken;
2748 // }
2749 } catch (InvalidInputException ex) {
2750 // ignore
2751 }
2752 }
2753 }
2754 //-----------------end switch while try--------------------
2755 } catch (IndexOutOfBoundsException | InvalidInputException e) {
2756 // ignore
2757 }
2758 return;
2759 }
2760 public final boolean jumpOverUnicodeWhiteSpace() throws InvalidInputException {
2761 //BOOLEAN
2762 //handle the case of unicode. Jump over the next whiteSpace
2763 //making startPosition pointing on the next available char
2764 //On false, the currentCharacter is filled up with a potential
2765 //correct char
2766
2767 this.wasAcr = false;
2768 getNextUnicodeChar();
2769 return CharOperation.isWhitespace(this.currentCharacter);
2770 }
2771
2772 final char[] optimizedCurrentTokenSource1() {
2773 //return always the same char[] build only once
2774
2775 //optimization at no speed cost of 99.5 % of the singleCharIdentifier
2776 char charOne = this.source[this.startPosition];
2777 switch (charOne) {
2778 case 'a' :
2779 return charArray_a;
2780 case 'b' :
2781 return charArray_b;
2782 case 'c' :
2783 return charArray_c;
2784 case 'd' :
2785 return charArray_d;
2786 case 'e' :
2787 return charArray_e;
2788 case 'f' :
2789 return charArray_f;
2790 case 'g' :
2791 return charArray_g;
2792 case 'h' :
2793 return charArray_h;
2794 case 'i' :
2795 return charArray_i;
2796 case 'j' :
2797 return charArray_j;
2798 case 'k' :
2799 return charArray_k;
2800 case 'l' :
2801 return charArray_l;
2802 case 'm' :
2803 return charArray_m;
2804 case 'n' :
2805 return charArray_n;
2806 case 'o' :
2807 return charArray_o;
2808 case 'p' :
2809 return charArray_p;
2810 case 'q' :
2811 return charArray_q;
2812 case 'r' :
2813 return charArray_r;
2814 case 's' :
2815 return charArray_s;
2816 case 't' :
2817 return charArray_t;
2818 case 'u' :
2819 return charArray_u;
2820 case 'v' :
2821 return charArray_v;
2822 case 'w' :
2823 return charArray_w;
2824 case 'x' :
2825 return charArray_x;
2826 case 'y' :
2827 return charArray_y;
2828 case 'z' :
2829 return charArray_z;
2830 default :
2831 return new char[] {charOne};
2832 }
2833 }
2834 final char[] optimizedCurrentTokenSource2() {
2835 //try to return the same char[] build only once
2836
2837 char[] src = this.source;
2838 int start = this.startPosition;
2839 char c0 , c1;
2840 int hash = (((c0=src[start]) << 6) + (c1=src[start+1])) % TableSize;
2841 char[][] table = this.charArray_length[0][hash];
2842 int i = this.newEntry2;
2843 while (++i < InternalTableSize) {
2844 char[] charArray = table[i];
2845 if ((c0 == charArray[0]) && (c1 == charArray[1]))
2846 return charArray;
2847 }
2848 //---------other side---------
2849 i = -1;
2850 int max = this.newEntry2;
2851 while (++i <= max) {
2852 char[] charArray = table[i];
2853 if ((c0 == charArray[0]) && (c1 == charArray[1]))
2854 return charArray;
2855 }
2856 //--------add the entry-------
2857 if (++max >= InternalTableSize) max = 0;
2858 char[] r;
2859 System.arraycopy(src, start, r= new char[2], 0, 2);
2860 //newIdentCount++;
2861 return table[this.newEntry2 = max] = r; //(r = new char[] {c0, c1});
2862 }
2863 final char[] optimizedCurrentTokenSource3() {
2864 //try to return the same char[] build only once
2865
2866 char[] src = this.source;
2867 int start = this.startPosition;
2868 char c0, c1=src[start+1], c2;
2869 int hash = (((c0=src[start])<< 6) + (c2=src[start+2])) % TableSize;
2870 // int hash = ((c0 << 12) + (c1<< 6) + c2) % TableSize;
2871 char[][] table = this.charArray_length[1][hash];
2872 int i = this.newEntry3;
2873 while (++i < InternalTableSize) {
2874 char[] charArray = table[i];
2875 if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]))
2876 return charArray;
2877 }
2878 //---------other side---------
2879 i = -1;
2880 int max = this.newEntry3;
2881 while (++i <= max) {
2882 char[] charArray = table[i];
2883 if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]))
2884 return charArray;
2885 }
2886 //--------add the entry-------
2887 if (++max >= InternalTableSize) max = 0;
2888 char[] r;
2889 System.arraycopy(src, start, r= new char[3], 0, 3);
2890 //newIdentCount++;
2891 return table[this.newEntry3 = max] = r; //(r = new char[] {c0, c1, c2});
2892 }
2893 final char[] optimizedCurrentTokenSource4() {
2894 //try to return the same char[] build only once
2895
2896 char[] src = this.source;
2897 int start = this.startPosition;
2898 char c0, c1 = src[start+1], c2, c3 = src[start+3];
2899 int hash = (((c0=src[start]) << 6) + (c2=src[start+2])) % TableSize;
2900 // int hash = (int) (((((long) c0) << 18) + (c1 << 12) + (c2 << 6) + c3) % TableSize);
2901 char[][] table = this.charArray_length[2][hash];
2902 int i = this.newEntry4;
2903 while (++i < InternalTableSize) {
2904 char[] charArray = table[i];
2905 if ((c0 == charArray[0])
2906 && (c1 == charArray[1])
2907 && (c2 == charArray[2])
2908 && (c3 == charArray[3]))
2909 return charArray;
2910 }
2911 //---------other side---------
2912 i = -1;
2913 int max = this.newEntry4;
2914 while (++i <= max) {
2915 char[] charArray = table[i];
2916 if ((c0 == charArray[0])
2917 && (c1 == charArray[1])
2918 && (c2 == charArray[2])
2919 && (c3 == charArray[3]))
2920 return charArray;
2921 }
2922 //--------add the entry-------
2923 if (++max >= InternalTableSize) max = 0;
2924 char[] r;
2925 System.arraycopy(src, start, r= new char[4], 0, 4);
2926 //newIdentCount++;
2927 return table[this.newEntry4 = max] = r; //(r = new char[] {c0, c1, c2, c3});
2928 }
2929 final char[] optimizedCurrentTokenSource5() {
2930 //try to return the same char[] build only once
2931
2932 char[] src = this.source;
2933 int start = this.startPosition;
2934 char c0, c1 = src[start+1], c2, c3 = src[start+3], c4;
2935 int hash = (((c0=src[start]) << 12) +((c2=src[start+2]) << 6) + (c4=src[start+4])) % TableSize;
2936 // int hash = (int) (((((long) c0) << 24) + (((long) c1) << 18) + (c2 << 12) + (c3 << 6) + c4) % TableSize);
2937 char[][] table = this.charArray_length[3][hash];
2938 int i = this.newEntry5;
2939 while (++i < InternalTableSize) {
2940 char[] charArray = table[i];
2941 if ((c0 == charArray[0])
2942 && (c1 == charArray[1])
2943 && (c2 == charArray[2])
2944 && (c3 == charArray[3])
2945 && (c4 == charArray[4]))
2946 return charArray;
2947 }
2948 //---------other side---------
2949 i = -1;
2950 int max = this.newEntry5;
2951 while (++i <= max) {
2952 char[] charArray = table[i];
2953 if ((c0 == charArray[0])
2954 && (c1 == charArray[1])
2955 && (c2 == charArray[2])
2956 && (c3 == charArray[3])
2957 && (c4 == charArray[4]))
2958 return charArray;
2959 }
2960 //--------add the entry-------
2961 if (++max >= InternalTableSize) max = 0;
2962 char[] r;
2963 System.arraycopy(src, start, r= new char[5], 0, 5);
2964 //newIdentCount++;
2965 return table[this.newEntry5 = max] = r; //(r = new char[] {c0, c1, c2, c3, c4});
2966 }
2967 final char[] optimizedCurrentTokenSource6() {
2968 //try to return the same char[] build only once
2969
2970 char[] src = this.source;
2971 int start = this.startPosition;
2972 char c0, c1 = src[start+1], c2, c3 = src[start+3], c4, c5 = src[start+5];
2973 int hash = (((c0=src[start]) << 12) +((c2=src[start+2]) << 6) + (c4=src[start+4])) % TableSize;
2974 // int hash = (int)(((((long) c0) << 32) + (((long) c1) << 24) + (((long) c2) << 18) + (c3 << 12) + (c4 << 6) + c5) % TableSize);
2975 char[][] table = this.charArray_length[4][hash];
2976 int i = this.newEntry6;
2977 while (++i < InternalTableSize) {
2978 char[] charArray = table[i];
2979 if ((c0 == charArray[0])
2980 && (c1 == charArray[1])
2981 && (c2 == charArray[2])
2982 && (c3 == charArray[3])
2983 && (c4 == charArray[4])
2984 && (c5 == charArray[5]))
2985 return charArray;
2986 }
2987 //---------other side---------
2988 i = -1;
2989 int max = this.newEntry6;
2990 while (++i <= max) {
2991 char[] charArray = table[i];
2992 if ((c0 == charArray[0])
2993 && (c1 == charArray[1])
2994 && (c2 == charArray[2])
2995 && (c3 == charArray[3])
2996 && (c4 == charArray[4])
2997 && (c5 == charArray[5]))
2998 return charArray;
2999 }
3000 //--------add the entry-------
3001 if (++max >= InternalTableSize) max = 0;
3002 char[] r;
3003 System.arraycopy(src, start, r= new char[6], 0, 6);
3004 //newIdentCount++;
3005 return table[this.newEntry6 = max] = r; //(r = new char[] {c0, c1, c2, c3, c4, c5});
3006 }
3007 public boolean isInModuleDeclaration() {
3008 return this.fakeInModule || this.insideModuleInfo ||
3009 (this.activeParser != null ? this.activeParser.isParsingModuleDeclaration() : false);
3010 }
3011 protected boolean areRestrictedModuleKeywordsActive() {
3012 return this.scanContext != null && this.scanContext != ScanContext.INACTIVE;
3013 }
3014 void updateScanContext(int token) {
3015 switch (token) {
3016 case TerminalTokens.TokenNameSEMICOLON: // next could be a KEYWORD
3017 case TerminalTokens.TokenNameRBRACE:
3018 case TokenNameRPAREN:
3019 this.scanContext = ScanContext.EXPECTING_KEYWORD;
3020 break;
3021 case TokenNameopen:
3022 this.scanContext = ScanContext.EXPECTING_KEYWORD;
3023 break;
3024 case TokenNamerequires:
3025 this.scanContext = ScanContext.AFTER_REQUIRES;
3026 break;
3027 case TokenNamemodule:
3028 case TokenNameexports:
3029 case TokenNameopens:
3030 case TokenNameuses:
3031 case TokenNameprovides:
3032 case TokenNameto:
3033 case TokenNamewith:
3034 case TokenNametransitive:
3035 case TokenNameDOT:
3036 case TokenNameimport:
3037 case TokenNameAT:
3038 case TokenNameAT308:
3039 case TokenNameCOMMA:
3040 this.scanContext = ScanContext.EXPECTING_IDENTIFIER;
3041 break;
3042 case TokenNameIdentifier:
3043 this.scanContext = ScanContext.EXPECTING_KEYWORD;
3044 break;
3045 case TerminalTokens.TokenNameLBRACE:
3046 this.scanContext = ScanContext.EXPECTING_KEYWORD;
3047 break;
3048 default: // anything else is unexpected and should not alter the context
3049 break;
3050 }
3051 }
3052
3053 private void parseTags() {
3054 int position = 0;
3055 final int currentStartPosition = this.startPosition;
3056 final int currentLinePtr = this.linePtr;
3057 if (currentLinePtr >= 0) {
3058 position = this.lineEnds[currentLinePtr] + 1;
3059 }
3060 while (ScannerHelper.isWhitespace(this.source[position])) {
3061 position++;
3062 }
3063 if (currentStartPosition == position) {
3064 // the whole line is commented out
3065 return;
3066 }
3067 char[] s = null;
3068 int sourceEnd = this.currentPosition;
3069 int sourceStart = currentStartPosition;
3070 int sourceDelta = 0;
3071 if (this.withoutUnicodePtr != 0) {
3072 // 0 is used as a fast test flag so the real first char is in position 1
3073 System.arraycopy(
3074 this.withoutUnicodeBuffer,
3075 1,
3076 s = new char[this.withoutUnicodePtr],
3077 0,
3078 this.withoutUnicodePtr);
3079 sourceEnd = this.withoutUnicodePtr;
3080 sourceStart = 1;
3081 sourceDelta = currentStartPosition;
3082 } else {
3083 s = this.source;
3084 }
3085 int pos;
3086 if (this.checkNonExternalizedStringLiterals &&
3087 (pos = CharOperation.indexOf(TAG_PREFIX, s, true, sourceStart, sourceEnd)) != -1) {
3088 if (this.nlsTags == null) {
3089 this.nlsTags = new NLSTag[10];
3090 this.nlsTagsPtr = 0;
3091 }
3092 while (pos != -1) {
3093 int start = pos + TAG_PREFIX_LENGTH;
3094 int end = CharOperation.indexOf(TAG_POSTFIX, s, start, sourceEnd);
3095 if (end != -1) {
3096 NLSTag currentTag = null;
3097 final int currentLine = currentLinePtr + 1;
3098 try {
3099 currentTag = new NLSTag(pos + sourceDelta, end + sourceDelta, currentLine, extractInt(s, start, end));
3100 } catch (NumberFormatException e) {
3101 currentTag = new NLSTag(pos + sourceDelta, end + sourceDelta, currentLine, -1);
3102 }
3103 if (this.nlsTagsPtr == this.nlsTags.length) {
3104 // resize
3105 System.arraycopy(this.nlsTags, 0, (this.nlsTags = new NLSTag[this.nlsTagsPtr + 10]), 0, this.nlsTagsPtr);
3106 }
3107 this.nlsTags[this.nlsTagsPtr++] = currentTag;
3108 } else {
3109 end = start;
3110 }
3111 pos = CharOperation.indexOf(TAG_PREFIX, s, true, end, sourceEnd);
3112 }
3113 }
3114
3115 if (this.checkUninternedIdentityComparison &&
3116 (pos = CharOperation.indexOf(IDENTITY_COMPARISON_TAG, s, true, sourceStart, sourceEnd)) != -1) {
3117 if (this.validIdentityComparisonLines == null) {
3118 this.validIdentityComparisonLines = new boolean[0];
3119 }
3120 int currentLine = currentLinePtr + 1;
3121 int length = this.validIdentityComparisonLines.length;
3122 System.arraycopy(this.validIdentityComparisonLines, 0, this.validIdentityComparisonLines = new boolean[currentLine + 1], 0, length);
3123 this.validIdentityComparisonLines[currentLine] = true;
3124 }
3125 }
3126 private int extractInt(char[] array, int start, int end) {
3127 int value = 0;
3128 for (int i = start; i < end; i++) {
3129 final char currentChar = array[i];
3130 int digit = 0;
3131 switch(currentChar) {
3132 case '0' :
3133 digit = 0;
3134 break;
3135 case '1' :
3136 digit = 1;
3137 break;
3138 case '2' :
3139 digit = 2;
3140 break;
3141 case '3' :
3142 digit = 3;
3143 break;
3144 case '4' :
3145 digit = 4;
3146 break;
3147 case '5' :
3148 digit = 5;
3149 break;
3150 case '6' :
3151 digit = 6;
3152 break;
3153 case '7' :
3154 digit = 7;
3155 break;
3156 case '8' :
3157 digit = 8;
3158 break;
3159 case '9' :
3160 digit = 9;
3161 break;
3162 default :
3163 throw new NumberFormatException();
3164 }
3165 value *= 10;
3166 if (digit < 0) throw new NumberFormatException();
3167 value += digit;
3168 }
3169 return value;
3170 }
3171 public final void pushLineSeparator() {
3172 //see comment on isLineDelimiter(char) for the use of '\n' and '\r'
3173 final int INCREMENT = 250;
3174 //currentCharacter is at position currentPosition-1
3175 // cr 000D
3176 if (this.currentCharacter == '\r') {
3177 int separatorPos = this.currentPosition - 1;
3178 if ((this.linePtr >= 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return;
3179 int length = this.lineEnds.length;
3180 if (++this.linePtr >= length)
3181 System.arraycopy(this.lineEnds, 0, this.lineEnds = new int[2*length + INCREMENT], 0, length);
3182 this.lineEnds[this.linePtr] = separatorPos;
3183 // look-ahead for merged cr+lf
3184 try {
3185 if (this.source[this.currentPosition] == '\n') {
3186 //System.out.println("look-ahead LF-" + this.currentPosition);
3187 this.lineEnds[this.linePtr] = this.currentPosition;
3188 this.currentPosition++;
3189 this.wasAcr = false;
3190 } else {
3191 this.wasAcr = true;
3192 }
3193 } catch(IndexOutOfBoundsException e) {
3194 this.wasAcr = true;
3195 }
3196 } else {
3197 // lf 000A
3198 if (this.currentCharacter == '\n') { //must merge eventual cr followed by lf
3199 if (this.wasAcr && (this.lineEnds[this.linePtr] == (this.currentPosition - 2))) {
3200 //System.out.println("merge LF-" + (this.currentPosition - 1));
3201 this.lineEnds[this.linePtr] = this.currentPosition - 1;
3202 } else {
3203 int separatorPos = this.currentPosition - 1;
3204 if ((this.linePtr >= 0) && (this.lineEnds[this.linePtr] >= separatorPos)) return;
3205 int length = this.lineEnds.length;
3206 if (++this.linePtr >= length)
3207 System.arraycopy(this.lineEnds, 0, this.lineEnds = new int[2*length + INCREMENT], 0, length);
3208 this.lineEnds[this.linePtr] = separatorPos;
3209 }
3210 this.wasAcr = false;
3211 }
3212 }
3213 }
3214 public final void pushUnicodeLineSeparator() {
3215 // cr 000D
3216 if (this.currentCharacter == '\r') {
3217 if (this.source[this.currentPosition] == '\n') {
3218 this.wasAcr = false;
3219 } else {
3220 this.wasAcr = true;
3221 }
3222 } else {
3223 // lf 000A
3224 if (this.currentCharacter == '\n') { //must merge eventual cr followed by lf
3225 this.wasAcr = false;
3226 }
3227 }
3228 }
3229
3230 public void recordComment(int token) {
3231 // compute position
3232 int commentStart = this.startPosition;
3233 int stopPosition = this.currentPosition;
3234 switch (token) {
3235 case TokenNameCOMMENT_LINE:
3236 // both positions are negative
3237 commentStart = -this.startPosition;
3238 stopPosition = -this.lastCommentLinePosition;
3239 break;
3240 case TokenNameCOMMENT_BLOCK:
3241 // only end position is negative
3242 stopPosition = -this.currentPosition;
3243 break;
3244 }
3245
3246 // a new comment is recorded
3247 int length = this.commentStops.length;
3248 if (++this.commentPtr >= length) {
3249 int newLength = length + COMMENT_ARRAYS_SIZE*10;
3250 System.arraycopy(this.commentStops, 0, this.commentStops = new int[newLength], 0, length);
3251 System.arraycopy(this.commentStarts, 0, this.commentStarts = new int[newLength], 0, length);
3252 System.arraycopy(this.commentTagStarts, 0, this.commentTagStarts = new int[newLength], 0, length);
3253 }
3254 this.commentStops[this.commentPtr] = stopPosition;
3255 this.commentStarts[this.commentPtr] = commentStart;
3256 }
3257
3258 /**
3259 * Reposition the scanner on some portion of the original source. The given endPosition is the last valid position.
3260 * Beyond this position, the scanner will answer EOF tokens (<code>ITerminalSymbols.TokenNameEOF</code>).
3261 *
3262 * @param begin the given start position
3263 * @param end the given end position
3264 */
3265 public void resetTo(int begin, int end) {
3266 resetTo(begin, end, isInModuleDeclaration());
3267 }
3268 public void resetTo(int begin, int end, boolean isModuleInfo) {
3269 resetTo(begin, end, isModuleInfo, null);
3270 }
3271 /**
3272 * Reposition the scanner on some portion of the original source. The given endPosition is the last valid position.
3273 * Beyond this position, the scanner will answer EOF tokens (<code>ITerminalSymbols.TokenNameEOF</code>).
3274 *
3275 * @param begin the given start position
3276 * @param end the given end position
3277 * @param isModuleInfo if true apply rules for restricted keywords even without a connection to a properly configured parser
3278 * @param context The scan context to use for restricted keyword support, use null to compute
3279 */
3280 public void resetTo(int begin, int end, boolean isModuleInfo, ScanContext context) {
3281 //reset the scanner to a given position where it may rescan again
3282
3283 this.diet = false;
3284 this.initialPosition = this.startPosition = this.currentPosition = begin;
3285 if (this.source != null && this.source.length < end) {
3286 this.eofPosition = this.source.length;
3287 } else {
3288 this.eofPosition = end < Integer.MAX_VALUE ? end + 1 : end;
3289 }
3290 this.commentPtr = -1; // reset comment stack
3291 this.foundTaskCount = 0;
3292 this.lookBack[0] = this.lookBack[1] = this.nextToken = TokenNameNotAToken;
3293 this.consumingEllipsisAnnotations = false;
3294 this.insideModuleInfo = isModuleInfo;
3295 this.scanContext = context == null ? getScanContext(begin) : context;
3296 }
3297
3298 private ScanContext getScanContext(int begin) {
3299 if (!isInModuleDeclaration())
3300 return ScanContext.INACTIVE;
3301 if (begin == 0)
3302 return ScanContext.EXPECTING_KEYWORD;
3303 CompilerOptions options = new CompilerOptions();
3304 options.complianceLevel = this.complianceLevel;
3305 options.sourceLevel = this.sourceLevel;
3306 ScanContextDetector parser = new ScanContextDetector(options);
3307 return parser.getScanContext(this.source, begin - 1);
3308 }
3309 protected final void scanEscapeCharacter() throws InvalidInputException {
3310 // the string with "\\u" is a legal string of two chars \ and u
3311 //thus we use a direct access to the source (for regular cases).
3312 switch (this.currentCharacter) {
3313 case 'b' :
3314 this.currentCharacter = '\b';
3315 break;
3316 case 't' :
3317 this.currentCharacter = '\t';
3318 break;
3319 case 'n' :
3320 this.currentCharacter = '\n';
3321 break;
3322 case 'f' :
3323 this.currentCharacter = '\f';
3324 break;
3325 case 'r' :
3326 this.currentCharacter = '\r';
3327 break;
3328 case '\"' :
3329 this.currentCharacter = '\"';
3330 break;
3331 case '\'' :
3332 this.currentCharacter = '\'';
3333 break;
3334 case 's' :
3335 this.currentCharacter = ' ';
3336 break;
3337 case '\\' :
3338 this.currentCharacter = '\\';
3339 break;
3340 default :
3341 // -----------octal escape--------------
3342 // OctalDigit
3343 // OctalDigit OctalDigit
3344 // ZeroToThree OctalDigit OctalDigit
3345
3346 int number = ScannerHelper.getHexadecimalValue(this.currentCharacter);
3347 if (number >= 0 && number <= 7) {
3348 boolean zeroToThreeNot = number > 3;
3349 if (ScannerHelper.isDigit(this.currentCharacter = this.source[this.currentPosition++])) {
3350 int digit = ScannerHelper.getHexadecimalValue(this.currentCharacter);
3351 if (digit >= 0 && digit <= 7) {
3352 number = (number * 8) + digit;
3353 if (ScannerHelper.isDigit(this.currentCharacter = this.source[this.currentPosition++])) {
3354 if (zeroToThreeNot) {// has read \NotZeroToThree OctalDigit Digit --> ignore last character
3355 this.currentPosition--;
3356 } else {
3357 digit = ScannerHelper.getHexadecimalValue(this.currentCharacter);
3358 if (digit >= 0 && digit <= 7){ // has read \ZeroToThree OctalDigit OctalDigit
3359 number = (number * 8) + digit;
3360 } else {// has read \ZeroToThree OctalDigit NonOctalDigit --> ignore last character
3361 this.currentPosition--;
3362 }
3363 }
3364 } else { // has read \OctalDigit NonDigit--> ignore last character
3365 this.currentPosition--;
3366 }
3367 } else { // has read \OctalDigit NonOctalDigit--> ignore last character
3368 this.currentPosition--;
3369 }
3370 } else { // has read \OctalDigit --> ignore last character
3371 this.currentPosition--;
3372 }
3373 if (number > 255)
3374 throw new InvalidInputException(INVALID_ESCAPE);
3375 this.currentCharacter = (char) number;
3376 } else
3377 throw new InvalidInputException(INVALID_ESCAPE);
3378 }
3379 }
3380 public int scanIdentifierOrKeywordWithBoundCheck() {
3381 //test keywords
3382
3383 //first dispatch on the first char.
3384 //then the length. If there are several
3385 //keywors with the same length AND the same first char, then do another
3386 //dispatch on the second char
3387 this.useAssertAsAnIndentifier = false;
3388 this.useEnumAsAnIndentifier = false;
3389
3390 char[] src = this.source;
3391 identLoop: {
3392 int pos;
3393 int srcLength = this.eofPosition;
3394 while (true) {
3395 if ((pos = this.currentPosition) >= srcLength) // handle the obvious case upfront
3396 break identLoop;
3397 char c = src[pos];
3398 if (c < ScannerHelper.MAX_OBVIOUS) {
3399 if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] &
3400 (ScannerHelper.C_UPPER_LETTER | ScannerHelper.C_LOWER_LETTER | ScannerHelper.C_IDENT_PART | ScannerHelper.C_DIGIT)) != 0) {
3401 if (this.withoutUnicodePtr != 0) {
3402 this.currentCharacter = c;
3403 unicodeStore();
3404 }
3405 this.currentPosition++;
3406 } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & (ScannerHelper.C_SEPARATOR | ScannerHelper.C_JLS_SPACE)) != 0) {
3407 this.currentCharacter = c;
3408 break identLoop;
3409 } else {
3410 //System.out.println("slow<=128: "+ c);
3411 while (getNextCharAsJavaIdentifierPartWithBoundCheck()){/*empty*/}
3412 break identLoop;
3413 }
3414 } else {
3415 //System.out.println("slow>>128: "+ c);
3416 while (getNextCharAsJavaIdentifierPartWithBoundCheck()){/*empty*/}
3417 break identLoop;
3418 }
3419 }
3420 }
3421
3422 int index, length;
3423 char[] data;
3424 if (this.withoutUnicodePtr == 0) {
3425 //quick test on length == 1 but not on length > 12 while most identifier
3426 //have a length which is <= 12...but there are lots of identifier with
3427 //only one char....
3428 if ((length = this.currentPosition - this.startPosition) == 1) {
3429 return TokenNameIdentifier;
3430 }
3431 data = this.source;
3432 index = this.startPosition;
3433 } else {
3434 if ((length = this.withoutUnicodePtr) == 1)
3435 return TokenNameIdentifier;
3436 data = this.withoutUnicodeBuffer;
3437 index = 1;
3438 }
3439
3440 return internalScanIdentifierOrKeyword(index, length, data);
3441 }
3442 public int scanIdentifierOrKeyword() {
3443 //test keywords
3444
3445 //first dispatch on the first char.
3446 //then the length. If there are several
3447 //keywords with the same length AND the same first char, then do another
3448 //dispatch on the second char
3449 this.useAssertAsAnIndentifier = false;
3450 this.useEnumAsAnIndentifier = false;
3451
3452 char[] src = this.source;
3453 identLoop: {
3454 int pos;
3455 int srcLength = this.eofPosition;
3456 while (true) {
3457 if ((pos = this.currentPosition) >= srcLength) // handle the obvious case upfront
3458 break identLoop;
3459 char c = src[pos];
3460 if (c < ScannerHelper.MAX_OBVIOUS) {
3461 if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] &
3462 (ScannerHelper.C_UPPER_LETTER | ScannerHelper.C_LOWER_LETTER | ScannerHelper.C_IDENT_PART | ScannerHelper.C_DIGIT)) != 0) {
3463 if (this.withoutUnicodePtr != 0) {
3464 this.currentCharacter = c;
3465 unicodeStore();
3466 }
3467 this.currentPosition++;
3468 } else if ((ScannerHelper.OBVIOUS_IDENT_CHAR_NATURES[c] & (ScannerHelper.C_SEPARATOR | ScannerHelper.C_JLS_SPACE)) != 0) {
3469 this.currentCharacter = c;
3470 break identLoop;
3471 } else {
3472 //System.out.println("slow<=128: "+ c);
3473 while (getNextCharAsJavaIdentifierPart()){/*empty*/}
3474 break identLoop;
3475 }
3476 } else {
3477 //System.out.println("slow>>128: "+ c);
3478 while (getNextCharAsJavaIdentifierPart()){/*empty*/}
3479 break identLoop;
3480 }
3481 }
3482 }
3483
3484 int index, length;
3485 char[] data;
3486 if (this.withoutUnicodePtr == 0) {
3487 //quick test on length == 1 but not on length > 12 while most identifier
3488 //have a length which is <= 12...but there are lots of identifier with
3489 //only one char....
3490 if ((length = this.currentPosition - this.startPosition) == 1) {
3491 return TokenNameIdentifier;
3492 }
3493 data = this.source;
3494 index = this.startPosition;
3495 } else {
3496 if ((length = this.withoutUnicodePtr) == 1)
3497 return TokenNameIdentifier;
3498 data = this.withoutUnicodeBuffer;
3499 index = 1;
3500 }
3501
3502 return internalScanIdentifierOrKeyword(index, length, data);
3503 }
3504 private int internalScanIdentifierOrKeyword(int index, int length, char[] data) {
3505 switch (data[index]) {
3506 case 'a' :
3507 switch(length) {
3508 case 8: //abstract
3509 if ((data[++index] == 'b')
3510 && (data[++index] == 's')
3511 && (data[++index] == 't')
3512 && (data[++index] == 'r')
3513 && (data[++index] == 'a')
3514 && (data[++index] == 'c')
3515 && (data[++index] == 't')) {
3516 return TokenNameabstract;
3517 } else {
3518 return TokenNameIdentifier;
3519 }
3520 case 6: // assert
3521 if ((data[++index] == 's')
3522 && (data[++index] == 's')
3523 && (data[++index] == 'e')
3524 && (data[++index] == 'r')
3525 && (data[++index] == 't')) {
3526 if (this.sourceLevel >= ClassFileConstants.JDK1_4) {
3527 this.containsAssertKeyword = true;
3528 return TokenNameassert;
3529 } else {
3530 this.useAssertAsAnIndentifier = true;
3531 return TokenNameIdentifier;
3532 }
3533 } else {
3534 return TokenNameIdentifier;
3535 }
3536 default:
3537 return TokenNameIdentifier;
3538 }
3539 case 'b' : //boolean break byte
3540 switch (length) {
3541 case 4 :
3542 if ((data[++index] == 'y') && (data[++index] == 't') && (data[++index] == 'e'))
3543 return TokenNamebyte;
3544 else
3545 return TokenNameIdentifier;
3546 case 5 :
3547 if ((data[++index] == 'r')
3548 && (data[++index] == 'e')
3549 && (data[++index] == 'a')
3550 && (data[++index] == 'k'))
3551 return TokenNamebreak;
3552 else
3553 return TokenNameIdentifier;
3554 case 7 :
3555 if ((data[++index] == 'o')
3556 && (data[++index] == 'o')
3557 && (data[++index] == 'l')
3558 && (data[++index] == 'e')
3559 && (data[++index] == 'a')
3560 && (data[++index] == 'n'))
3561 return TokenNameboolean;
3562 else
3563 return TokenNameIdentifier;
3564 default :
3565 return TokenNameIdentifier;
3566 }
3567
3568 case 'c' : //case char catch const class continue
3569 switch (length) {
3570 case 4 :
3571 if (data[++index] == 'a')
3572 if ((data[++index] == 's') && (data[++index] == 'e'))
3573 return TokenNamecase;
3574 else
3575 return TokenNameIdentifier;
3576 else
3577 if ((data[index] == 'h') && (data[++index] == 'a') && (data[++index] == 'r'))
3578 return TokenNamechar;
3579 else
3580 return TokenNameIdentifier;
3581 case 5 :
3582 if (data[++index] == 'a')
3583 if ((data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h'))
3584 return TokenNamecatch;
3585 else
3586 return TokenNameIdentifier;
3587 else
3588 if (data[index] == 'l')
3589 if ((data[++index] == 'a')
3590 && (data[++index] == 's')
3591 && (data[++index] == 's'))
3592 return TokenNameclass;
3593 else
3594 return TokenNameIdentifier;
3595 else if ((data[index] == 'o')
3596 && (data[++index] == 'n')
3597 && (data[++index] == 's')
3598 && (data[++index] == 't'))
3599 return TokenNameconst; //const is not used in java ???????
3600 else
3601 return TokenNameIdentifier;
3602 case 8 :
3603 if ((data[++index] == 'o')
3604 && (data[++index] == 'n')
3605 && (data[++index] == 't')
3606 && (data[++index] == 'i')
3607 && (data[++index] == 'n')
3608 && (data[++index] == 'u')
3609 && (data[++index] == 'e'))
3610 return TokenNamecontinue;
3611 else
3612 return TokenNameIdentifier;
3613 default :
3614 return TokenNameIdentifier;
3615 }
3616
3617 case 'd' : //default do double
3618 switch (length) {
3619 case 2 :
3620 if ((data[++index] == 'o'))
3621 return TokenNamedo;
3622 else
3623 return TokenNameIdentifier;
3624 case 6 :
3625 if ((data[++index] == 'o')
3626 && (data[++index] == 'u')
3627 && (data[++index] == 'b')
3628 && (data[++index] == 'l')
3629 && (data[++index] == 'e'))
3630 return TokenNamedouble;
3631 else
3632 return TokenNameIdentifier;
3633 case 7 :
3634 if ((data[++index] == 'e')
3635 && (data[++index] == 'f')
3636 && (data[++index] == 'a')
3637 && (data[++index] == 'u')
3638 && (data[++index] == 'l')
3639 && (data[++index] == 't'))
3640 return TokenNamedefault;
3641 else
3642 return TokenNameIdentifier;
3643 default :
3644 return TokenNameIdentifier;
3645 }
3646 case 'e' : //else extends exports
3647 switch (length) {
3648 case 4 :
3649 if (data[++index] == 'l') {
3650 if ((data[++index] == 's') && (data[++index] == 'e')) {
3651 return TokenNameelse;
3652 } else {
3653 return TokenNameIdentifier;
3654 }
3655 } else if ((data[index] == 'n')
3656 && (data[++index] == 'u')
3657 && (data[++index] == 'm')) {
3658 if (this.sourceLevel >= ClassFileConstants.JDK1_5) {
3659 return TokenNameenum;
3660 } else {
3661 this.useEnumAsAnIndentifier = true;
3662 return TokenNameIdentifier;
3663 }
3664 }
3665 return TokenNameIdentifier;
3666 case 7 :
3667 if ((data[++index] == 'x')) {
3668 if ((data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'n')
3669 && (data[++index] == 'd') && (data[++index] == 's')) {
3670 return TokenNameextends;
3671 } else if (areRestrictedModuleKeywordsActive()
3672 && (data[index] == 'p') && (data[++index] == 'o') && (data[++index] == 'r')
3673 && (data[++index] == 't') && (data[++index] == 's')) {
3674 return TokenNameexports;
3675 } else
3676 return TokenNameIdentifier;
3677 } else
3678 return TokenNameIdentifier;
3679 default :
3680 return TokenNameIdentifier;
3681 }
3682
3683 case 'f' : //final finally float for false
3684 switch (length) {
3685 case 3 :
3686 if ((data[++index] == 'o') && (data[++index] == 'r'))
3687 return TokenNamefor;
3688 else
3689 return TokenNameIdentifier;
3690 case 5 :
3691 if (data[++index] == 'i')
3692 if ((data[++index] == 'n')
3693 && (data[++index] == 'a')
3694 && (data[++index] == 'l')) {
3695 return TokenNamefinal;
3696 } else
3697 return TokenNameIdentifier;
3698 else
3699 if (data[index] == 'l')
3700 if ((data[++index] == 'o')
3701 && (data[++index] == 'a')
3702 && (data[++index] == 't'))
3703 return TokenNamefloat;
3704 else
3705 return TokenNameIdentifier;
3706 else
3707 if ((data[index] == 'a')
3708 && (data[++index] == 'l')
3709 && (data[++index] == 's')
3710 && (data[++index] == 'e'))
3711 return TokenNamefalse;
3712 else
3713 return TokenNameIdentifier;
3714 case 7 :
3715 if ((data[++index] == 'i')
3716 && (data[++index] == 'n')
3717 && (data[++index] == 'a')
3718 && (data[++index] == 'l')
3719 && (data[++index] == 'l')
3720 && (data[++index] == 'y'))
3721 return TokenNamefinally;
3722 else
3723 return TokenNameIdentifier;
3724
3725 default :
3726 return TokenNameIdentifier;
3727 }
3728 case 'g' : //goto
3729 if (length == 4) {
3730 if ((data[++index] == 'o')
3731 && (data[++index] == 't')
3732 && (data[++index] == 'o')) {
3733 return TokenNamegoto;
3734 }
3735 } //no goto in java are allowed, so why java removes this keyword ???
3736 return TokenNameIdentifier;
3737
3738 case 'i' : //if implements import instanceof int interface
3739 switch (length) {
3740 case 2 :
3741 if (data[++index] == 'f')
3742 return TokenNameif;
3743 else
3744 return TokenNameIdentifier;
3745 case 3 :
3746 if ((data[++index] == 'n') && (data[++index] == 't'))
3747 return TokenNameint;
3748 else
3749 return TokenNameIdentifier;
3750 case 6 :
3751 if ((data[++index] == 'm')
3752 && (data[++index] == 'p')
3753 && (data[++index] == 'o')
3754 && (data[++index] == 'r')
3755 && (data[++index] == 't'))
3756 return TokenNameimport;
3757 else
3758 return TokenNameIdentifier;
3759 case 9 :
3760 if ((data[++index] == 'n')
3761 && (data[++index] == 't')
3762 && (data[++index] == 'e')
3763 && (data[++index] == 'r')
3764 && (data[++index] == 'f')
3765 && (data[++index] == 'a')
3766 && (data[++index] == 'c')
3767 && (data[++index] == 'e'))
3768 return TokenNameinterface;
3769 else
3770 return TokenNameIdentifier;
3771 case 10 :
3772 if (data[++index] == 'm')
3773 if ((data[++index] == 'p')
3774 && (data[++index] == 'l')
3775 && (data[++index] == 'e')
3776 && (data[++index] == 'm')
3777 && (data[++index] == 'e')
3778 && (data[++index] == 'n')
3779 && (data[++index] == 't')
3780 && (data[++index] == 's'))
3781 return TokenNameimplements;
3782 else
3783 return TokenNameIdentifier;
3784 else
3785 if ((data[index] == 'n')
3786 && (data[++index] == 's')
3787 && (data[++index] == 't')
3788 && (data[++index] == 'a')
3789 && (data[++index] == 'n')
3790 && (data[++index] == 'c')
3791 && (data[++index] == 'e')
3792 && (data[++index] == 'o')
3793 && (data[++index] == 'f'))
3794 return TokenNameinstanceof;
3795 else
3796 return TokenNameIdentifier;
3797
3798 default :
3799 return TokenNameIdentifier;
3800 }
3801
3802 case 'l' : //long
3803 if (length == 4) {
3804 if ((data[++index] == 'o')
3805 && (data[++index] == 'n')
3806 && (data[++index] == 'g')) {
3807 return TokenNamelong;
3808 }
3809 }
3810 return TokenNameIdentifier;
3811
3812 case 'm': //module
3813 switch (length) {
3814 case 6 :
3815 if (areRestrictedModuleKeywordsActive()
3816 && (data[++index] == 'o')
3817 && (data[++index] == 'd')
3818 && (data[++index] == 'u')
3819 && (data[++index] == 'l')
3820 && (data[++index] == 'e'))
3821 return TokenNamemodule;
3822 else
3823 return TokenNameIdentifier;
3824 default :
3825 return TokenNameIdentifier;
3826 }
3827
3828 case 'n' : //native new null
3829 switch (length) {
3830 case 3 :
3831 if ((data[++index] == 'e') && (data[++index] == 'w'))
3832 return TokenNamenew;
3833 else
3834 return TokenNameIdentifier;
3835 case 4 :
3836 if ((data[++index] == 'u') && (data[++index] == 'l') && (data[++index] == 'l'))
3837 return TokenNamenull;
3838 else
3839 return TokenNameIdentifier;
3840 case 6 :
3841 if ((data[++index] == 'a')
3842 && (data[++index] == 't')
3843 && (data[++index] == 'i')
3844 && (data[++index] == 'v')
3845 && (data[++index] == 'e')) {
3846 return TokenNamenative;
3847 } else
3848 return TokenNameIdentifier;
3849 default :
3850 return TokenNameIdentifier;
3851 }
3852
3853 case 'o':
3854 switch (length) {
3855 case 4 :
3856 if (areRestrictedModuleKeywordsActive() && (data[++index] == 'p') && (data[++index] == 'e') && (data[++index] == 'n'))
3857 return TokenNameopen;
3858 else
3859 return TokenNameIdentifier;
3860 case 5 :
3861 if (areRestrictedModuleKeywordsActive()
3862 && (data[++index] == 'p')
3863 && (data[++index] == 'e')
3864 && (data[++index] == 'n')
3865 && (data[++index] == 's'))
3866 return TokenNameopens;
3867 else
3868 return TokenNameIdentifier;
3869 default :
3870 return TokenNameIdentifier;
3871 }
3872 case 'p' : //package private protected public provides
3873 switch (length) {
3874 case 6 :
3875 if ((data[++index] == 'u')
3876 && (data[++index] == 'b')
3877 && (data[++index] == 'l')
3878 && (data[++index] == 'i')
3879 && (data[++index] == 'c')) {
3880 return TokenNamepublic;
3881 } else
3882 return TokenNameIdentifier;
3883 case 7 :
3884 if (data[++index] == 'a')
3885 if ((data[++index] == 'c')
3886 && (data[++index] == 'k')
3887 && (data[++index] == 'a')
3888 && (data[++index] == 'g')
3889 && (data[++index] == 'e'))
3890 return TokenNamepackage;
3891 else
3892 return TokenNameIdentifier;
3893 else
3894 if ((data[index] == 'r')
3895 && (data[++index] == 'i')
3896 && (data[++index] == 'v')
3897 && (data[++index] == 'a')
3898 && (data[++index] == 't')
3899 && (data[++index] == 'e')) {
3900 return TokenNameprivate;
3901 } else
3902 return TokenNameIdentifier;
3903 case 8 :
3904 if (areRestrictedModuleKeywordsActive()
3905 && (data[++index] == 'r')
3906 && (data[++index] == 'o')
3907 && (data[++index] == 'v')
3908 && (data[++index] == 'i')
3909 && (data[++index] == 'd')
3910 && (data[++index] == 'e')
3911 && (data[++index] == 's')) {
3912 return TokenNameprovides;
3913 } else
3914 return TokenNameIdentifier;
3915 case 9 :
3916 if ((data[++index] == 'r')
3917 && (data[++index] == 'o')
3918 && (data[++index] == 't')
3919 && (data[++index] == 'e')
3920 && (data[++index] == 'c')
3921 && (data[++index] == 't')
3922 && (data[++index] == 'e')
3923 && (data[++index] == 'd')) {
3924 return TokenNameprotected;
3925 } else
3926 return TokenNameIdentifier;
3927
3928 default :
3929 return TokenNameIdentifier;
3930 }
3931
3932 case 'r' : //return requires
3933 switch (length) {
3934 case 6:
3935 if (data[++index] == 'e') {
3936 if ((data[++index] == 't')
3937 && (data[++index] == 'u')
3938 && (data[++index] == 'r')
3939 && (data[++index] == 'n'))
3940 return TokenNamereturn;
3941 else if ((data[index] == 'c')
3942 && (data[++index] == 'o')
3943 && (data[++index] == 'r')
3944 && (data[++index] == 'd'))
3945 return disambiguatedRestrictedIdentifierrecord(TokenNameRestrictedIdentifierrecord);
3946 }
3947 return TokenNameIdentifier;
3948 case 8:
3949 if (areRestrictedModuleKeywordsActive()
3950 && (data[++index] == 'e')
3951 && (data[++index] == 'q')
3952 && (data[++index] == 'u')
3953 && (data[++index] == 'i')
3954 && (data[++index] == 'r')
3955 && (data[++index] == 'e')
3956 && (data[++index] == 's')) {
3957 return TokenNamerequires;
3958 } else
3959 return TokenNameIdentifier;
3960 }
3961 return TokenNameIdentifier;
3962
3963 case 's' : //short static super switch synchronized strictfp
3964 switch (length) {
3965 case 5 :
3966 if (data[++index] == 'h')
3967 if ((data[++index] == 'o') && (data[++index] == 'r') && (data[++index] == 't'))
3968 return TokenNameshort;
3969 else
3970 return TokenNameIdentifier;
3971 else
3972 if ((data[index] == 'u')
3973 && (data[++index] == 'p')
3974 && (data[++index] == 'e')
3975 && (data[++index] == 'r'))
3976 return TokenNamesuper;
3977 else
3978 return TokenNameIdentifier;
3979
3980 case 6 :
3981 if (data[++index] == 't')
3982 if ((data[++index] == 'a')
3983 && (data[++index] == 't')
3984 && (data[++index] == 'i')
3985 && (data[++index] == 'c')) {
3986 return TokenNamestatic;
3987 } else
3988 return TokenNameIdentifier;
3989 else
3990 if ((data[index] == 'w')
3991 && (data[++index] == 'i')
3992 && (data[++index] == 't')
3993 && (data[++index] == 'c')
3994 && (data[++index] == 'h'))
3995 return TokenNameswitch;
3996 else
3997 return TokenNameIdentifier;
3998 case 8 :
3999 if ((data[++index] == 't')
4000 && (data[++index] == 'r')
4001 && (data[++index] == 'i')
4002 && (data[++index] == 'c')
4003 && (data[++index] == 't')
4004 && (data[++index] == 'f')
4005 && (data[++index] == 'p'))
4006 return TokenNamestrictfp;
4007 else
4008 return TokenNameIdentifier;
4009 case 12 :
4010 if ((data[++index] == 'y')
4011 && (data[++index] == 'n')
4012 && (data[++index] == 'c')
4013 && (data[++index] == 'h')
4014 && (data[++index] == 'r')
4015 && (data[++index] == 'o')
4016 && (data[++index] == 'n')
4017 && (data[++index] == 'i')
4018 && (data[++index] == 'z')
4019 && (data[++index] == 'e')
4020 && (data[++index] == 'd')) {
4021 return TokenNamesynchronized;
4022 } else
4023 return TokenNameIdentifier;
4024 default :
4025 return TokenNameIdentifier;
4026 }
4027
4028 case 't' : //try throw throws transient this true
4029 switch (length) {
4030 case 2:
4031 if (areRestrictedModuleKeywordsActive() && data[++index] == 'o')
4032 return TokenNameto;
4033 else
4034 return TokenNameIdentifier;
4035 case 3 :
4036 if ((data[++index] == 'r') && (data[++index] == 'y'))
4037 return TokenNametry;
4038 else
4039 return TokenNameIdentifier;
4040 case 4 :
4041 if (data[++index] == 'h')
4042 if ((data[++index] == 'i') && (data[++index] == 's'))
4043 return TokenNamethis;
4044 else
4045 return TokenNameIdentifier;
4046 else
4047 if ((data[index] == 'r') && (data[++index] == 'u') && (data[++index] == 'e'))
4048 return TokenNametrue;
4049 else
4050 return TokenNameIdentifier;
4051 case 5 :
4052 if ((data[++index] == 'h')
4053 && (data[++index] == 'r')
4054 && (data[++index] == 'o')
4055 && (data[++index] == 'w'))
4056 return TokenNamethrow;
4057 else
4058 return TokenNameIdentifier;
4059 case 6 :
4060 if ((data[++index] == 'h')
4061 && (data[++index] == 'r')
4062 && (data[++index] == 'o')
4063 && (data[++index] == 'w')
4064 && (data[++index] == 's'))
4065 return TokenNamethrows;
4066 else
4067 return TokenNameIdentifier;
4068 case 9 :
4069 if ((data[++index] == 'r')
4070 && (data[++index] == 'a')
4071 && (data[++index] == 'n')
4072 && (data[++index] == 's')
4073 && (data[++index] == 'i')
4074 && (data[++index] == 'e')
4075 && (data[++index] == 'n')
4076 && (data[++index] == 't')) {
4077 return TokenNametransient;
4078 } else
4079 return TokenNameIdentifier;
4080 case 10:
4081 if (areRestrictedModuleKeywordsActive() && (data[++index] == 'r')
4082 && (data[++index] == 'a')
4083 && (data[++index] == 'n')
4084 && (data[++index] == 's')
4085 && (data[++index] == 'i')
4086 && (data[++index] == 't')
4087 && (data[++index] == 'i')
4088 && (data[++index] == 'v')
4089 && (data[++index] == 'e')) {
4090 return TokenNametransitive;
4091 } else
4092 return TokenNameIdentifier;
4093 default :
4094 return TokenNameIdentifier;
4095 }
4096 case 'u' : //uses
4097 switch(length) {
4098 case 4 :
4099 if (areRestrictedModuleKeywordsActive()
4100 && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 's'))
4101 return TokenNameuses;
4102 else
4103 return TokenNameIdentifier;
4104 default :
4105 return TokenNameIdentifier;
4106 }
4107 case 'v' : //void volatile
4108 switch (length) {
4109 case 4 :
4110 if ((data[++index] == 'o') && (data[++index] == 'i') && (data[++index] == 'd'))
4111 return TokenNamevoid;
4112 else
4113 return TokenNameIdentifier;
4114 case 8 :
4115 if ((data[++index] == 'o')
4116 && (data[++index] == 'l')
4117 && (data[++index] == 'a')
4118 && (data[++index] == 't')
4119 && (data[++index] == 'i')
4120 && (data[++index] == 'l')
4121 && (data[++index] == 'e')) {
4122 return TokenNamevolatile;
4123 } else
4124 return TokenNameIdentifier;
4125
4126 default :
4127 return TokenNameIdentifier;
4128 }
4129
4130 case 'w' : //while widefp with
4131 switch (length) {
4132 case 4:
4133 if (areRestrictedModuleKeywordsActive()
4134 && (data[++index] == 'i')
4135 && (data[++index] == 't')
4136 && (data[++index] == 'h'))
4137 return TokenNamewith;
4138 else
4139 return TokenNameIdentifier;
4140 case 5 :
4141 if ((data[++index] == 'h')
4142 && (data[++index] == 'i')
4143 && (data[++index] == 'l')
4144 && (data[++index] == 'e'))
4145 return TokenNamewhile;
4146 else
4147 return TokenNameIdentifier;
4148 //case 6:if ( (data[++index] =='i') && (data[++index]=='d') && (data[++index]=='e') && (data[++index]=='f')&& (data[++index]=='p'))
4149 //return TokenNamewidefp ;
4150 //else
4151 //return TokenNameIdentifier;
4152 default :
4153 return TokenNameIdentifier;
4154 }
4155
4156 case 'y' :
4157 switch (length) {
4158 case 5 :
4159 if ((data[++index] == 'i')
4160 && (data[++index] == 'e')
4161 && (data[++index] == 'l')
4162 && (data[++index] == 'd'))
4163 return disambiguatedRestrictedIdentifierYield(TokenNameRestrictedIdentifierYield);
4164 //$FALL-THROUGH$
4165 default :
4166 return TokenNameIdentifier;
4167 }
4168
4169 default :
4170 return TokenNameIdentifier;
4171 }
4172 }
4173
4174
4175 public int scanNumber(boolean dotPrefix) throws InvalidInputException {
4176
4177 //when entering this method the currentCharacter is the first
4178 //digit of the number. It may be preceeded by a '.' when
4179 //dotPrefix is true
4180
4181 boolean floating = dotPrefix;
4182 if (!dotPrefix && (this.currentCharacter == '0')) {
4183 if (getNextChar('x', 'X') >= 0) { //----------hexa-----------------
4184 int start = this.currentPosition;
4185 consumeDigits(16, true);
4186 int end = this.currentPosition;
4187 if (getNextChar('l', 'L') >= 0) {
4188 if (end == start) {
4189 throw new InvalidInputException(INVALID_HEXA);
4190 }
4191 return TokenNameLongLiteral;
4192 } else if (getNextChar('.')) {
4193 // hexadecimal floating point literal
4194 // read decimal part
4195 boolean hasNoDigitsBeforeDot = end == start;
4196 start = this.currentPosition;
4197 consumeDigits(16, true);
4198 end = this.currentPosition;
4199 if (hasNoDigitsBeforeDot && end == start) {
4200 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4201 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4202 }
4203 throw new InvalidInputException(INVALID_HEXA);
4204 }
4205
4206 if (getNextChar('p', 'P') >= 0) { // consume next character
4207 this.unicodeAsBackSlash = false;
4208 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
4209 && (this.source[this.currentPosition] == 'u')) {
4210 getNextUnicodeChar();
4211 } else {
4212 if (this.withoutUnicodePtr != 0) {
4213 unicodeStore();
4214 }
4215 }
4216
4217 if ((this.currentCharacter == '-')
4218 || (this.currentCharacter == '+')) { // consume next character
4219 this.unicodeAsBackSlash = false;
4220 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
4221 && (this.source[this.currentPosition] == 'u')) {
4222 getNextUnicodeChar();
4223 } else {
4224 if (this.withoutUnicodePtr != 0) {
4225 unicodeStore();
4226 }
4227 }
4228 }
4229 if (!ScannerHelper.isDigit(this.currentCharacter)) {
4230 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4231 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4232 }
4233 if (this.currentCharacter == '_') {
4234 // wrongly place '_'
4235 consumeDigits(10);
4236 throw new InvalidInputException(INVALID_UNDERSCORE);
4237 }
4238 throw new InvalidInputException(INVALID_HEXA);
4239 }
4240 consumeDigits(10);
4241 if (getNextChar('f', 'F') >= 0) {
4242 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4243 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4244 }
4245 return TokenNameFloatingPointLiteral;
4246 }
4247 if (getNextChar('d', 'D') >= 0) {
4248 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4249 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4250 }
4251 return TokenNameDoubleLiteral;
4252 }
4253 if (getNextChar('l', 'L') >= 0) {
4254 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4255 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4256 }
4257 throw new InvalidInputException(INVALID_HEXA);
4258 }
4259 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4260 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4261 }
4262 return TokenNameDoubleLiteral;
4263 } else {
4264 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4265 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4266 }
4267 throw new InvalidInputException(INVALID_HEXA);
4268 }
4269 } else if (getNextChar('p', 'P') >= 0) { // consume next character
4270 if (end == start) { // Has no digits before exponent
4271 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4272 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4273 }
4274 throw new InvalidInputException(INVALID_HEXA);
4275 }
4276 this.unicodeAsBackSlash = false;
4277 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
4278 && (this.source[this.currentPosition] == 'u')) {
4279 getNextUnicodeChar();
4280 } else {
4281 if (this.withoutUnicodePtr != 0) {
4282 unicodeStore();
4283 }
4284 }
4285
4286 if ((this.currentCharacter == '-')
4287 || (this.currentCharacter == '+')) { // consume next character
4288 this.unicodeAsBackSlash = false;
4289 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
4290 && (this.source[this.currentPosition] == 'u')) {
4291 getNextUnicodeChar();
4292 } else {
4293 if (this.withoutUnicodePtr != 0) {
4294 unicodeStore();
4295 }
4296 }
4297 }
4298 if (!ScannerHelper.isDigit(this.currentCharacter)) {
4299 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4300 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4301 }
4302 if (this.currentCharacter == '_') {
4303 // wrongly place '_'
4304 consumeDigits(10);
4305 throw new InvalidInputException(INVALID_UNDERSCORE);
4306 }
4307 throw new InvalidInputException(INVALID_FLOAT);
4308 }
4309 consumeDigits(10);
4310 if (getNextChar('f', 'F') >= 0) {
4311 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4312 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4313 }
4314 return TokenNameFloatingPointLiteral;
4315 }
4316 if (getNextChar('d', 'D') >= 0) {
4317 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4318 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4319 }
4320 return TokenNameDoubleLiteral;
4321 }
4322 if (getNextChar('l', 'L') >= 0) {
4323 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4324 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4325 }
4326 throw new InvalidInputException(INVALID_HEXA);
4327 }
4328 if (this.sourceLevel < ClassFileConstants.JDK1_5) {
4329 throw new InvalidInputException(ILLEGAL_HEXA_LITERAL);
4330 }
4331 return TokenNameDoubleLiteral;
4332 } else {
4333 if (end == start)
4334 throw new InvalidInputException(INVALID_HEXA);
4335 return TokenNameIntegerLiteral;
4336 }
4337 } else if (getNextChar('b', 'B') >= 0) { //----------binary-----------------
4338 int start = this.currentPosition;
4339 consumeDigits(2, true);
4340 int end = this.currentPosition;
4341 if (end == start) {
4342 if (this.sourceLevel < ClassFileConstants.JDK1_7) {
4343 throw new InvalidInputException(BINARY_LITERAL_NOT_BELOW_17);
4344 }
4345 throw new InvalidInputException(INVALID_BINARY);
4346 }
4347 if (getNextChar('l', 'L') >= 0) {
4348 if (this.sourceLevel < ClassFileConstants.JDK1_7) {
4349 throw new InvalidInputException(BINARY_LITERAL_NOT_BELOW_17);
4350 }
4351 return TokenNameLongLiteral;
4352 }
4353 if (this.sourceLevel < ClassFileConstants.JDK1_7) {
4354 throw new InvalidInputException(BINARY_LITERAL_NOT_BELOW_17);
4355 }
4356 return TokenNameIntegerLiteral;
4357 }
4358
4359 //there is no x or X nor b or B in the number
4360 //potential octal
4361 if (getNextCharAsDigit()) { //-------------potential octal-----------------
4362 consumeDigits(10);
4363
4364 if (getNextChar('l', 'L') >= 0) {
4365 return TokenNameLongLiteral;
4366 }
4367
4368 if (getNextChar('f', 'F') >= 0) {
4369 return TokenNameFloatingPointLiteral;
4370 }
4371
4372 if (getNextChar('d', 'D') >= 0) {
4373 return TokenNameDoubleLiteral;
4374 } else { //make the distinction between octal and float ....
4375 boolean isInteger = true;
4376 if (getNextChar('.')) {
4377 isInteger = false;
4378 consumeDigits(10);
4379 }
4380 if (getNextChar('e', 'E') >= 0) { // consume next character
4381 isInteger = false;
4382 this.unicodeAsBackSlash = false;
4383 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
4384 && (this.source[this.currentPosition] == 'u')) {
4385 getNextUnicodeChar();
4386 } else {
4387 if (this.withoutUnicodePtr != 0) {
4388 unicodeStore();
4389 }
4390 }
4391
4392 if ((this.currentCharacter == '-')
4393 || (this.currentCharacter == '+')) { // consume next character
4394 this.unicodeAsBackSlash = false;
4395 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
4396 && (this.source[this.currentPosition] == 'u')) {
4397 getNextUnicodeChar();
4398 } else {
4399 if (this.withoutUnicodePtr != 0) {
4400 unicodeStore();
4401 }
4402 }
4403 }
4404 if (!ScannerHelper.isDigit(this.currentCharacter)) {
4405 if (this.currentCharacter == '_') {
4406 // wrongly place '_'
4407 consumeDigits(10);
4408 throw new InvalidInputException(INVALID_UNDERSCORE);
4409 }
4410 throw new InvalidInputException(INVALID_FLOAT);
4411 }
4412 consumeDigits(10);
4413 }
4414 if (getNextChar('f', 'F') >= 0)
4415 return TokenNameFloatingPointLiteral;
4416 if (getNextChar('d', 'D') >= 0 || !isInteger)
4417 return TokenNameDoubleLiteral;
4418 return TokenNameIntegerLiteral;
4419 }
4420 } else {
4421 /* carry on */
4422 }
4423 }
4424
4425 consumeDigits(10);
4426
4427 if ((!dotPrefix) && (getNextChar('l', 'L') >= 0))
4428 return TokenNameLongLiteral;
4429
4430 if ((!dotPrefix) && (getNextChar('.'))) { //decimal part that can be empty
4431 consumeDigits(10, true);
4432 floating = true;
4433 }
4434
4435 //if floating is true both exponant and suffix may be optional
4436
4437 if (getNextChar('e', 'E') >= 0) {
4438 floating = true;
4439 // consume next character
4440 this.unicodeAsBackSlash = false;
4441 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
4442 && (this.source[this.currentPosition] == 'u')) {
4443 getNextUnicodeChar();
4444 } else {
4445 if (this.withoutUnicodePtr != 0) {
4446 unicodeStore();
4447 }
4448 }
4449
4450 if ((this.currentCharacter == '-')
4451 || (this.currentCharacter == '+')) { // consume next character
4452 this.unicodeAsBackSlash = false;
4453 if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
4454 && (this.source[this.currentPosition] == 'u')) {
4455 getNextUnicodeChar();
4456 } else {
4457 if (this.withoutUnicodePtr != 0) {
4458 unicodeStore();
4459 }
4460 }
4461 }
4462 if (!ScannerHelper.isDigit(this.currentCharacter)) {
4463 if (this.currentCharacter == '_') {
4464 // wrongly place '_'
4465 consumeDigits(10);
4466 throw new InvalidInputException(INVALID_UNDERSCORE);
4467 }
4468 throw new InvalidInputException(INVALID_FLOAT);
4469 }
4470 // current character is a digit so we expect no digit first (the next character could be an underscore)
4471 consumeDigits(10);
4472 }
4473
4474 if (getNextChar('d', 'D') >= 0)
4475 return TokenNameDoubleLiteral;
4476 if (getNextChar('f', 'F') >= 0)
4477 return TokenNameFloatingPointLiteral;
4478
4479 //the long flag has been tested before
4480
4481 return floating ? TokenNameDoubleLiteral : TokenNameIntegerLiteral;
4482 }
4483
4484 /**
4485 * Search the line number corresponding to a specific position
4486 * @param position int
4487 * @return int
4488 */
4489 public final int getLineNumber(int position) {
4490 return Util.getLineNumber(position, this.lineEnds, 0, this.linePtr);
4491 }
4492 public final void setSource(char[] sourceString){
4493 //the source-buffer is set to sourceString
4494
4495 int sourceLength;
4496 if (sourceString == null) {
4497 this.source = CharOperation.NO_CHAR;
4498 sourceLength = 0;
4499 } else {
4500 this.source = sourceString;
4501 sourceLength = sourceString.length;
4502 }
4503 this.startPosition = -1;
4504 this.eofPosition = sourceLength;
4505 this.initialPosition = this.currentPosition = 0;
4506 this.containsAssertKeyword = false;
4507 this.linePtr = -1;
4508 this.scanContext = null;
4509 this.yieldColons = -1;
4510 this.insideModuleInfo = false;
4511 }
4512 /*
4513 * Should be used if a parse (usually a diet parse) has already been performed on the unit,
4514 * so as to get the already computed line end positions.
4515 */
4516 public final void setSource(char[] contents, CompilationResult compilationResult) {
4517 if (contents == null) {
4518 char[] cuContents = compilationResult.compilationUnit.getContents();
4519 setSource(cuContents);
4520 } else {
4521 setSource(contents);
4522 }
4523 int[] lineSeparatorPositions = compilationResult.lineSeparatorPositions;
4524 if (lineSeparatorPositions != null) {
4525 this.lineEnds = lineSeparatorPositions;
4526 this.linePtr = lineSeparatorPositions.length - 1;
4527 }
4528 }
4529 /*
4530 * Should be used if a parse (usually a diet parse) has already been performed on the unit,
4531 * so as to get the already computed line end positions.
4532 */
4533 public final void setSource(CompilationResult compilationResult) {
4534 setSource(null, compilationResult);
4535 }
4536 @Override
4537 public String toString() {
4538 if (this.startPosition == this.eofPosition)
4539 return "EOF\n\n" + new String(this.source); //$NON-NLS-1$
4540 if (this.currentPosition > this.eofPosition)
4541 return "behind the EOF\n\n" + new String(this.source); //$NON-NLS-1$
4542 if (this.currentPosition <= 0)
4543 return "NOT started!\n\n"+ (this.source != null ? new String(this.source) : ""); //$NON-NLS-1$ //$NON-NLS-2$
4544
4545 StringBuffer buffer = new StringBuffer();
4546 if (this.startPosition < 1000) {
4547 buffer.append(this.source, 0, this.startPosition);
4548 } else {
4549 buffer.append("<source beginning>\n...\n"); //$NON-NLS-1$
4550 int line = Util.getLineNumber(this.startPosition-1000, this.lineEnds, 0, this.linePtr);
4551 int lineStart = getLineStart(line);
4552 buffer.append(this.source, lineStart, this.startPosition-lineStart);
4553 }
4554
4555 buffer.append("\n===============================\nStarts here -->"); //$NON-NLS-1$
4556 int middleLength = (this.currentPosition - 1) - this.startPosition + 1;
4557 if (middleLength > -1) {
4558 buffer.append(this.source, this.startPosition, middleLength);
4559 }
4560 if (this.nextToken != TerminalTokens.TokenNameNotAToken) {
4561 buffer.append("<-- Ends here [in pipeline " + toStringAction(this.nextToken) + "]\n===============================\n"); //$NON-NLS-1$ //$NON-NLS-2$
4562 } else {
4563 buffer.append("<-- Ends here\n===============================\n"); //$NON-NLS-1$
4564 }
4565
4566 buffer.append(this.source, (this.currentPosition - 1) + 1, this.eofPosition - (this.currentPosition - 1) - 1);
4567
4568 return buffer.toString();
4569 }
4570 public String toStringAction(int act) {
4571 switch (act) {
4572 case TokenNameIdentifier :
4573 return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4574 case TokenNameRestrictedIdentifierYield :
4575 return "yield"; //$NON-NLS-1$
4576 case TokenNameRestrictedIdentifierrecord :
4577 return "record"; //$NON-NLS-1$
4578 case TokenNameabstract :
4579 return "abstract"; //$NON-NLS-1$
4580 case TokenNameboolean :
4581 return "boolean"; //$NON-NLS-1$
4582 case TokenNamebreak :
4583 return "break"; //$NON-NLS-1$
4584 case TokenNamebyte :
4585 return "byte"; //$NON-NLS-1$
4586 case TokenNamecase :
4587 return "case"; //$NON-NLS-1$
4588 case TokenNamecatch :
4589 return "catch"; //$NON-NLS-1$
4590 case TokenNamechar :
4591 return "char"; //$NON-NLS-1$
4592 case TokenNameclass :
4593 return "class"; //$NON-NLS-1$
4594 case TokenNamecontinue :
4595 return "continue"; //$NON-NLS-1$
4596 case TokenNamedefault :
4597 return "default"; //$NON-NLS-1$
4598 case TokenNamedo :
4599 return "do"; //$NON-NLS-1$
4600 case TokenNamedouble :
4601 return "double"; //$NON-NLS-1$
4602 case TokenNameelse :
4603 return "else"; //$NON-NLS-1$
4604 case TokenNameextends :
4605 return "extends"; //$NON-NLS-1$
4606 case TokenNamefalse :
4607 return "false"; //$NON-NLS-1$
4608 case TokenNamefinal :
4609 return "final"; //$NON-NLS-1$
4610 case TokenNamefinally :
4611 return "finally"; //$NON-NLS-1$
4612 case TokenNamefloat :
4613 return "float"; //$NON-NLS-1$
4614 case TokenNamefor :
4615 return "for"; //$NON-NLS-1$
4616 case TokenNameif :
4617 return "if"; //$NON-NLS-1$
4618 case TokenNameimplements :
4619 return "implements"; //$NON-NLS-1$
4620 case TokenNameimport :
4621 return "import"; //$NON-NLS-1$
4622 case TokenNameinstanceof :
4623 return "instanceof"; //$NON-NLS-1$
4624 case TokenNameint :
4625 return "int"; //$NON-NLS-1$
4626 case TokenNameinterface :
4627 return "interface"; //$NON-NLS-1$
4628 case TokenNamelong :
4629 return "long"; //$NON-NLS-1$
4630 case TokenNamenative :
4631 return "native"; //$NON-NLS-1$
4632 case TokenNamenew :
4633 return "new"; //$NON-NLS-1$
4634 case TokenNamenull :
4635 return "null"; //$NON-NLS-1$
4636 case TokenNamepackage :
4637 return "package"; //$NON-NLS-1$
4638 case TokenNameprivate :
4639 return "private"; //$NON-NLS-1$
4640 case TokenNameprotected :
4641 return "protected"; //$NON-NLS-1$
4642 case TokenNamepublic :
4643 return "public"; //$NON-NLS-1$
4644 case TokenNamereturn :
4645 return "return"; //$NON-NLS-1$
4646 case TokenNameshort :
4647 return "short"; //$NON-NLS-1$
4648 case TokenNamestatic :
4649 return "static"; //$NON-NLS-1$
4650 case TokenNamesuper :
4651 return "super"; //$NON-NLS-1$
4652 case TokenNameswitch :
4653 return "switch"; //$NON-NLS-1$
4654 case TokenNamesynchronized :
4655 return "synchronized"; //$NON-NLS-1$
4656 case TokenNamethis :
4657 return "this"; //$NON-NLS-1$
4658 case TokenNamethrow :
4659 return "throw"; //$NON-NLS-1$
4660 case TokenNamethrows :
4661 return "throws"; //$NON-NLS-1$
4662 case TokenNametransient :
4663 return "transient"; //$NON-NLS-1$
4664 case TokenNametrue :
4665 return "true"; //$NON-NLS-1$
4666 case TokenNametry :
4667 return "try"; //$NON-NLS-1$
4668 case TokenNamevoid :
4669 return "void"; //$NON-NLS-1$
4670 case TokenNamevolatile :
4671 return "volatile"; //$NON-NLS-1$
4672 case TokenNamewhile :
4673 return "while"; //$NON-NLS-1$
4674 case TokenNamemodule :
4675 return "module"; //$NON-NLS-1$
4676 case TokenNamerequires :
4677 return "requires"; //$NON-NLS-1$
4678 case TokenNameexports :
4679 return "exports"; //$NON-NLS-1$
4680
4681 case TokenNameIntegerLiteral :
4682 return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4683 case TokenNameLongLiteral :
4684 return "Long(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4685 case TokenNameFloatingPointLiteral :
4686 return "Float(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4687 case TokenNameDoubleLiteral :
4688 return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4689 case TokenNameCharacterLiteral :
4690 return "Char(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4691 case TokenNameStringLiteral :
4692 return "String(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4693 case TokenNameTextBlock :
4694 return "String(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4695 case TokenNamePLUS_PLUS :
4696 return "++"; //$NON-NLS-1$
4697 case TokenNameMINUS_MINUS :
4698 return "--"; //$NON-NLS-1$
4699 case TokenNameEQUAL_EQUAL :
4700 return "=="; //$NON-NLS-1$
4701 case TokenNameLESS_EQUAL :
4702 return "<="; //$NON-NLS-1$
4703 case TokenNameGREATER_EQUAL :
4704 return ">="; //$NON-NLS-1$
4705 case TokenNameNOT_EQUAL :
4706 return "!="; //$NON-NLS-1$
4707 case TokenNameLEFT_SHIFT :
4708 return "<<"; //$NON-NLS-1$
4709 case TokenNameRIGHT_SHIFT :
4710 return ">>"; //$NON-NLS-1$
4711 case TokenNameUNSIGNED_RIGHT_SHIFT :
4712 return ">>>"; //$NON-NLS-1$
4713 case TokenNamePLUS_EQUAL :
4714 return "+="; //$NON-NLS-1$
4715 case TokenNameMINUS_EQUAL :
4716 return "-="; //$NON-NLS-1$
4717 case TokenNameARROW :
4718 return "->"; //$NON-NLS-1$
4719 case TokenNameMULTIPLY_EQUAL :
4720 return "*="; //$NON-NLS-1$
4721 case TokenNameDIVIDE_EQUAL :
4722 return "/="; //$NON-NLS-1$
4723 case TokenNameAND_EQUAL :
4724 return "&="; //$NON-NLS-1$
4725 case TokenNameOR_EQUAL :
4726 return "|="; //$NON-NLS-1$
4727 case TokenNameXOR_EQUAL :
4728 return "^="; //$NON-NLS-1$
4729 case TokenNameREMAINDER_EQUAL :
4730 return "%="; //$NON-NLS-1$
4731 case TokenNameLEFT_SHIFT_EQUAL :
4732 return "<<="; //$NON-NLS-1$
4733 case TokenNameRIGHT_SHIFT_EQUAL :
4734 return ">>="; //$NON-NLS-1$
4735 case TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL :
4736 return ">>>="; //$NON-NLS-1$
4737 case TokenNameOR_OR :
4738 return "||"; //$NON-NLS-1$
4739 case TokenNameAND_AND :
4740 return "&&"; //$NON-NLS-1$
4741 case TokenNamePLUS :
4742 return "+"; //$NON-NLS-1$
4743 case TokenNameMINUS :
4744 return "-"; //$NON-NLS-1$
4745 case TokenNameNOT :
4746 return "!"; //$NON-NLS-1$
4747 case TokenNameREMAINDER :
4748 return "%"; //$NON-NLS-1$
4749 case TokenNameXOR :
4750 return "^"; //$NON-NLS-1$
4751 case TokenNameAND :
4752 return "&"; //$NON-NLS-1$
4753 case TokenNameMULTIPLY :
4754 return "*"; //$NON-NLS-1$
4755 case TokenNameOR :
4756 return "|"; //$NON-NLS-1$
4757 case TokenNameTWIDDLE :
4758 return "~"; //$NON-NLS-1$
4759 case TokenNameDIVIDE :
4760 return "/"; //$NON-NLS-1$
4761 case TokenNameGREATER :
4762 return ">"; //$NON-NLS-1$
4763 case TokenNameLESS :
4764 return "<"; //$NON-NLS-1$
4765 case TokenNameLPAREN :
4766 return "("; //$NON-NLS-1$
4767 case TokenNameRPAREN :
4768 return ")"; //$NON-NLS-1$
4769 case TokenNameLBRACE :
4770 return "{"; //$NON-NLS-1$
4771 case TokenNameRBRACE :
4772 return "}"; //$NON-NLS-1$
4773 case TokenNameLBRACKET :
4774 return "["; //$NON-NLS-1$
4775 case TokenNameRBRACKET :
4776 return "]"; //$NON-NLS-1$
4777 case TokenNameSEMICOLON :
4778 return ";"; //$NON-NLS-1$
4779 case TokenNameQUESTION :
4780 return "?"; //$NON-NLS-1$
4781 case TokenNameCOLON :
4782 return ":"; //$NON-NLS-1$
4783 case TokenNameCOLON_COLON :
4784 return "::"; //$NON-NLS-1$
4785 case TokenNameCOMMA :
4786 return ","; //$NON-NLS-1$
4787 case TokenNameDOT :
4788 return "."; //$NON-NLS-1$
4789 case TokenNameEQUAL :
4790 return "="; //$NON-NLS-1$
4791 case TokenNameEOF :
4792 return "EOF"; //$NON-NLS-1$
4793 case TokenNameWHITESPACE :
4794 return "white_space(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4795 default :
4796 return "not-a-token"; //$NON-NLS-1$
4797 }
4798 }
4799 public void unicodeInitializeBuffer(int length) {
4800 this.withoutUnicodePtr = length;
4801 if (this.withoutUnicodeBuffer == null) this.withoutUnicodeBuffer = new char[length+(1+10)];
4802 int bLength = this.withoutUnicodeBuffer.length;
4803 if (1+length >= bLength) {
4804 System.arraycopy(this.withoutUnicodeBuffer, 0, this.withoutUnicodeBuffer = new char[length + (1+10)], 0, bLength);
4805 }
4806 System.arraycopy(this.source, this.startPosition, this.withoutUnicodeBuffer, 1, length);
4807 }
4808 public void unicodeStore() {
4809 int pos = ++this.withoutUnicodePtr;
4810 if (this.withoutUnicodeBuffer == null) this.withoutUnicodeBuffer = new char[10];
4811 int length = this.withoutUnicodeBuffer.length;
4812 if (pos == length) {
4813 System.arraycopy(this.withoutUnicodeBuffer, 0, this.withoutUnicodeBuffer = new char[length * 2], 0, length);
4814 }
4815 this.withoutUnicodeBuffer[pos] = this.currentCharacter;
4816 }
4817 public void unicodeStore(char character) {
4818 int pos = ++this.withoutUnicodePtr;
4819 if (this.withoutUnicodeBuffer == null) this.withoutUnicodeBuffer = new char[10];
4820 int length = this.withoutUnicodeBuffer.length;
4821 if (pos == length) {
4822 System.arraycopy(this.withoutUnicodeBuffer, 0, this.withoutUnicodeBuffer = new char[length * 2], 0, length);
4823 }
4824 this.withoutUnicodeBuffer[pos] = character;
4825 }
4826
4827 public static boolean isIdentifier(int token) {
4828 return token == TerminalTokens.TokenNameIdentifier;
4829 }
4830
4831 public static boolean isLiteral(int token) {
4832 switch(token) {
4833 case TerminalTokens.TokenNameIntegerLiteral:
4834 case TerminalTokens.TokenNameLongLiteral:
4835 case TerminalTokens.TokenNameFloatingPointLiteral:
4836 case TerminalTokens.TokenNameDoubleLiteral:
4837 case TerminalTokens.TokenNameStringLiteral:
4838 case TerminalTokens.TokenNameTextBlock:
4839 case TerminalTokens.TokenNameCharacterLiteral:
4840 return true;
4841 default:
4842 return false;
4843 }
4844 }
4845
4846 public static boolean isKeyword(int token) {
4847 switch(token) {
4848 case TerminalTokens.TokenNameabstract:
4849 case TerminalTokens.TokenNameassert:
4850 case TerminalTokens.TokenNamebyte:
4851 case TerminalTokens.TokenNamebreak:
4852 case TerminalTokens.TokenNameboolean:
4853 case TerminalTokens.TokenNamecase:
4854 case TerminalTokens.TokenNamechar:
4855 case TerminalTokens.TokenNamecatch:
4856 case TerminalTokens.TokenNameclass:
4857 case TerminalTokens.TokenNamecontinue:
4858 case TerminalTokens.TokenNamedo:
4859 case TerminalTokens.TokenNamedouble:
4860 case TerminalTokens.TokenNamedefault:
4861 case TerminalTokens.TokenNameelse:
4862 case TerminalTokens.TokenNameextends:
4863 case TerminalTokens.TokenNamefor:
4864 case TerminalTokens.TokenNamefinal:
4865 case TerminalTokens.TokenNamefloat:
4866 case TerminalTokens.TokenNamefalse:
4867 case TerminalTokens.TokenNamefinally:
4868 case TerminalTokens.TokenNameif:
4869 case TerminalTokens.TokenNameint:
4870 case TerminalTokens.TokenNameimport:
4871 case TerminalTokens.TokenNameinterface:
4872 case TerminalTokens.TokenNameimplements:
4873 case TerminalTokens.TokenNameinstanceof:
4874 case TerminalTokens.TokenNamelong:
4875 case TerminalTokens.TokenNamenew:
4876 case TerminalTokens.TokenNamenull:
4877 case TerminalTokens.TokenNamenative:
4878 case TerminalTokens.TokenNamepublic:
4879 case TerminalTokens.TokenNamepackage:
4880 case TerminalTokens.TokenNameprivate:
4881 case TerminalTokens.TokenNameprotected:
4882 case TerminalTokens.TokenNamereturn:
4883 case TerminalTokens.TokenNameshort:
4884 case TerminalTokens.TokenNamesuper:
4885 case TerminalTokens.TokenNamestatic:
4886 case TerminalTokens.TokenNameswitch:
4887 case TerminalTokens.TokenNamestrictfp:
4888 case TerminalTokens.TokenNamesynchronized:
4889 case TerminalTokens.TokenNametry:
4890 case TerminalTokens.TokenNamethis:
4891 case TerminalTokens.TokenNametrue:
4892 case TerminalTokens.TokenNamethrow:
4893 case TerminalTokens.TokenNamethrows:
4894 case TerminalTokens.TokenNametransient:
4895 case TerminalTokens.TokenNamevoid:
4896 case TerminalTokens.TokenNamevolatile:
4897 case TerminalTokens.TokenNamewhile:
4898 return true;
4899 case TerminalTokens.TokenNameRestrictedIdentifierYield:
4900 case TerminalTokens.TokenNameRestrictedIdentifierrecord:
4901 // making explicit - not a (restricted) keyword but restricted identifier.
4902 //$FALL-THROUGH$
4903 default:
4904 return false;
4905 }
4906 }
4907
4908 // Vanguard Scanner - A Private utility helper class for the scanner.
4909 private static final class VanguardScanner extends Scanner {
4910
4911 public VanguardScanner(long sourceLevel, long complianceLevel, boolean previewEnabled) {
4912 super (false /*comment*/, false /*whitespace*/, false /*nls*/, sourceLevel, complianceLevel, null/*taskTag*/,
4913 null/*taskPriorities*/, false /*taskCaseSensitive*/, previewEnabled);
4914 }
4915
4916 @Override
4917 public int getNextToken() throws InvalidInputException {
4918 int token;
4919 if (this.nextToken != TokenNameNotAToken) {
4920 token = this.nextToken;
4921 this.nextToken = TokenNameNotAToken;
4922 return token; // presumed to be unambiguous.
4923 }
4924 if (this.scanContext == null) { // init lazily, since isInModuleDeclaration may need the parser to be known
4925 this.scanContext = isInModuleDeclaration() ? ScanContext.EXPECTING_KEYWORD : ScanContext.INACTIVE;
4926 }
4927 token = getNextToken0();
4928 if (areRestrictedModuleKeywordsActive()) {
4929 if (isRestrictedKeyword(token))
4930 token = disambiguatedRestrictedKeyword(token);
4931 updateScanContext(token);
4932 }
4933 if (token == TokenNameAT && atTypeAnnotation()) {
4934 if (((VanguardParser) this.activeParser).currentGoal == Goal.LambdaParameterListGoal) {
4935 token = disambiguatedToken(token);
4936 } else {
4937 token = TokenNameAT308;
4938 }
4939 }
4940 return token == TokenNameEOF ? TokenNameNotAToken : token;
4941 }
4942 }
4943
4944 private static class Goal {
4945
4946 int first; // steer the parser towards a single minded pursuit.
4947 int [] follow; // the definite terminal symbols that signal the successful reduction to goal.
4948 int rule;
4949
4950 static int LambdaParameterListRule = 0;
4951 static int IntersectionCastRule = 0;
4952 static int ReferenceExpressionRule = 0;
4953 static int VarargTypeAnnotationsRule = 0;
4954 static int BlockStatementoptRule = 0;
4955 static int YieldStatementRule = 0;
4956
4957 static Goal LambdaParameterListGoal;
4958 static Goal IntersectionCastGoal;
4959 static Goal VarargTypeAnnotationGoal;
4960 static Goal ReferenceExpressionGoal;
4961 static Goal BlockStatementoptGoal;
4962 static Goal YieldStatementGoal;
4963
4964 static {
4965
4966 for (int i = 1; i <= ParserBasicInformation.NUM_RULES; i++) { // 0 == $acc
4967 if ("ParenthesizedLambdaParameterList".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$
4968 LambdaParameterListRule = i;
4969 else
4970 if ("ParenthesizedCastNameAndBounds".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$
4971 IntersectionCastRule = i;
4972 else
4973 if ("ReferenceExpressionTypeArgumentsAndTrunk".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$
4974 ReferenceExpressionRule = i;
4975 else
4976 if ("TypeAnnotations".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$
4977 VarargTypeAnnotationsRule = i;
4978 else
4979 if ("BlockStatementopt".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$
4980 BlockStatementoptRule = i;
4981 else
4982 if ("YieldStatement".equals(Parser.name[Parser.non_terminal_index[Parser.lhs[i]]])) //$NON-NLS-1$
4983 YieldStatementRule = i;
4984
4985 }
4986
4987 LambdaParameterListGoal = new Goal(TokenNameARROW, new int[] { TokenNameARROW }, LambdaParameterListRule);
4988 IntersectionCastGoal = new Goal(TokenNameLPAREN, followSetOfCast(), IntersectionCastRule);
4989 VarargTypeAnnotationGoal = new Goal(TokenNameAT, new int[] { TokenNameELLIPSIS }, VarargTypeAnnotationsRule);
4990 ReferenceExpressionGoal = new Goal(TokenNameLESS, new int[] { TokenNameCOLON_COLON }, ReferenceExpressionRule);
4991 BlockStatementoptGoal = new Goal(TokenNameLBRACE, new int [0], BlockStatementoptRule);
4992 YieldStatementGoal = new Goal(TokenNameARROW, new int [0], YieldStatementRule);
4993 }
4994
4995
4996 Goal(int first, int [] follow, int rule) {
4997 this.first = first;
4998 this.follow = follow;
4999 this.rule = rule;
5000 }
5001
5002 boolean hasBeenReached(int act, int token) {
5003 /*
5004 System.out.println("[Goal = " + Parser.name[Parser.non_terminal_index[Parser.lhs[this.rule]]] + "] " + "Saw: " + Parser.name[Parser.non_terminal_index[Parser.lhs[act]]] + "::" + //$NON-NLS-1$//$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
5005 Parser.name[Parser.terminal_index[token]]);
5006 */
5007 if (act == this.rule) {
5008 final int length = this.follow.length;
5009 if (length == 0)
5010 return true;
5011 for (int i = 0; i < length; i++)
5012 if (this.follow[i] == token)
5013 return true;
5014 }
5015 return false;
5016 }
5017
5018 private static int [] followSetOfCast() {
5019 return new int [] { TokenNameIdentifier, TokenNamenew, TokenNamesuper, TokenNamethis,
5020 TokenNamefalse, TokenNametrue, TokenNamenull,
5021 TokenNameIntegerLiteral, TokenNameLongLiteral, TokenNameFloatingPointLiteral, TokenNameDoubleLiteral, TokenNameCharacterLiteral, TokenNameStringLiteral, TokenNameTextBlock,
5022 TokenNameNOT, TokenNameTWIDDLE, TokenNameLPAREN
5023 };
5024 }
5025 }
5026 // Vanguard Parser - A Private utility helper class for the scanner.
5027 private static class VanguardParser extends Parser {
5028
5029 public static final boolean SUCCESS = true;
5030 public static final boolean FAILURE = false;
5031
5032 Goal currentGoal;
5033
5034 public VanguardParser(VanguardScanner scanner) {
5035 this.scanner = scanner;
5036 }
5037
5038 public VanguardParser(ProblemReporter reporter) {
5039 super(reporter, false);
5040 }
5041
5042 // Canonical LALR pushdown automaton identical to Parser.parse() minus side effects of any kind, returns the rule reduced.
5043 protected boolean parse(Goal goal) {
5044 this.currentGoal = goal;
5045 try {
5046 int act = START_STATE;
5047 this.stateStackTop = -1;
5048 this.currentToken = goal.first;
5049 ProcessTerminals : for (;;) {
5050 int stackLength = this.stack.length;
5051 if (++this.stateStackTop >= stackLength) {
5052 System.arraycopy(
5053 this.stack, 0,
5054 this.stack = new int[stackLength + StackIncrement], 0,
5055 stackLength);
5056 }
5057 this.stack[this.stateStackTop] = act;
5058
5059 act = Parser.tAction(act, this.currentToken);
5060 if (act == ERROR_ACTION) {
5061 return FAILURE;
5062 }
5063 if (act <= NUM_RULES) {
5064 this.stateStackTop--;
5065 } else if (act > ERROR_ACTION) { /* shift-reduce */
5066 this.unstackedAct = act;
5067 try {
5068 this.currentToken = this.scanner.getNextToken();
5069 } finally {
5070 this.unstackedAct = ERROR_ACTION;
5071 }
5072 act -= ERROR_ACTION;
5073 } else {
5074 if (act < ACCEPT_ACTION) { /* shift */
5075 this.unstackedAct = act;
5076 try {
5077 this.currentToken = this.scanner.getNextToken();
5078 } finally {
5079 this.unstackedAct = ERROR_ACTION;
5080 }
5081 continue ProcessTerminals;
5082 }
5083 return FAILURE; // accept - we should never reach this state, we accept at reduce with a right member of follow set below.
5084 }
5085
5086 // ProcessNonTerminals :
5087 do { /* reduce */
5088 if (goal.hasBeenReached(act, this.currentToken))
5089 return SUCCESS;
5090 this.stateStackTop -= (Parser.rhs[act] - 1);
5091 act = Parser.ntAction(this.stack[this.stateStackTop], Parser.lhs[act]);
5092 } while (act <= NUM_RULES);
5093 }
5094 } catch (Exception e) {
5095 return FAILURE;
5096 }
5097 }
5098 @Override
5099 public String toString() {
5100 return "\n\n\n----------------Scanner--------------\n" + this.scanner.toString(); //$NON-NLS-1$;
5101 }
5102 }
5103
5104 private class ScanContextDetector extends VanguardParser {
5105 ScanContextDetector(CompilerOptions options) {
5106 super(new ProblemReporter(
5107 DefaultErrorHandlingPolicies.ignoreAllProblems(),
5108 options,
5109 new DefaultProblemFactory()));
5110 this.problemReporter.options.performStatementsRecovery = false;
5111 this.reportSyntaxErrorIsRequired = false;
5112 this.reportOnlyOneSyntaxError = false;
5113 }
5114
5115 @Override
5116 public void initializeScanner(){
5117 this.scanner = new Scanner(
5118 false /*comment*/,
5119 false /*whitespace*/,
5120 false, /* will be set in initialize(boolean) */
5121 this.options.sourceLevel /*sourceLevel*/,
5122 this.options.complianceLevel /*complianceLevel*/,
5123 this.options.taskTags/*taskTags*/,
5124 this.options.taskPriorities/*taskPriorities*/,
5125 this.options.isTaskCaseSensitive/*taskCaseSensitive*/,
5126 this.options.enablePreviewFeatures /*isPreviewEnabled*/)
5127 {
5128 @Override
5129 void updateScanContext(int token) {
5130 if (token != TokenNameEOF)
5131 super.updateScanContext(token);
5132 }
5133 };
5134 this.scanner.recordLineSeparator = false;
5135 this.scanner.setActiveParser(this);
5136 this.scanner.previewEnabled = this.options.enablePreviewFeatures;
5137 }
5138
5139 @Override
5140 public boolean isParsingModuleDeclaration() {
5141 return true;
5142 }
5143
5144 public ScanContext getScanContext(char[] src, int begin) {
5145 this.scanner.setSource(src);
5146 this.scanner.resetTo(0, begin);
5147 goForCompilationUnit();
5148 Goal goal = new Goal(TokenNamePLUS_PLUS, null, 0) {
5149 @Override
5150 boolean hasBeenReached(int act, int token) {
5151 return token == TokenNameEOF;
5152 }
5153 };
5154 parse(goal);
5155 return this.scanner.scanContext;
5156 }
5157 }
5158
5159 private VanguardParser getVanguardParser() {
5160 if (this.vanguardParser == null) {
5161 this.vanguardScanner = new VanguardScanner(this.sourceLevel, this.complianceLevel, this.previewEnabled);
5162 this.vanguardParser = new VanguardParser(this.vanguardScanner);
5163 this.vanguardScanner.setActiveParser(this.vanguardParser);
5164 }
5165 this.vanguardScanner.setSource(this.source);
5166 this.vanguardScanner.resetTo(this.startPosition, this.eofPosition - 1, isInModuleDeclaration(), this.scanContext);
5167 return this.vanguardParser;
5168 }
5169 protected final boolean mayBeAtBreakPreview() {
5170 return this.breakPreviewAllowed && this.lookBack[1] != TokenNameARROW;
5171 }
5172
5173 protected final boolean maybeAtLambdaOrCast() { // Could the '(' we saw just now herald a lambda parameter list or a cast expression ? (the possible locations for both are identical.)
5174
5175 switch (this.lookBack[1]) {
5176 case TokenNameIdentifier:
5177 case TokenNamecatch:
5178 case TokenNamethis:
5179 case TokenNamesuper:
5180 case TokenNameif:
5181 case TokenNameswitch:
5182 case TokenNamewhile:
5183 case TokenNamefor:
5184 case TokenNamesynchronized:
5185 case TokenNametry:
5186 return false; // not a viable prefix for cast or lambda.
5187 default:
5188 return this.activeParser.atConflictScenario(TokenNameLPAREN);
5189 }
5190 }
5191
5192
5193 protected final boolean maybeAtReferenceExpression() { // Did the '<' we saw just now herald a reference expression's type arguments and trunk ?
5194 switch (this.lookBack[1]) {
5195 case TokenNameIdentifier:
5196 switch (this.lookBack[0]) {
5197 case TokenNameSEMICOLON: // for (int i = 0; i < 10; i++);
5198 case TokenNameRBRACE: // class X { void foo() {} X<String> x = null; }
5199 case TokenNameclass: // class X<T> {}
5200 case TokenNameinterface: // interface I<T> {}
5201 case TokenNameenum: // enum E<T> {}
5202 case TokenNamefinal: // final Collection<String>
5203 case TokenNameLESS: // Collection<IScalarData<AbstractData>>
5204 case TokenNameGREATER: // public <T> List<T> foo() { /* */ }
5205 case TokenNameRIGHT_SHIFT:// static <T extends SelfType<T>> List<T> makeSingletonList(T t) { /* */ }
5206 case TokenNamenew: // new ArrayList<String>();
5207 case TokenNamepublic: // public List<String> foo() {}
5208 case TokenNameabstract: // abstract List<String> foo() {}
5209 case TokenNameprivate: // private List<String> foo() {}
5210 case TokenNameprotected: // protected List<String> foo() {}
5211 case TokenNamestatic: // public static List<String> foo() {}
5212 case TokenNameextends: // <T extends Y<Z>>
5213 case TokenNamesuper: // ? super Context<N>
5214 case TokenNameAND: // T extends Object & Comparable<? super T>
5215 case TokenNameimplements: // class A implements I<Z>
5216 case TokenNamethrows: // throws Y<Z>
5217 case TokenNameAT: // @Deprecated <T> void foo() {}
5218 case TokenNameinstanceof: // if (o instanceof List<E>[])
5219 return false;
5220 default:
5221 break;
5222 }
5223 break;
5224 case TokenNameNotAToken: // Not kosher, don't touch.
5225 break;
5226 default:
5227 return false;
5228 }
5229 return this.activeParser.atConflictScenario(TokenNameLESS);
5230 }
5231 private final boolean maybeAtEllipsisAnnotationsStart() { // Did the '@' we saw just now herald a type annotation on a ... ? Presumed to be at type annotation already.
5232 if (this.consumingEllipsisAnnotations)
5233 return false;
5234 switch (this.lookBack[1]) {
5235 case TokenNamenew:
5236 case TokenNameCOMMA:
5237 case TokenNameextends:
5238 case TokenNamesuper:
5239 case TokenNameimplements:
5240 case TokenNameDOT:
5241 case TokenNameLBRACE:
5242 case TokenNameinstanceof:
5243 case TokenNameLESS:
5244 case TokenNameAND:
5245 case TokenNamethrows:
5246 return false;
5247 default:
5248 return true;
5249 }
5250 }
5251 protected final boolean atTypeAnnotation() { // Did the '@' we saw just now herald a type annotation ? We should not ask the parser whether it would shift @308 !
5252 return !this.activeParser.atConflictScenario(TokenNameAT);
5253 }
5254
5255 public void setActiveParser(ConflictedParser parser) {
5256 this.activeParser = parser;
5257 this.lookBack[0] = this.lookBack[1] = TokenNameNotAToken; // no hand me downs please.
5258 if (parser != null) {
5259 this.insideModuleInfo = parser.isParsingModuleDeclaration();
5260 }
5261 }
5262 public static boolean isRestrictedKeyword(int token) {
5263 switch(token) {
5264 case TokenNameopen:
5265 case TokenNamemodule:
5266 case TokenNamerequires:
5267 case TokenNametransitive:
5268 case TokenNameexports:
5269 case TokenNameto:
5270 case TokenNameopens:
5271 case TokenNameuses:
5272 case TokenNameprovides:
5273 case TokenNamewith:
5274 return true;
5275 default:
5276 return false;
5277 }
5278 }
5279 private boolean mayBeAtAnYieldStatement() {
5280 // preceded by ;, {, }, ), or -> [Ref: http://mail.openjdk.java.net/pipermail/amber-spec-experts/2019-May/001401.html]
5281 // above comment is super-seded by http://mail.openjdk.java.net/pipermail/amber-spec-experts/2019-May/001414.html
5282 switch (this.lookBack[1]) {
5283 case TokenNameLBRACE:
5284 case TokenNameRBRACE:
5285 case TokenNameRPAREN:
5286 case TokenNameSEMICOLON:
5287 case TokenNameelse:
5288 case TokenNamedo:
5289 return true;
5290 case TokenNameCOLON:
5291 return this.lookBack[0] == TokenNamedefault || this.yieldColons == 1;
5292 case TokenNameDOT:
5293 case TokenNameARROW:
5294 default:
5295 return false;
5296 }
5297 }
5298 int disambiguatedRestrictedIdentifierrecord(int restrictedIdentifierToken) {
5299 // and here's the kludge
5300 if (restrictedIdentifierToken != TokenNameRestrictedIdentifierrecord)
5301 return restrictedIdentifierToken;
5302 if (this.sourceLevel < ClassFileConstants.JDK14 || !this.previewEnabled)
5303 return TokenNameIdentifier;
5304
5305 return disambiguaterecordWithLookAhead() ?
5306 restrictedIdentifierToken : TokenNameIdentifier;
5307 }
5308 private int getNextTokenAfterTypeParameterHeader() {
5309 int count = 1;
5310 try {
5311 int token;
5312 while ((token = this.vanguardScanner.getNextToken()) != TokenNameNotAToken) {
5313 if (token == TokenNameEOF)
5314 break;
5315 if (token == TokenNameLESS)
5316 ++count;
5317 if (token == TokenNameGREATER)
5318 --count;
5319 if (token == TokenNameRIGHT_SHIFT)
5320 count= count -2;
5321 if (token == TokenNameUNSIGNED_RIGHT_SHIFT)
5322 count= count -3;
5323 if (count <= 0)
5324 return this.vanguardScanner.getNextToken();
5325 }
5326 } catch (InvalidInputException e) {
5327 if (e.getMessage().equals(INVALID_CHAR_IN_STRING)) {
5328 //Ignore
5329 } else {
5330 // Shouldn't happen, but log the error
5331 e.printStackTrace();
5332 }
5333 }
5334 return TokenNameEOF;
5335 }
5336 private boolean disambiguaterecordWithLookAhead() {
5337 getVanguardParser();
5338 this.vanguardScanner.resetTo(this.currentPosition, this.eofPosition - 1);
5339 try {
5340 int lookAhead1 = this.vanguardScanner.getNextToken();
5341 if (lookAhead1 == TokenNameIdentifier) {
5342 int lookAhead2 = this.vanguardScanner.getNextToken();
5343 lookAhead2 = lookAhead2 == TokenNameLESS ? getNextTokenAfterTypeParameterHeader() : lookAhead2;
5344 return lookAhead2 == TokenNameLPAREN;
5345 }
5346 } catch (InvalidInputException e) {
5347 if (e.getMessage().equals(INVALID_CHAR_IN_STRING)) {
5348 //Ignore
5349 } else {
5350 // Shouldn't happen, but log the error
5351 e.printStackTrace();
5352 }
5353 }
5354 return false; // IIE event;
5355 }
5356 private boolean disambiguateYieldWithLookAhead() {
5357 getVanguardParser();
5358 this.vanguardScanner.resetTo(this.currentPosition, this.eofPosition - 1);
5359 try {
5360 int lookAhead1 = this.vanguardScanner.getNextToken();
5361 switch (lookAhead1) {
5362 case TokenNameEQUAL_EQUAL :
5363 case TokenNameLESS_EQUAL :
5364 case TokenNameGREATER_EQUAL :
5365 case TokenNameNOT_EQUAL :
5366 case TokenNameLEFT_SHIFT :
5367 case TokenNameRIGHT_SHIFT :
5368 case TokenNameUNSIGNED_RIGHT_SHIFT :
5369 case TokenNamePLUS_EQUAL :
5370 case TokenNameMINUS_EQUAL :
5371 case TokenNameMULTIPLY_EQUAL :
5372 case TokenNameDIVIDE_EQUAL :
5373 case TokenNameAND_EQUAL :
5374 case TokenNameOR_EQUAL :
5375 case TokenNameXOR_EQUAL :
5376 case TokenNameREMAINDER_EQUAL :
5377 case TokenNameLEFT_SHIFT_EQUAL :
5378 case TokenNameRIGHT_SHIFT_EQUAL :
5379 case TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL :
5380 case TokenNameOR_OR :
5381 case TokenNameAND_AND :
5382 case TokenNameREMAINDER :
5383 case TokenNameXOR :
5384 case TokenNameAND :
5385 case TokenNameMULTIPLY :
5386 case TokenNameOR :
5387 case TokenNameTWIDDLE :
5388 case TokenNameDIVIDE :
5389 case TokenNameGREATER :
5390 case TokenNameLESS :
5391 case TokenNameLBRACE :
5392 case TokenNameRBRACE :
5393 case TokenNameLBRACKET :
5394 case TokenNameRBRACKET :
5395 case TokenNameSEMICOLON :
5396 case TokenNameQUESTION :
5397 case TokenNameCOLON :
5398 case TokenNameCOMMA :
5399 case TokenNameDOT :
5400 case TokenNameEQUAL :
5401 case TokenNameAT :
5402 case TokenNameELLIPSIS :
5403 case TokenNameARROW :
5404 case TokenNameCOLON_COLON :
5405 return false;
5406 case TokenNameMINUS_MINUS :
5407 case TokenNamePLUS_PLUS :
5408 int lookAhead2 = this.vanguardScanner.getNextToken();
5409 return lookAhead2 == TokenNameIdentifier;
5410 default : return true;
5411 }
5412 } catch (InvalidInputException e) {
5413 if (e.getMessage().equals(INVALID_CHAR_IN_STRING)) {
5414 //Ignore
5415 } else {
5416 // Shouldn't happen, but log the error
5417 e.printStackTrace();
5418 }
5419 }
5420 return false; // IIE event;
5421 }
5422 int disambiguatedRestrictedIdentifierYield(int restrictedIdentifierToken) {
5423 // and here's the kludge
5424 if (restrictedIdentifierToken != TokenNameRestrictedIdentifierYield)
5425 return restrictedIdentifierToken;
5426 if (this.sourceLevel < ClassFileConstants.JDK14)
5427 return TokenNameIdentifier;
5428
5429 return mayBeAtAnYieldStatement() && disambiguateYieldWithLookAhead() ?
5430 restrictedIdentifierToken : TokenNameIdentifier;
5431 }
5432 int disambiguatedRestrictedKeyword(int restrictedKeywordToken) {
5433 int token = restrictedKeywordToken;
5434 if (this.scanContext == ScanContext.EXPECTING_IDENTIFIER)
5435 return TokenNameIdentifier;
5436
5437 switch(restrictedKeywordToken) {
5438 case TokenNametransitive:
5439 if (this.scanContext != ScanContext.AFTER_REQUIRES) {
5440 token = TokenNameIdentifier;
5441 } else {
5442 getVanguardParser();
5443 this.vanguardScanner.resetTo(this.currentPosition, this.eofPosition - 1, true, ScanContext.EXPECTING_IDENTIFIER);
5444 try {
5445 int lookAhead = this.vanguardScanner.getNextToken();
5446 if (lookAhead == TokenNameSEMICOLON)
5447 token = TokenNameIdentifier;
5448 } catch (InvalidInputException e) {
5449 //
5450 }
5451 }
5452 break;
5453 case TokenNameopen:
5454 case TokenNamemodule:
5455 case TokenNameexports:
5456 case TokenNameopens:
5457 case TokenNamerequires:
5458 case TokenNameprovides:
5459 case TokenNameuses:
5460 case TokenNameto:
5461 case TokenNamewith:
5462 if (this.scanContext != ScanContext.EXPECTING_KEYWORD) {
5463 token = TokenNameIdentifier;
5464 }
5465 break;
5466 }
5467 return token;
5468 }
5469 int disambiguatedToken(int token) {
5470 final VanguardParser parser = getVanguardParser();
5471 if (token == TokenNameARROW && this.inCase) {
5472 this.nextToken = TokenNameARROW;
5473 this.inCase = false;
5474 return TokenNameBeginCaseExpr;
5475 } else if (token == TokenNameLPAREN && maybeAtLambdaOrCast()) {
5476 if (parser.parse(Goal.LambdaParameterListGoal) == VanguardParser.SUCCESS) {
5477 this.nextToken = TokenNameLPAREN;
5478 return TokenNameBeginLambda;
5479 }
5480 this.vanguardScanner.resetTo(this.startPosition, this.eofPosition - 1);
5481 if (parser.parse(Goal.IntersectionCastGoal) == VanguardParser.SUCCESS) {
5482 this.nextToken = TokenNameLPAREN;
5483 return TokenNameBeginIntersectionCast;
5484 }
5485 } else if (token == TokenNameLESS && maybeAtReferenceExpression()) {
5486 if (parser.parse(Goal.ReferenceExpressionGoal) == VanguardParser.SUCCESS) {
5487 this.nextToken = TokenNameLESS;
5488 return TokenNameBeginTypeArguments;
5489 }
5490 } else if (token == TokenNameAT && atTypeAnnotation()) {
5491 token = TokenNameAT308;
5492 if (maybeAtEllipsisAnnotationsStart()) {
5493 if (parser.parse(Goal.VarargTypeAnnotationGoal) == VanguardParser.SUCCESS) {
5494 this.consumingEllipsisAnnotations = true;
5495 this.nextToken = TokenNameAT308;
5496 return TokenNameAT308DOTDOTDOT;
5497 }
5498 }
5499 }
5500 return token;
5501 }
5502
5503 protected boolean isAtAssistIdentifier() {
5504 return false;
5505 }
5506
5507 // Position the scanner at the next block statement and return the start token. We recognize empty statements.
5508 public int fastForward(Statement unused) {
5509
5510 int token;
5511
5512 while (true) {
5513 try {
5514 token = getNextToken();
5515 } catch (InvalidInputException e) {
5516 return TokenNameEOF;
5517 }
5518 /* FOLLOW map of BlockStatement, since the non-terminal is recursive is a super set of its own FIRST set.
5519 We use FOLLOW rather than FIRST since we want to recognize empty statements. i.e if (x > 10) { x = 0 }
5520 */
5521 switch(token) {
5522 case TokenNameIdentifier:
5523 if (isAtAssistIdentifier()) // do not fast forward past the assist identifier ! We don't handle collections as of now.
5524 return token;
5525 //$FALL-THROUGH$
5526 case TokenNameabstract:
5527 case TokenNameassert:
5528 case TokenNameboolean:
5529 case TokenNamebreak:
5530 case TokenNamebyte:
5531 case TokenNamecase:
5532 case TokenNamechar:
5533 case TokenNameclass:
5534 case TokenNamecontinue:
5535 case TokenNamedefault:
5536 case TokenNamedo:
5537 case TokenNamedouble:
5538 case TokenNameenum:
5539 case TokenNamefalse:
5540 case TokenNamefinal:
5541 case TokenNamefloat:
5542 case TokenNamefor:
5543 case TokenNameif:
5544 case TokenNameint:
5545 case TokenNameinterface:
5546 case TokenNamelong:
5547 case TokenNamenative:
5548 case TokenNamenew:
5549 case TokenNamenull:
5550 case TokenNameprivate:
5551 case TokenNameprotected:
5552 case TokenNamepublic:
5553 case TokenNamereturn:
5554 case TokenNameshort:
5555 case TokenNamestatic:
5556 case TokenNamestrictfp:
5557 case TokenNamesuper:
5558 case TokenNameswitch:
5559 case TokenNamesynchronized:
5560 case TokenNamethis:
5561 case TokenNamethrow:
5562 case TokenNametransient:
5563 case TokenNametrue:
5564 case TokenNametry:
5565 case TokenNamevoid:
5566 case TokenNamevolatile:
5567 case TokenNamewhile:
5568 case TokenNameIntegerLiteral: // ??!
5569 case TokenNameLongLiteral:
5570 case TokenNameFloatingPointLiteral:
5571 case TokenNameDoubleLiteral:
5572 case TokenNameCharacterLiteral:
5573 case TokenNameStringLiteral:
5574 case TokenNameTextBlock:
5575 case TokenNamePLUS_PLUS:
5576 case TokenNameMINUS_MINUS:
5577 case TokenNameLESS:
5578 case TokenNameLPAREN:
5579 case TokenNameLBRACE:
5580 case TokenNameAT:
5581 case TokenNameBeginLambda:
5582 case TokenNameAT308:
5583 case TokenNameRestrictedIdentifierYield: // can be in FOLLOW of Block
5584 if(getVanguardParser().parse(Goal.BlockStatementoptGoal) == VanguardParser.SUCCESS)
5585 return token;
5586 break;
5587 case TokenNameSEMICOLON:
5588 case TokenNameEOF:
5589 return token;
5590 case TokenNameRBRACE: // simulate empty statement.
5591 ungetToken(token);
5592 return TokenNameSEMICOLON;
5593 default:
5594 break;
5595 }
5596 }
5597 }
5598
5599 /** Overridable hook, to allow CompletionScanner to hide a faked identifier token. */
5600 protected int getNextNotFakedToken() throws InvalidInputException {
5601 return getNextToken();
5602 }
5603 }
5604