1 /******************************************************************************* 2 * Copyright (c) 2014, 2015 Mateusz Matela and others. 3 * 4 * This program and the accompanying materials 5 * are made available under the terms of the Eclipse Public License 2.0 6 * which accompanies this distribution, and is available at 7 * https://www.eclipse.org/legal/epl-2.0/ 8 * 9 * SPDX-License-Identifier: EPL-2.0 10 * 11 * Contributors: 12 * Mateusz Matela <mateusz.matela@gmail.com> - [formatter] Formatter does not format Java code correctly, especially when max line width is set - https://bugs.eclipse.org/303519 13 * Till Brychcy - Java Code Formatter breaks code if single line comments contain unicode escape - https://bugs.eclipse.org/471090 14 *******************************************************************************/ 15 package org.eclipse.jdt.internal.formatter; 16 17 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_BLOCK; 18 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_JAVADOC; 19 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_LINE; 20 21 import java.util.List; 22 23 import org.eclipse.jdt.internal.compiler.parser.Scanner; 24 import org.eclipse.jdt.internal.compiler.parser.TerminalTokens; 25 26 /** 27 * Stores a token's type, position and all its properties like surrounding whitespace, wrapping behavior and so on. 28 */ 29 public class Token { 30 31 public static enum WrapMode { 32 /** 33 * Wrap mode for the "Do not wrap" policy. Tokens still should be indented as if wrapped when a preceding line 34 * break cannot be removed due to a line comment or formatting region restriction. 35 */ 36 DISABLED, 37 /** Wrap mode for the "Wrap where necessary" policies. */ 38 WHERE_NECESSARY, 39 /** Wrap mode for the "Wrap all elements" policies. */ 40 TOP_PRIORITY, 41 /** Wrap mode for tokens that must be wrapped due to "Force wrap" setting. */ 42 FORCE, 43 /** 44 * Wrap mode used for lines in anonymous class and lambda body. Means that tokens that are already in new line 45 * before wrapping, but their indentation should be adjusted in similar way to wrapping. 46 */ 47 BLOCK_INDENT 48 } 49 50 public static class WrapPolicy { 51 52 /** Policy used to mark tokens that should never be wrapped */ 53 public final static WrapPolicy DISABLE_WRAP = new WrapPolicy(WrapMode.DISABLED, 0, 0); 54 55 /** 56 * Policy used for internal structure of multiline comments to mark tokens that can be wrapped only in lines 57 * that have no other tokens to wrap. 58 */ 59 public final static WrapPolicy SUBSTITUTE_ONLY = new WrapPolicy(WrapMode.DISABLED, 0, 0); 60 61 /** Policy used to mark comments on first column that should not be indented. */ 62 public final static WrapPolicy FORCE_FIRST_COLUMN = new WrapPolicy(WrapMode.DISABLED, 0, 0); 63 64 public final WrapMode wrapMode; 65 public final int wrapParentIndex; 66 public final int groupEndIndex; 67 public final int extraIndent; 68 public final int structureDepth; 69 public final float penaltyMultiplier; 70 public final boolean isFirstInGroup; 71 public final boolean indentOnColumn; 72 WrapPolicy(WrapMode wrapMode, int wrapParentIndex, int groupEndIndex, int extraIndent, int structureDepth, float penaltyMultiplier, boolean isFirstInGroup, boolean indentOnColumn)73 public WrapPolicy(WrapMode wrapMode, int wrapParentIndex, int groupEndIndex, int extraIndent, 74 int structureDepth, float penaltyMultiplier, boolean isFirstInGroup, boolean indentOnColumn) { 75 assert wrapMode != null && (wrapParentIndex < groupEndIndex || groupEndIndex == -1); 76 77 this.wrapMode = wrapMode; 78 this.wrapParentIndex = wrapParentIndex; 79 this.groupEndIndex = groupEndIndex; 80 this.extraIndent = extraIndent; 81 this.structureDepth = structureDepth; 82 this.penaltyMultiplier = penaltyMultiplier; 83 this.isFirstInGroup = isFirstInGroup; 84 this.indentOnColumn = indentOnColumn; 85 } 86 87 public WrapPolicy(WrapMode wrapMode, int wrapParentIndex, int extraIndent) { 88 this(wrapMode, wrapParentIndex, -1, extraIndent, 0, 1, false, false); 89 } 90 } 91 92 /** Position in source of the first character. */ 93 public final int originalStart; 94 /** Position in source of the last character (this position is included in the token). */ 95 public final int originalEnd; 96 /** Type of this token. See {@link TerminalTokens} for constants definition. */ 97 public final int tokenType; 98 private boolean spaceBefore, spaceAfter; 99 private int lineBreaksBefore, lineBreaksAfter; 100 private boolean preserveLineBreaksBefore = true, preserveLineBreaksAfter = true; 101 private boolean wrapped; 102 private int indent; 103 private int emptyLineIndentAdjustment; 104 private int align; 105 private boolean toEscape; 106 107 private boolean nextLineOnWrap; 108 private WrapPolicy wrapPolicy; 109 110 private Token nlsTagToken; 111 112 private List<Token> internalStructure; 113 114 public Token(int sourceStart, int sourceEnd, int tokenType) { 115 assert sourceStart <= sourceEnd; 116 this.originalStart = sourceStart; 117 this.originalEnd = sourceEnd; 118 this.tokenType = tokenType; 119 } 120 121 public Token(Token tokenToCopy) { 122 this(tokenToCopy, tokenToCopy.originalStart, tokenToCopy.originalEnd, tokenToCopy.tokenType); 123 } 124 125 public Token(Token tokenToCopy, int newOriginalStart, int newOriginalEnd, int newTokenType) { 126 this.originalStart = newOriginalStart; 127 this.originalEnd = newOriginalEnd; 128 this.tokenType = newTokenType; 129 this.spaceBefore = tokenToCopy.spaceBefore; 130 this.spaceAfter = tokenToCopy.spaceAfter; 131 this.lineBreaksBefore = tokenToCopy.lineBreaksBefore; 132 this.lineBreaksAfter = tokenToCopy.lineBreaksAfter; 133 this.preserveLineBreaksBefore = tokenToCopy.preserveLineBreaksBefore; 134 this.preserveLineBreaksAfter = tokenToCopy.preserveLineBreaksAfter; 135 this.indent = tokenToCopy.indent; 136 this.nextLineOnWrap = tokenToCopy.nextLineOnWrap; 137 this.wrapPolicy = tokenToCopy.wrapPolicy; 138 this.nlsTagToken = tokenToCopy.nlsTagToken; 139 this.internalStructure = tokenToCopy.internalStructure; 140 } 141 142 public static Token fromCurrent(Scanner scanner, int currentToken) { 143 int start = scanner.getCurrentTokenStartPosition(); 144 int end = scanner.getCurrentTokenEndPosition(); 145 if (currentToken == TokenNameCOMMENT_LINE) { 146 // don't include line separator 147 while(end >= start) { 148 char c = scanner.source[end]; 149 if (c != '\r' && c != '\n') 150 break; 151 end--; 152 } 153 } 154 Token token = new Token(start, end, currentToken); 155 return token; 156 } 157 158 /** Adds space before this token */ 159 public void spaceBefore() { 160 this.spaceBefore = true; 161 } 162 163 /** Removes space before this token */ 164 public void clearSpaceBefore() { 165 this.spaceBefore = false; 166 } 167 168 public boolean isSpaceBefore() { 169 return this.spaceBefore; 170 } 171 172 /** Adds space after this token */ 173 public void spaceAfter() { 174 this.spaceAfter = true; 175 } 176 177 /** Removes space after this token */ 178 public void clearSpaceAfter() { 179 this.spaceAfter = false; 180 } 181 182 public boolean isSpaceAfter() { 183 return this.spaceAfter; 184 } 185 186 public void breakBefore() { 187 putLineBreaksBefore(1); 188 } 189 190 public void putLineBreaksBefore(int lineBreaks) { 191 this.lineBreaksBefore = Math.max(this.lineBreaksBefore, lineBreaks); 192 } 193 194 public int getLineBreaksBefore() { 195 return this.wrapped ? 1 : this.lineBreaksBefore; 196 } 197 198 /** Can be used to temporarily force preceding line break without losing the original number of line breaks. */ 199 public void setWrapped(boolean wrapped) { 200 this.wrapped = wrapped; 201 } 202 203 public void clearLineBreaksBefore() { 204 this.lineBreaksBefore = 0; 205 } 206 207 public void breakAfter() { 208 putLineBreaksAfter(1); 209 } 210 211 public void putLineBreaksAfter(int lineBreaks) { 212 this.lineBreaksAfter = Math.max(this.lineBreaksAfter, lineBreaks); 213 } 214 215 public int getLineBreaksAfter() { 216 return this.lineBreaksAfter; 217 } 218 219 public void clearLineBreaksAfter() { 220 this.lineBreaksAfter = 0; 221 } 222 223 public void setPreserveLineBreaksBefore(boolean preserveLineBreaksBefore) { 224 this.preserveLineBreaksBefore = preserveLineBreaksBefore; 225 } 226 227 public boolean isPreserveLineBreaksBefore() { 228 return this.preserveLineBreaksBefore; 229 } 230 231 public void setPreserveLineBreaksAfter(boolean preserveLineBreaksAfter) { 232 this.preserveLineBreaksAfter = preserveLineBreaksAfter; 233 } 234 235 public boolean isPreserveLineBreaksAfter() { 236 return this.preserveLineBreaksAfter; 237 } 238 239 /** Increases this token's indentation by one position */ 240 public void indent() { 241 this.indent++; 242 } 243 244 /** Decreses this token's indentation by one position */ 245 public void unindent() { 246 this.indent--; 247 } 248 249 public void setIndent(int indent) { 250 assert indent >= 0; 251 this.indent = indent; 252 } 253 254 public int getIndent() { 255 return this.indent; 256 } 257 258 public void setEmptyLineIndentAdjustment(int adjustment) { 259 this.emptyLineIndentAdjustment = adjustment; 260 } 261 262 public int getEmptyLineIndentAdjustment() { 263 return this.emptyLineIndentAdjustment; 264 } 265 266 public void setAlign(int align) { 267 this.align = align; 268 } 269 270 public int getAlign() { 271 return this.align; 272 } 273 274 public void setToEscape(boolean shouldEscape) { 275 this.toEscape = shouldEscape; 276 } 277 278 public boolean isToEscape() { 279 return this.toEscape; 280 } 281 282 public void setNextLineOnWrap() { 283 this.nextLineOnWrap = true; 284 } 285 286 public boolean isNextLineOnWrap() { 287 return this.nextLineOnWrap; 288 } 289 290 public void setWrapPolicy(WrapPolicy wrapPolicy) { 291 this.wrapPolicy = wrapPolicy; 292 } 293 294 public WrapPolicy getWrapPolicy() { 295 return this.wrapPolicy; 296 } 297 298 public boolean isWrappable() { 299 WrapPolicy wp = this.wrapPolicy; 300 return wp != null && wp.wrapMode != WrapMode.DISABLED && wp.wrapMode != WrapMode.BLOCK_INDENT; 301 } 302 303 public void setNLSTag(Token nlsTagToken) { 304 this.nlsTagToken = nlsTagToken; 305 } 306 307 public boolean hasNLSTag() { 308 return this.nlsTagToken != null; 309 } 310 311 public Token getNLSTag() { 312 return this.nlsTagToken; 313 } 314 315 public void setInternalStructure(List<Token> internalStructure) { 316 this.internalStructure = internalStructure; 317 } 318 319 public List<Token> getInternalStructure() { 320 return this.internalStructure; 321 } 322 323 public boolean isComment() { 324 switch (this.tokenType) { 325 case TokenNameCOMMENT_BLOCK: 326 case TokenNameCOMMENT_JAVADOC: 327 case TokenNameCOMMENT_LINE: 328 return true; 329 } 330 return false; 331 } 332 333 public String toString(String source) { 334 return source.substring(this.originalStart, this.originalEnd + 1); 335 } 336 337 public int countChars() { 338 return this.originalEnd - this.originalStart + 1; 339 } 340 341 /* 342 * Conceptually, Token abstracts away from the source so it doesn't need to know how 343 * the source looks like. However, it's useful to see the actual token contents while debugging. 344 * Uncomment this field, commented code in toString() below and in DefaultCodeFormatter.init(String source) 345 * during debugging sessions to easily recognize tokens. 346 */ 347 // public static String source; 348 349 @Override 350 public String toString() { 351 // if (source != null) // see comment above 352 // return toString(source); 353 return "[" + this.originalStart + "-" + this.originalEnd + "]"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ 354 } 355 } 356