1 //===--- FormatToken.cpp - Format C++ code --------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements specific functions of \c FormatTokens and their
11 /// roles.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #include "FormatToken.h"
16 #include "ContinuationIndenter.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/Support/Debug.h"
19 #include <climits>
20 
21 namespace clang {
22 namespace format {
23 
getTokenTypeName(TokenType Type)24 const char *getTokenTypeName(TokenType Type) {
25   static const char *const TokNames[] = {
26 #define TYPE(X) #X,
27       LIST_TOKEN_TYPES
28 #undef TYPE
29       nullptr};
30 
31   if (Type < NUM_TOKEN_TYPES)
32     return TokNames[Type];
33   llvm_unreachable("unknown TokenType");
34   return nullptr;
35 }
36 
37 // FIXME: This is copy&pasted from Sema. Put it in a common place and remove
38 // duplication.
isSimpleTypeSpecifier() const39 bool FormatToken::isSimpleTypeSpecifier() const {
40   switch (Tok.getKind()) {
41   case tok::kw_short:
42   case tok::kw_long:
43   case tok::kw___int64:
44   case tok::kw___int128:
45   case tok::kw_signed:
46   case tok::kw_unsigned:
47   case tok::kw_void:
48   case tok::kw_char:
49   case tok::kw_int:
50   case tok::kw_half:
51   case tok::kw_float:
52   case tok::kw_double:
53   case tok::kw___bf16:
54   case tok::kw__Float16:
55   case tok::kw___float128:
56   case tok::kw___ibm128:
57   case tok::kw_wchar_t:
58   case tok::kw_bool:
59 #define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
60 #include "clang/Basic/TransformTypeTraits.def"
61   case tok::annot_typename:
62   case tok::kw_char8_t:
63   case tok::kw_char16_t:
64   case tok::kw_char32_t:
65   case tok::kw_typeof:
66   case tok::kw_decltype:
67   case tok::kw__Atomic:
68     return true;
69   default:
70     return false;
71   }
72 }
73 
isTypeOrIdentifier() const74 bool FormatToken::isTypeOrIdentifier() const {
75   return isSimpleTypeSpecifier() || Tok.isOneOf(tok::kw_auto, tok::identifier);
76 }
77 
isBlockIndentedInitRBrace(const FormatStyle & Style) const78 bool FormatToken::isBlockIndentedInitRBrace(const FormatStyle &Style) const {
79   assert(is(tok::r_brace));
80   if (!Style.Cpp11BracedListStyle ||
81       Style.AlignAfterOpenBracket != FormatStyle::BAS_BlockIndent) {
82     return false;
83   }
84   const auto *LBrace = MatchingParen;
85   assert(LBrace && LBrace->is(tok::l_brace));
86   if (LBrace->is(BK_BracedInit))
87     return true;
88   if (LBrace->Previous && LBrace->Previous->is(tok::equal))
89     return true;
90   return false;
91 }
92 
opensBlockOrBlockTypeList(const FormatStyle & Style) const93 bool FormatToken::opensBlockOrBlockTypeList(const FormatStyle &Style) const {
94   // C# Does not indent object initialisers as continuations.
95   if (is(tok::l_brace) && getBlockKind() == BK_BracedInit && Style.isCSharp())
96     return true;
97   if (is(TT_TemplateString) && opensScope())
98     return true;
99   return is(TT_ArrayInitializerLSquare) || is(TT_ProtoExtensionLSquare) ||
100          (is(tok::l_brace) &&
101           (getBlockKind() == BK_Block || is(TT_DictLiteral) ||
102            (!Style.Cpp11BracedListStyle && NestingLevel == 0))) ||
103          (is(tok::less) && Style.isProto());
104 }
105 
~TokenRole()106 TokenRole::~TokenRole() {}
107 
precomputeFormattingInfos(const FormatToken * Token)108 void TokenRole::precomputeFormattingInfos(const FormatToken *Token) {}
109 
formatAfterToken(LineState & State,ContinuationIndenter * Indenter,bool DryRun)110 unsigned CommaSeparatedList::formatAfterToken(LineState &State,
111                                               ContinuationIndenter *Indenter,
112                                               bool DryRun) {
113   if (!State.NextToken || !State.NextToken->Previous)
114     return 0;
115 
116   if (Formats.size() <= 1)
117     return 0; // Handled by formatFromToken (1) or avoid severe penalty (0).
118 
119   // Ensure that we start on the opening brace.
120   const FormatToken *LBrace =
121       State.NextToken->Previous->getPreviousNonComment();
122   if (!LBrace || !LBrace->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
123       LBrace->is(BK_Block) || LBrace->is(TT_DictLiteral) ||
124       LBrace->Next->is(TT_DesignatedInitializerPeriod)) {
125     return 0;
126   }
127 
128   // Calculate the number of code points we have to format this list. As the
129   // first token is already placed, we have to subtract it.
130   unsigned RemainingCodePoints =
131       Style.ColumnLimit - State.Column + State.NextToken->Previous->ColumnWidth;
132 
133   // Find the best ColumnFormat, i.e. the best number of columns to use.
134   const ColumnFormat *Format = getColumnFormat(RemainingCodePoints);
135 
136   // If no ColumnFormat can be used, the braced list would generally be
137   // bin-packed. Add a severe penalty to this so that column layouts are
138   // preferred if possible.
139   if (!Format)
140     return 10000;
141 
142   // Format the entire list.
143   unsigned Penalty = 0;
144   unsigned Column = 0;
145   unsigned Item = 0;
146   while (State.NextToken != LBrace->MatchingParen) {
147     bool NewLine = false;
148     unsigned ExtraSpaces = 0;
149 
150     // If the previous token was one of our commas, we are now on the next item.
151     if (Item < Commas.size() && State.NextToken->Previous == Commas[Item]) {
152       if (!State.NextToken->isTrailingComment()) {
153         ExtraSpaces += Format->ColumnSizes[Column] - ItemLengths[Item];
154         ++Column;
155       }
156       ++Item;
157     }
158 
159     if (Column == Format->Columns || State.NextToken->MustBreakBefore) {
160       Column = 0;
161       NewLine = true;
162     }
163 
164     // Place token using the continuation indenter and store the penalty.
165     Penalty += Indenter->addTokenToState(State, NewLine, DryRun, ExtraSpaces);
166   }
167   return Penalty;
168 }
169 
formatFromToken(LineState & State,ContinuationIndenter * Indenter,bool DryRun)170 unsigned CommaSeparatedList::formatFromToken(LineState &State,
171                                              ContinuationIndenter *Indenter,
172                                              bool DryRun) {
173   // Formatting with 1 Column isn't really a column layout, so we don't need the
174   // special logic here. We can just avoid bin packing any of the parameters.
175   if (Formats.size() == 1 || HasNestedBracedList)
176     State.Stack.back().AvoidBinPacking = true;
177   return 0;
178 }
179 
180 // Returns the lengths in code points between Begin and End (both included),
181 // assuming that the entire sequence is put on a single line.
CodePointsBetween(const FormatToken * Begin,const FormatToken * End)182 static unsigned CodePointsBetween(const FormatToken *Begin,
183                                   const FormatToken *End) {
184   assert(End->TotalLength >= Begin->TotalLength);
185   return End->TotalLength - Begin->TotalLength + Begin->ColumnWidth;
186 }
187 
precomputeFormattingInfos(const FormatToken * Token)188 void CommaSeparatedList::precomputeFormattingInfos(const FormatToken *Token) {
189   // FIXME: At some point we might want to do this for other lists, too.
190   if (!Token->MatchingParen ||
191       !Token->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare)) {
192     return;
193   }
194 
195   // In C++11 braced list style, we should not format in columns unless they
196   // have many items (20 or more) or we allow bin-packing of function call
197   // arguments.
198   if (Style.Cpp11BracedListStyle && !Style.BinPackArguments &&
199       Commas.size() < 19) {
200     return;
201   }
202 
203   // Limit column layout for JavaScript array initializers to 20 or more items
204   // for now to introduce it carefully. We can become more aggressive if this
205   // necessary.
206   if (Token->is(TT_ArrayInitializerLSquare) && Commas.size() < 19)
207     return;
208 
209   // Column format doesn't really make sense if we don't align after brackets.
210   if (Style.AlignAfterOpenBracket == FormatStyle::BAS_DontAlign)
211     return;
212 
213   FormatToken *ItemBegin = Token->Next;
214   while (ItemBegin->isTrailingComment())
215     ItemBegin = ItemBegin->Next;
216   SmallVector<bool, 8> MustBreakBeforeItem;
217 
218   // The lengths of an item if it is put at the end of the line. This includes
219   // trailing comments which are otherwise ignored for column alignment.
220   SmallVector<unsigned, 8> EndOfLineItemLength;
221   MustBreakBeforeItem.reserve(Commas.size() + 1);
222   EndOfLineItemLength.reserve(Commas.size() + 1);
223   ItemLengths.reserve(Commas.size() + 1);
224 
225   bool HasSeparatingComment = false;
226   for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) {
227     assert(ItemBegin);
228     // Skip comments on their own line.
229     while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment()) {
230       ItemBegin = ItemBegin->Next;
231       HasSeparatingComment = i > 0;
232     }
233 
234     MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore);
235     if (ItemBegin->is(tok::l_brace))
236       HasNestedBracedList = true;
237     const FormatToken *ItemEnd = nullptr;
238     if (i == Commas.size()) {
239       ItemEnd = Token->MatchingParen;
240       const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment();
241       ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd));
242       if (Style.Cpp11BracedListStyle &&
243           !ItemEnd->Previous->isTrailingComment()) {
244         // In Cpp11 braced list style, the } and possibly other subsequent
245         // tokens will need to stay on a line with the last element.
246         while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore)
247           ItemEnd = ItemEnd->Next;
248       } else {
249         // In other braced lists styles, the "}" can be wrapped to the new line.
250         ItemEnd = Token->MatchingParen->Previous;
251       }
252     } else {
253       ItemEnd = Commas[i];
254       // The comma is counted as part of the item when calculating the length.
255       ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd));
256 
257       // Consume trailing comments so the are included in EndOfLineItemLength.
258       if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline &&
259           ItemEnd->Next->isTrailingComment()) {
260         ItemEnd = ItemEnd->Next;
261       }
262     }
263     EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd));
264     // If there is a trailing comma in the list, the next item will start at the
265     // closing brace. Don't create an extra item for this.
266     if (ItemEnd->getNextNonComment() == Token->MatchingParen)
267       break;
268     ItemBegin = ItemEnd->Next;
269   }
270 
271   // Don't use column layout for lists with few elements and in presence of
272   // separating comments.
273   if (Commas.size() < 5 || HasSeparatingComment)
274     return;
275 
276   if (Token->NestingLevel != 0 && Token->is(tok::l_brace) && Commas.size() < 19)
277     return;
278 
279   // We can never place more than ColumnLimit / 3 items in a row (because of the
280   // spaces and the comma).
281   unsigned MaxItems = Style.ColumnLimit / 3;
282   SmallVector<unsigned> MinSizeInColumn;
283   MinSizeInColumn.reserve(MaxItems);
284   for (unsigned Columns = 1; Columns <= MaxItems; ++Columns) {
285     ColumnFormat Format;
286     Format.Columns = Columns;
287     Format.ColumnSizes.resize(Columns);
288     MinSizeInColumn.assign(Columns, UINT_MAX);
289     Format.LineCount = 1;
290     bool HasRowWithSufficientColumns = false;
291     unsigned Column = 0;
292     for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
293       assert(i < MustBreakBeforeItem.size());
294       if (MustBreakBeforeItem[i] || Column == Columns) {
295         ++Format.LineCount;
296         Column = 0;
297       }
298       if (Column == Columns - 1)
299         HasRowWithSufficientColumns = true;
300       unsigned Length =
301           (Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i];
302       Format.ColumnSizes[Column] = std::max(Format.ColumnSizes[Column], Length);
303       MinSizeInColumn[Column] = std::min(MinSizeInColumn[Column], Length);
304       ++Column;
305     }
306     // If all rows are terminated early (e.g. by trailing comments), we don't
307     // need to look further.
308     if (!HasRowWithSufficientColumns)
309       break;
310     Format.TotalWidth = Columns - 1; // Width of the N-1 spaces.
311 
312     for (unsigned i = 0; i < Columns; ++i)
313       Format.TotalWidth += Format.ColumnSizes[i];
314 
315     // Don't use this Format, if the difference between the longest and shortest
316     // element in a column exceeds a threshold to avoid excessive spaces.
317     if ([&] {
318           for (unsigned i = 0; i < Columns - 1; ++i)
319             if (Format.ColumnSizes[i] - MinSizeInColumn[i] > 10)
320               return true;
321           return false;
322         }()) {
323       continue;
324     }
325 
326     // Ignore layouts that are bound to violate the column limit.
327     if (Format.TotalWidth > Style.ColumnLimit && Columns > 1)
328       continue;
329 
330     Formats.push_back(Format);
331   }
332 }
333 
334 const CommaSeparatedList::ColumnFormat *
getColumnFormat(unsigned RemainingCharacters) const335 CommaSeparatedList::getColumnFormat(unsigned RemainingCharacters) const {
336   const ColumnFormat *BestFormat = nullptr;
337   for (const ColumnFormat &Format : llvm::reverse(Formats)) {
338     if (Format.TotalWidth <= RemainingCharacters || Format.Columns == 1) {
339       if (BestFormat && Format.LineCount > BestFormat->LineCount)
340         break;
341       BestFormat = &Format;
342     }
343   }
344   return BestFormat;
345 }
346 
347 } // namespace format
348 } // namespace clang
349