1 //===--- LiteralSupport.cpp - Code to parse and process literals ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the NumericLiteralParser, CharLiteralParser, and
10 // StringLiteralParser interfaces.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Lex/LiteralSupport.h"
15 #include "clang/Basic/CharInfo.h"
16 #include "clang/Basic/LangOptions.h"
17 #include "clang/Basic/SourceLocation.h"
18 #include "clang/Basic/TargetInfo.h"
19 #include "clang/Lex/LexDiagnostic.h"
20 #include "clang/Lex/Lexer.h"
21 #include "clang/Lex/Preprocessor.h"
22 #include "clang/Lex/Token.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/StringExtras.h"
26 #include "llvm/ADT/StringSwitch.h"
27 #include "llvm/Support/ConvertUTF.h"
28 #include "llvm/Support/Error.h"
29 #include "llvm/Support/ErrorHandling.h"
30 #include "llvm/Support/Unicode.h"
31 #include <algorithm>
32 #include <cassert>
33 #include <cstddef>
34 #include <cstdint>
35 #include <cstring>
36 #include <string>
37 
38 using namespace clang;
39 
getCharWidth(tok::TokenKind kind,const TargetInfo & Target)40 static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) {
41   switch (kind) {
42   default: llvm_unreachable("Unknown token type!");
43   case tok::char_constant:
44   case tok::string_literal:
45   case tok::utf8_char_constant:
46   case tok::utf8_string_literal:
47     return Target.getCharWidth();
48   case tok::wide_char_constant:
49   case tok::wide_string_literal:
50     return Target.getWCharWidth();
51   case tok::utf16_char_constant:
52   case tok::utf16_string_literal:
53     return Target.getChar16Width();
54   case tok::utf32_char_constant:
55   case tok::utf32_string_literal:
56     return Target.getChar32Width();
57   }
58 }
59 
MakeCharSourceRange(const LangOptions & Features,FullSourceLoc TokLoc,const char * TokBegin,const char * TokRangeBegin,const char * TokRangeEnd)60 static CharSourceRange MakeCharSourceRange(const LangOptions &Features,
61                                            FullSourceLoc TokLoc,
62                                            const char *TokBegin,
63                                            const char *TokRangeBegin,
64                                            const char *TokRangeEnd) {
65   SourceLocation Begin =
66     Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin,
67                                    TokLoc.getManager(), Features);
68   SourceLocation End =
69     Lexer::AdvanceToTokenCharacter(Begin, TokRangeEnd - TokRangeBegin,
70                                    TokLoc.getManager(), Features);
71   return CharSourceRange::getCharRange(Begin, End);
72 }
73 
74 /// Produce a diagnostic highlighting some portion of a literal.
75 ///
76 /// Emits the diagnostic \p DiagID, highlighting the range of characters from
77 /// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be
78 /// a substring of a spelling buffer for the token beginning at \p TokBegin.
Diag(DiagnosticsEngine * Diags,const LangOptions & Features,FullSourceLoc TokLoc,const char * TokBegin,const char * TokRangeBegin,const char * TokRangeEnd,unsigned DiagID)79 static DiagnosticBuilder Diag(DiagnosticsEngine *Diags,
80                               const LangOptions &Features, FullSourceLoc TokLoc,
81                               const char *TokBegin, const char *TokRangeBegin,
82                               const char *TokRangeEnd, unsigned DiagID) {
83   SourceLocation Begin =
84     Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin,
85                                    TokLoc.getManager(), Features);
86   return Diags->Report(Begin, DiagID) <<
87     MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd);
88 }
89 
90 /// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
91 /// either a character or a string literal.
ProcessCharEscape(const char * ThisTokBegin,const char * & ThisTokBuf,const char * ThisTokEnd,bool & HadError,FullSourceLoc Loc,unsigned CharWidth,DiagnosticsEngine * Diags,const LangOptions & Features)92 static unsigned ProcessCharEscape(const char *ThisTokBegin,
93                                   const char *&ThisTokBuf,
94                                   const char *ThisTokEnd, bool &HadError,
95                                   FullSourceLoc Loc, unsigned CharWidth,
96                                   DiagnosticsEngine *Diags,
97                                   const LangOptions &Features) {
98   const char *EscapeBegin = ThisTokBuf;
99   bool Delimited = false;
100   bool EndDelimiterFound = false;
101 
102   // Skip the '\' char.
103   ++ThisTokBuf;
104 
105   // We know that this character can't be off the end of the buffer, because
106   // that would have been \", which would not have been the end of string.
107   unsigned ResultChar = *ThisTokBuf++;
108   switch (ResultChar) {
109   // These map to themselves.
110   case '\\': case '\'': case '"': case '?': break;
111 
112     // These have fixed mappings.
113   case 'a':
114     // TODO: K&R: the meaning of '\\a' is different in traditional C
115     ResultChar = 7;
116     break;
117   case 'b':
118     ResultChar = 8;
119     break;
120   case 'e':
121     if (Diags)
122       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
123            diag::ext_nonstandard_escape) << "e";
124     ResultChar = 27;
125     break;
126   case 'E':
127     if (Diags)
128       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
129            diag::ext_nonstandard_escape) << "E";
130     ResultChar = 27;
131     break;
132   case 'f':
133     ResultChar = 12;
134     break;
135   case 'n':
136     ResultChar = 10;
137     break;
138   case 'r':
139     ResultChar = 13;
140     break;
141   case 't':
142     ResultChar = 9;
143     break;
144   case 'v':
145     ResultChar = 11;
146     break;
147   case 'x': { // Hex escape.
148     ResultChar = 0;
149     if (ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
150       Delimited = true;
151       ThisTokBuf++;
152       if (*ThisTokBuf == '}') {
153         Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
154              diag::err_delimited_escape_empty);
155         return ResultChar;
156       }
157     } else if (ThisTokBuf == ThisTokEnd || !isHexDigit(*ThisTokBuf)) {
158       if (Diags)
159         Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
160              diag::err_hex_escape_no_digits) << "x";
161       return ResultChar;
162     }
163 
164     // Hex escapes are a maximal series of hex digits.
165     bool Overflow = false;
166     for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) {
167       if (Delimited && *ThisTokBuf == '}') {
168         ThisTokBuf++;
169         EndDelimiterFound = true;
170         break;
171       }
172       int CharVal = llvm::hexDigitValue(*ThisTokBuf);
173       if (CharVal == -1) {
174         // Non delimited hex escape sequences stop at the first non-hex digit.
175         if (!Delimited)
176           break;
177         HadError = true;
178         if (Diags)
179           Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
180                diag::err_delimited_escape_invalid)
181               << StringRef(ThisTokBuf, 1);
182         continue;
183       }
184       // About to shift out a digit?
185       if (ResultChar & 0xF0000000)
186         Overflow = true;
187       ResultChar <<= 4;
188       ResultChar |= CharVal;
189     }
190     // See if any bits will be truncated when evaluated as a character.
191     if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
192       Overflow = true;
193       ResultChar &= ~0U >> (32-CharWidth);
194     }
195 
196     // Check for overflow.
197     if (!HadError && Overflow) { // Too many digits to fit in
198       HadError = true;
199       if (Diags)
200         Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
201              diag::err_escape_too_large)
202             << 0;
203     }
204     break;
205   }
206   case '0': case '1': case '2': case '3':
207   case '4': case '5': case '6': case '7': {
208     // Octal escapes.
209     --ThisTokBuf;
210     ResultChar = 0;
211 
212     // Octal escapes are a series of octal digits with maximum length 3.
213     // "\0123" is a two digit sequence equal to "\012" "3".
214     unsigned NumDigits = 0;
215     do {
216       ResultChar <<= 3;
217       ResultChar |= *ThisTokBuf++ - '0';
218       ++NumDigits;
219     } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 &&
220              ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
221 
222     // Check for overflow.  Reject '\777', but not L'\777'.
223     if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
224       if (Diags)
225         Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
226              diag::err_escape_too_large) << 1;
227       ResultChar &= ~0U >> (32-CharWidth);
228     }
229     break;
230   }
231   case 'o': {
232     bool Overflow = false;
233     if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') {
234       HadError = true;
235       if (Diags)
236         Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
237              diag::err_delimited_escape_missing_brace)
238             << "o";
239 
240       break;
241     }
242     ResultChar = 0;
243     Delimited = true;
244     ++ThisTokBuf;
245     if (*ThisTokBuf == '}') {
246       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
247            diag::err_delimited_escape_empty);
248       return ResultChar;
249     }
250 
251     while (ThisTokBuf != ThisTokEnd) {
252       if (*ThisTokBuf == '}') {
253         EndDelimiterFound = true;
254         ThisTokBuf++;
255         break;
256       }
257       if (*ThisTokBuf < '0' || *ThisTokBuf > '7') {
258         HadError = true;
259         if (Diags)
260           Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
261                diag::err_delimited_escape_invalid)
262               << StringRef(ThisTokBuf, 1);
263         ThisTokBuf++;
264         continue;
265       }
266       if (ResultChar & 0x020000000)
267         Overflow = true;
268 
269       ResultChar <<= 3;
270       ResultChar |= *ThisTokBuf++ - '0';
271     }
272     // Check for overflow.  Reject '\777', but not L'\777'.
273     if (!HadError &&
274         (Overflow || (CharWidth != 32 && (ResultChar >> CharWidth) != 0))) {
275       HadError = true;
276       if (Diags)
277         Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
278              diag::err_escape_too_large)
279             << 1;
280       ResultChar &= ~0U >> (32 - CharWidth);
281     }
282     break;
283   }
284     // Otherwise, these are not valid escapes.
285   case '(': case '{': case '[': case '%':
286     // GCC accepts these as extensions.  We warn about them as such though.
287     if (Diags)
288       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
289            diag::ext_nonstandard_escape)
290         << std::string(1, ResultChar);
291     break;
292   default:
293     if (!Diags)
294       break;
295 
296     if (isPrintable(ResultChar))
297       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
298            diag::ext_unknown_escape)
299         << std::string(1, ResultChar);
300     else
301       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
302            diag::ext_unknown_escape)
303         << "x" + llvm::utohexstr(ResultChar);
304     break;
305   }
306 
307   if (Delimited && Diags) {
308     if (!EndDelimiterFound)
309       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
310            diag::err_expected)
311           << tok::r_brace;
312     else if (!HadError) {
313       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
314            Features.CPlusPlus2b ? diag::warn_cxx2b_delimited_escape_sequence
315                                 : diag::ext_delimited_escape_sequence)
316           << /*delimited*/ 0 << (Features.CPlusPlus ? 1 : 0);
317     }
318   }
319 
320   return ResultChar;
321 }
322 
appendCodePoint(unsigned Codepoint,llvm::SmallVectorImpl<char> & Str)323 static void appendCodePoint(unsigned Codepoint,
324                             llvm::SmallVectorImpl<char> &Str) {
325   char ResultBuf[4];
326   char *ResultPtr = ResultBuf;
327   if (llvm::ConvertCodePointToUTF8(Codepoint, ResultPtr))
328     Str.append(ResultBuf, ResultPtr);
329 }
330 
expandUCNs(SmallVectorImpl<char> & Buf,StringRef Input)331 void clang::expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input) {
332   for (StringRef::iterator I = Input.begin(), E = Input.end(); I != E; ++I) {
333     if (*I != '\\') {
334       Buf.push_back(*I);
335       continue;
336     }
337 
338     ++I;
339     char Kind = *I;
340     ++I;
341 
342     assert(Kind == 'u' || Kind == 'U' || Kind == 'N');
343     uint32_t CodePoint = 0;
344 
345     if (Kind == 'u' && *I == '{') {
346       for (++I; *I != '}'; ++I) {
347         unsigned Value = llvm::hexDigitValue(*I);
348         assert(Value != -1U);
349         CodePoint <<= 4;
350         CodePoint += Value;
351       }
352       appendCodePoint(CodePoint, Buf);
353       continue;
354     }
355 
356     if (Kind == 'N') {
357       assert(*I == '{');
358       ++I;
359       auto Delim = std::find(I, Input.end(), '}');
360       assert(Delim != Input.end());
361       std::optional<llvm::sys::unicode::LooseMatchingResult> Res =
362           llvm::sys::unicode::nameToCodepointLooseMatching(
363               StringRef(I, std::distance(I, Delim)));
364       assert(Res);
365       CodePoint = Res->CodePoint;
366       assert(CodePoint != 0xFFFFFFFF);
367       appendCodePoint(CodePoint, Buf);
368       I = Delim;
369       continue;
370     }
371 
372     unsigned NumHexDigits;
373     if (Kind == 'u')
374       NumHexDigits = 4;
375     else
376       NumHexDigits = 8;
377 
378     assert(I + NumHexDigits <= E);
379 
380     for (; NumHexDigits != 0; ++I, --NumHexDigits) {
381       unsigned Value = llvm::hexDigitValue(*I);
382       assert(Value != -1U);
383 
384       CodePoint <<= 4;
385       CodePoint += Value;
386     }
387 
388     appendCodePoint(CodePoint, Buf);
389     --I;
390   }
391 }
392 
ProcessNumericUCNEscape(const char * ThisTokBegin,const char * & ThisTokBuf,const char * ThisTokEnd,uint32_t & UcnVal,unsigned short & UcnLen,bool & Delimited,FullSourceLoc Loc,DiagnosticsEngine * Diags,const LangOptions & Features,bool in_char_string_literal=false)393 static bool ProcessNumericUCNEscape(const char *ThisTokBegin,
394                                     const char *&ThisTokBuf,
395                                     const char *ThisTokEnd, uint32_t &UcnVal,
396                                     unsigned short &UcnLen, bool &Delimited,
397                                     FullSourceLoc Loc, DiagnosticsEngine *Diags,
398                                     const LangOptions &Features,
399                                     bool in_char_string_literal = false) {
400   const char *UcnBegin = ThisTokBuf;
401   bool HasError = false;
402   bool EndDelimiterFound = false;
403 
404   // Skip the '\u' char's.
405   ThisTokBuf += 2;
406   Delimited = false;
407   if (UcnBegin[1] == 'u' && in_char_string_literal &&
408       ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
409     Delimited = true;
410     ThisTokBuf++;
411   } else if (ThisTokBuf == ThisTokEnd || !isHexDigit(*ThisTokBuf)) {
412     if (Diags)
413       Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
414            diag::err_hex_escape_no_digits)
415           << StringRef(&ThisTokBuf[-1], 1);
416     return false;
417   }
418   UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
419 
420   bool Overflow = false;
421   unsigned short Count = 0;
422   for (; ThisTokBuf != ThisTokEnd && (Delimited || Count != UcnLen);
423        ++ThisTokBuf) {
424     if (Delimited && *ThisTokBuf == '}') {
425       ++ThisTokBuf;
426       EndDelimiterFound = true;
427       break;
428     }
429     int CharVal = llvm::hexDigitValue(*ThisTokBuf);
430     if (CharVal == -1) {
431       HasError = true;
432       if (!Delimited)
433         break;
434       if (Diags) {
435         Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
436              diag::err_delimited_escape_invalid)
437             << StringRef(ThisTokBuf, 1);
438       }
439       Count++;
440       continue;
441     }
442     if (UcnVal & 0xF0000000) {
443       Overflow = true;
444       continue;
445     }
446     UcnVal <<= 4;
447     UcnVal |= CharVal;
448     Count++;
449   }
450 
451   if (Overflow) {
452     if (Diags)
453       Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
454            diag::err_escape_too_large)
455           << 0;
456     return false;
457   }
458 
459   if (Delimited && !EndDelimiterFound) {
460     if (Diags) {
461       Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
462            diag::err_expected)
463           << tok::r_brace;
464     }
465     return false;
466   }
467 
468   // If we didn't consume the proper number of digits, there is a problem.
469   if (Count == 0 || (!Delimited && Count != UcnLen)) {
470     if (Diags)
471       Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
472            Delimited ? diag::err_delimited_escape_empty
473                      : diag::err_ucn_escape_incomplete);
474     return false;
475   }
476   return !HasError;
477 }
478 
DiagnoseInvalidUnicodeCharacterName(DiagnosticsEngine * Diags,const LangOptions & Features,FullSourceLoc Loc,const char * TokBegin,const char * TokRangeBegin,const char * TokRangeEnd,llvm::StringRef Name)479 static void DiagnoseInvalidUnicodeCharacterName(
480     DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc Loc,
481     const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd,
482     llvm::StringRef Name) {
483 
484   Diag(Diags, Features, Loc, TokBegin, TokRangeBegin, TokRangeEnd,
485        diag::err_invalid_ucn_name)
486       << Name;
487 
488   namespace u = llvm::sys::unicode;
489 
490   std::optional<u::LooseMatchingResult> Res =
491       u::nameToCodepointLooseMatching(Name);
492   if (Res) {
493     Diag(Diags, Features, Loc, TokBegin, TokRangeBegin, TokRangeEnd,
494          diag::note_invalid_ucn_name_loose_matching)
495         << FixItHint::CreateReplacement(
496                MakeCharSourceRange(Features, Loc, TokBegin, TokRangeBegin,
497                                    TokRangeEnd),
498                Res->Name);
499     return;
500   }
501 
502   unsigned Distance = 0;
503   SmallVector<u::MatchForCodepointName> Matches =
504       u::nearestMatchesForCodepointName(Name, 5);
505   assert(!Matches.empty() && "No unicode characters found");
506 
507   for (const auto &Match : Matches) {
508     if (Distance == 0)
509       Distance = Match.Distance;
510     if (std::max(Distance, Match.Distance) -
511             std::min(Distance, Match.Distance) >
512         3)
513       break;
514     Distance = Match.Distance;
515 
516     std::string Str;
517     llvm::UTF32 V = Match.Value;
518     bool Converted =
519         llvm::convertUTF32ToUTF8String(llvm::ArrayRef<llvm::UTF32>(&V, 1), Str);
520     (void)Converted;
521     assert(Converted && "Found a match wich is not a unicode character");
522 
523     Diag(Diags, Features, Loc, TokBegin, TokRangeBegin, TokRangeEnd,
524          diag::note_invalid_ucn_name_candidate)
525         << Match.Name << llvm::utohexstr(Match.Value)
526         << Str // FIXME: Fix the rendering of non printable characters
527         << FixItHint::CreateReplacement(
528                MakeCharSourceRange(Features, Loc, TokBegin, TokRangeBegin,
529                                    TokRangeEnd),
530                Match.Name);
531   }
532 }
533 
ProcessNamedUCNEscape(const char * ThisTokBegin,const char * & ThisTokBuf,const char * ThisTokEnd,uint32_t & UcnVal,unsigned short & UcnLen,FullSourceLoc Loc,DiagnosticsEngine * Diags,const LangOptions & Features)534 static bool ProcessNamedUCNEscape(const char *ThisTokBegin,
535                                   const char *&ThisTokBuf,
536                                   const char *ThisTokEnd, uint32_t &UcnVal,
537                                   unsigned short &UcnLen, FullSourceLoc Loc,
538                                   DiagnosticsEngine *Diags,
539                                   const LangOptions &Features) {
540   const char *UcnBegin = ThisTokBuf;
541   assert(UcnBegin[0] == '\\' && UcnBegin[1] == 'N');
542   ThisTokBuf += 2;
543   if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') {
544     if (Diags) {
545       Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
546            diag::err_delimited_escape_missing_brace)
547           << StringRef(&ThisTokBuf[-1], 1);
548     }
549     return false;
550   }
551   ThisTokBuf++;
552   const char *ClosingBrace = std::find_if(ThisTokBuf, ThisTokEnd, [](char C) {
553     return C == '}' || isVerticalWhitespace(C);
554   });
555   bool Incomplete = ClosingBrace == ThisTokEnd;
556   bool Empty = ClosingBrace == ThisTokBuf;
557   if (Incomplete || Empty) {
558     if (Diags) {
559       Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
560            Incomplete ? diag::err_ucn_escape_incomplete
561                       : diag::err_delimited_escape_empty)
562           << StringRef(&UcnBegin[1], 1);
563     }
564     ThisTokBuf = ClosingBrace == ThisTokEnd ? ClosingBrace : ClosingBrace + 1;
565     return false;
566   }
567   StringRef Name(ThisTokBuf, ClosingBrace - ThisTokBuf);
568   ThisTokBuf = ClosingBrace + 1;
569   std::optional<char32_t> Res = llvm::sys::unicode::nameToCodepointStrict(Name);
570   if (!Res) {
571     if (Diags)
572       DiagnoseInvalidUnicodeCharacterName(Diags, Features, Loc, ThisTokBegin,
573                                           &UcnBegin[3], ClosingBrace, Name);
574     return false;
575   }
576   UcnVal = *Res;
577   UcnLen = UcnVal > 0xFFFF ? 8 : 4;
578   return true;
579 }
580 
581 /// ProcessUCNEscape - Read the Universal Character Name, check constraints and
582 /// return the UTF32.
ProcessUCNEscape(const char * ThisTokBegin,const char * & ThisTokBuf,const char * ThisTokEnd,uint32_t & UcnVal,unsigned short & UcnLen,FullSourceLoc Loc,DiagnosticsEngine * Diags,const LangOptions & Features,bool in_char_string_literal=false)583 static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
584                              const char *ThisTokEnd, uint32_t &UcnVal,
585                              unsigned short &UcnLen, FullSourceLoc Loc,
586                              DiagnosticsEngine *Diags,
587                              const LangOptions &Features,
588                              bool in_char_string_literal = false) {
589 
590   bool HasError;
591   const char *UcnBegin = ThisTokBuf;
592   bool IsDelimitedEscapeSequence = false;
593   bool IsNamedEscapeSequence = false;
594   if (ThisTokBuf[1] == 'N') {
595     IsNamedEscapeSequence = true;
596     HasError = !ProcessNamedUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd,
597                                       UcnVal, UcnLen, Loc, Diags, Features);
598   } else {
599     HasError =
600         !ProcessNumericUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
601                                  UcnLen, IsDelimitedEscapeSequence, Loc, Diags,
602                                  Features, in_char_string_literal);
603   }
604   if (HasError)
605     return false;
606 
607   // Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2]
608   if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints
609       UcnVal > 0x10FFFF) {                      // maximum legal UTF32 value
610     if (Diags)
611       Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
612            diag::err_ucn_escape_invalid);
613     return false;
614   }
615 
616   // C++11 allows UCNs that refer to control characters and basic source
617   // characters inside character and string literals
618   if (UcnVal < 0xa0 &&
619       (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) {  // $, @, `
620     bool IsError = (!Features.CPlusPlus11 || !in_char_string_literal);
621     if (Diags) {
622       char BasicSCSChar = UcnVal;
623       if (UcnVal >= 0x20 && UcnVal < 0x7f)
624         Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
625              IsError ? diag::err_ucn_escape_basic_scs :
626                        diag::warn_cxx98_compat_literal_ucn_escape_basic_scs)
627             << StringRef(&BasicSCSChar, 1);
628       else
629         Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
630              IsError ? diag::err_ucn_control_character :
631                        diag::warn_cxx98_compat_literal_ucn_control_character);
632     }
633     if (IsError)
634       return false;
635   }
636 
637   if (!Features.CPlusPlus && !Features.C99 && Diags)
638     Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
639          diag::warn_ucn_not_valid_in_c89_literal);
640 
641   if ((IsDelimitedEscapeSequence || IsNamedEscapeSequence) && Diags)
642     Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
643          Features.CPlusPlus2b ? diag::warn_cxx2b_delimited_escape_sequence
644                               : diag::ext_delimited_escape_sequence)
645         << (IsNamedEscapeSequence ? 1 : 0) << (Features.CPlusPlus ? 1 : 0);
646 
647   return true;
648 }
649 
650 /// MeasureUCNEscape - Determine the number of bytes within the resulting string
651 /// which this UCN will occupy.
MeasureUCNEscape(const char * ThisTokBegin,const char * & ThisTokBuf,const char * ThisTokEnd,unsigned CharByteWidth,const LangOptions & Features,bool & HadError)652 static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
653                             const char *ThisTokEnd, unsigned CharByteWidth,
654                             const LangOptions &Features, bool &HadError) {
655   // UTF-32: 4 bytes per escape.
656   if (CharByteWidth == 4)
657     return 4;
658 
659   uint32_t UcnVal = 0;
660   unsigned short UcnLen = 0;
661   FullSourceLoc Loc;
662 
663   if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
664                         UcnLen, Loc, nullptr, Features, true)) {
665     HadError = true;
666     return 0;
667   }
668 
669   // UTF-16: 2 bytes for BMP, 4 bytes otherwise.
670   if (CharByteWidth == 2)
671     return UcnVal <= 0xFFFF ? 2 : 4;
672 
673   // UTF-8.
674   if (UcnVal < 0x80)
675     return 1;
676   if (UcnVal < 0x800)
677     return 2;
678   if (UcnVal < 0x10000)
679     return 3;
680   return 4;
681 }
682 
683 /// EncodeUCNEscape - Read the Universal Character Name, check constraints and
684 /// convert the UTF32 to UTF8 or UTF16. This is a subroutine of
685 /// StringLiteralParser. When we decide to implement UCN's for identifiers,
686 /// we will likely rework our support for UCN's.
EncodeUCNEscape(const char * ThisTokBegin,const char * & ThisTokBuf,const char * ThisTokEnd,char * & ResultBuf,bool & HadError,FullSourceLoc Loc,unsigned CharByteWidth,DiagnosticsEngine * Diags,const LangOptions & Features)687 static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
688                             const char *ThisTokEnd,
689                             char *&ResultBuf, bool &HadError,
690                             FullSourceLoc Loc, unsigned CharByteWidth,
691                             DiagnosticsEngine *Diags,
692                             const LangOptions &Features) {
693   typedef uint32_t UTF32;
694   UTF32 UcnVal = 0;
695   unsigned short UcnLen = 0;
696   if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen,
697                         Loc, Diags, Features, true)) {
698     HadError = true;
699     return;
700   }
701 
702   assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth == 4) &&
703          "only character widths of 1, 2, or 4 bytes supported");
704 
705   (void)UcnLen;
706   assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
707 
708   if (CharByteWidth == 4) {
709     // FIXME: Make the type of the result buffer correct instead of
710     // using reinterpret_cast.
711     llvm::UTF32 *ResultPtr = reinterpret_cast<llvm::UTF32*>(ResultBuf);
712     *ResultPtr = UcnVal;
713     ResultBuf += 4;
714     return;
715   }
716 
717   if (CharByteWidth == 2) {
718     // FIXME: Make the type of the result buffer correct instead of
719     // using reinterpret_cast.
720     llvm::UTF16 *ResultPtr = reinterpret_cast<llvm::UTF16*>(ResultBuf);
721 
722     if (UcnVal <= (UTF32)0xFFFF) {
723       *ResultPtr = UcnVal;
724       ResultBuf += 2;
725       return;
726     }
727 
728     // Convert to UTF16.
729     UcnVal -= 0x10000;
730     *ResultPtr     = 0xD800 + (UcnVal >> 10);
731     *(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF);
732     ResultBuf += 4;
733     return;
734   }
735 
736   assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters");
737 
738   // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
739   // The conversion below was inspired by:
740   //   http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
741   // First, we determine how many bytes the result will require.
742   typedef uint8_t UTF8;
743 
744   unsigned short bytesToWrite = 0;
745   if (UcnVal < (UTF32)0x80)
746     bytesToWrite = 1;
747   else if (UcnVal < (UTF32)0x800)
748     bytesToWrite = 2;
749   else if (UcnVal < (UTF32)0x10000)
750     bytesToWrite = 3;
751   else
752     bytesToWrite = 4;
753 
754   const unsigned byteMask = 0xBF;
755   const unsigned byteMark = 0x80;
756 
757   // Once the bits are split out into bytes of UTF8, this is a mask OR-ed
758   // into the first byte, depending on how many bytes follow.
759   static const UTF8 firstByteMark[5] = {
760     0x00, 0x00, 0xC0, 0xE0, 0xF0
761   };
762   // Finally, we write the bytes into ResultBuf.
763   ResultBuf += bytesToWrite;
764   switch (bytesToWrite) { // note: everything falls through.
765   case 4:
766     *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
767     [[fallthrough]];
768   case 3:
769     *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
770     [[fallthrough]];
771   case 2:
772     *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
773     [[fallthrough]];
774   case 1:
775     *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
776   }
777   // Update the buffer.
778   ResultBuf += bytesToWrite;
779 }
780 
781 ///       integer-constant: [C99 6.4.4.1]
782 ///         decimal-constant integer-suffix
783 ///         octal-constant integer-suffix
784 ///         hexadecimal-constant integer-suffix
785 ///         binary-literal integer-suffix [GNU, C++1y]
786 ///       user-defined-integer-literal: [C++11 lex.ext]
787 ///         decimal-literal ud-suffix
788 ///         octal-literal ud-suffix
789 ///         hexadecimal-literal ud-suffix
790 ///         binary-literal ud-suffix [GNU, C++1y]
791 ///       decimal-constant:
792 ///         nonzero-digit
793 ///         decimal-constant digit
794 ///       octal-constant:
795 ///         0
796 ///         octal-constant octal-digit
797 ///       hexadecimal-constant:
798 ///         hexadecimal-prefix hexadecimal-digit
799 ///         hexadecimal-constant hexadecimal-digit
800 ///       hexadecimal-prefix: one of
801 ///         0x 0X
802 ///       binary-literal:
803 ///         0b binary-digit
804 ///         0B binary-digit
805 ///         binary-literal binary-digit
806 ///       integer-suffix:
807 ///         unsigned-suffix [long-suffix]
808 ///         unsigned-suffix [long-long-suffix]
809 ///         long-suffix [unsigned-suffix]
810 ///         long-long-suffix [unsigned-sufix]
811 ///       nonzero-digit:
812 ///         1 2 3 4 5 6 7 8 9
813 ///       octal-digit:
814 ///         0 1 2 3 4 5 6 7
815 ///       hexadecimal-digit:
816 ///         0 1 2 3 4 5 6 7 8 9
817 ///         a b c d e f
818 ///         A B C D E F
819 ///       binary-digit:
820 ///         0
821 ///         1
822 ///       unsigned-suffix: one of
823 ///         u U
824 ///       long-suffix: one of
825 ///         l L
826 ///       long-long-suffix: one of
827 ///         ll LL
828 ///
829 ///       floating-constant: [C99 6.4.4.2]
830 ///         TODO: add rules...
831 ///
NumericLiteralParser(StringRef TokSpelling,SourceLocation TokLoc,const SourceManager & SM,const LangOptions & LangOpts,const TargetInfo & Target,DiagnosticsEngine & Diags)832 NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
833                                            SourceLocation TokLoc,
834                                            const SourceManager &SM,
835                                            const LangOptions &LangOpts,
836                                            const TargetInfo &Target,
837                                            DiagnosticsEngine &Diags)
838     : SM(SM), LangOpts(LangOpts), Diags(Diags),
839       ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
840 
841   s = DigitsBegin = ThisTokBegin;
842   saw_exponent = false;
843   saw_period = false;
844   saw_ud_suffix = false;
845   saw_fixed_point_suffix = false;
846   isLong = false;
847   isUnsigned = false;
848   isLongLong = false;
849   isSizeT = false;
850   isHalf = false;
851   isFloat = false;
852   isImaginary = false;
853   isFloat16 = false;
854   isFloat128 = false;
855   MicrosoftInteger = 0;
856   isFract = false;
857   isAccum = false;
858   hadError = false;
859   isBitInt = false;
860 
861   // This routine assumes that the range begin/end matches the regex for integer
862   // and FP constants (specifically, the 'pp-number' regex), and assumes that
863   // the byte at "*end" is both valid and not part of the regex.  Because of
864   // this, it doesn't have to check for 'overscan' in various places.
865   if (isPreprocessingNumberBody(*ThisTokEnd)) {
866     Diags.Report(TokLoc, diag::err_lexing_numeric);
867     hadError = true;
868     return;
869   }
870 
871   if (*s == '0') { // parse radix
872     ParseNumberStartingWithZero(TokLoc);
873     if (hadError)
874       return;
875   } else { // the first digit is non-zero
876     radix = 10;
877     s = SkipDigits(s);
878     if (s == ThisTokEnd) {
879       // Done.
880     } else {
881       ParseDecimalOrOctalCommon(TokLoc);
882       if (hadError)
883         return;
884     }
885   }
886 
887   SuffixBegin = s;
888   checkSeparator(TokLoc, s, CSK_AfterDigits);
889 
890   // Initial scan to lookahead for fixed point suffix.
891   if (LangOpts.FixedPoint) {
892     for (const char *c = s; c != ThisTokEnd; ++c) {
893       if (*c == 'r' || *c == 'k' || *c == 'R' || *c == 'K') {
894         saw_fixed_point_suffix = true;
895         break;
896       }
897     }
898   }
899 
900   // Parse the suffix.  At this point we can classify whether we have an FP or
901   // integer constant.
902   bool isFixedPointConstant = isFixedPointLiteral();
903   bool isFPConstant = isFloatingLiteral();
904   bool HasSize = false;
905 
906   // Loop over all of the characters of the suffix.  If we see something bad,
907   // we break out of the loop.
908   for (; s != ThisTokEnd; ++s) {
909     switch (*s) {
910     case 'R':
911     case 'r':
912       if (!LangOpts.FixedPoint)
913         break;
914       if (isFract || isAccum) break;
915       if (!(saw_period || saw_exponent)) break;
916       isFract = true;
917       continue;
918     case 'K':
919     case 'k':
920       if (!LangOpts.FixedPoint)
921         break;
922       if (isFract || isAccum) break;
923       if (!(saw_period || saw_exponent)) break;
924       isAccum = true;
925       continue;
926     case 'h':      // FP Suffix for "half".
927     case 'H':
928       // OpenCL Extension v1.2 s9.5 - h or H suffix for half type.
929       if (!(LangOpts.Half || LangOpts.FixedPoint))
930         break;
931       if (isIntegerLiteral()) break;  // Error for integer constant.
932       if (HasSize)
933         break;
934       HasSize = true;
935       isHalf = true;
936       continue;  // Success.
937     case 'f':      // FP Suffix for "float"
938     case 'F':
939       if (!isFPConstant) break;  // Error for integer constant.
940       if (HasSize)
941         break;
942       HasSize = true;
943 
944       // CUDA host and device may have different _Float16 support, therefore
945       // allows f16 literals to avoid false alarm.
946       // When we compile for OpenMP target offloading on NVPTX, f16 suffix
947       // should also be supported.
948       // ToDo: more precise check for CUDA.
949       // TODO: AMDGPU might also support it in the future.
950       if ((Target.hasFloat16Type() || LangOpts.CUDA ||
951            (LangOpts.OpenMPIsDevice && Target.getTriple().isNVPTX())) &&
952           s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') {
953         s += 2; // success, eat up 2 characters.
954         isFloat16 = true;
955         continue;
956       }
957 
958       isFloat = true;
959       continue;  // Success.
960     case 'q':    // FP Suffix for "__float128"
961     case 'Q':
962       if (!isFPConstant) break;  // Error for integer constant.
963       if (HasSize)
964         break;
965       HasSize = true;
966       isFloat128 = true;
967       continue;  // Success.
968     case 'u':
969     case 'U':
970       if (isFPConstant) break;  // Error for floating constant.
971       if (isUnsigned) break;    // Cannot be repeated.
972       isUnsigned = true;
973       continue;  // Success.
974     case 'l':
975     case 'L':
976       if (HasSize)
977         break;
978       HasSize = true;
979 
980       // Check for long long.  The L's need to be adjacent and the same case.
981       if (s[1] == s[0]) {
982         assert(s + 1 < ThisTokEnd && "didn't maximally munch?");
983         if (isFPConstant) break;        // long long invalid for floats.
984         isLongLong = true;
985         ++s;  // Eat both of them.
986       } else {
987         isLong = true;
988       }
989       continue; // Success.
990     case 'z':
991     case 'Z':
992       if (isFPConstant)
993         break; // Invalid for floats.
994       if (HasSize)
995         break;
996       HasSize = true;
997       isSizeT = true;
998       continue;
999     case 'i':
1000     case 'I':
1001       if (LangOpts.MicrosoftExt && !isFPConstant) {
1002         // Allow i8, i16, i32, and i64. First, look ahead and check if
1003         // suffixes are Microsoft integers and not the imaginary unit.
1004         uint8_t Bits = 0;
1005         size_t ToSkip = 0;
1006         switch (s[1]) {
1007         case '8': // i8 suffix
1008           Bits = 8;
1009           ToSkip = 2;
1010           break;
1011         case '1':
1012           if (s[2] == '6') { // i16 suffix
1013             Bits = 16;
1014             ToSkip = 3;
1015           }
1016           break;
1017         case '3':
1018           if (s[2] == '2') { // i32 suffix
1019             Bits = 32;
1020             ToSkip = 3;
1021           }
1022           break;
1023         case '6':
1024           if (s[2] == '4') { // i64 suffix
1025             Bits = 64;
1026             ToSkip = 3;
1027           }
1028           break;
1029         default:
1030           break;
1031         }
1032         if (Bits) {
1033           if (HasSize)
1034             break;
1035           HasSize = true;
1036           MicrosoftInteger = Bits;
1037           s += ToSkip;
1038           assert(s <= ThisTokEnd && "didn't maximally munch?");
1039           break;
1040         }
1041       }
1042       [[fallthrough]];
1043     case 'j':
1044     case 'J':
1045       if (isImaginary) break;   // Cannot be repeated.
1046       isImaginary = true;
1047       continue;  // Success.
1048     case 'w':
1049     case 'W':
1050       if (isFPConstant)
1051         break; // Invalid for floats.
1052       if (HasSize)
1053         break; // Invalid if we already have a size for the literal.
1054 
1055       // wb and WB are allowed, but a mixture of cases like Wb or wB is not. We
1056       // explicitly do not support the suffix in C++ as an extension because a
1057       // library-based UDL that resolves to a library type may be more
1058       // appropriate there.
1059       if (!LangOpts.CPlusPlus && ((s[0] == 'w' && s[1] == 'b') ||
1060           (s[0] == 'W' && s[1] == 'B'))) {
1061         isBitInt = true;
1062         HasSize = true;
1063         ++s; // Skip both characters (2nd char skipped on continue).
1064         continue; // Success.
1065       }
1066     }
1067     // If we reached here, there was an error or a ud-suffix.
1068     break;
1069   }
1070 
1071   // "i", "if", and "il" are user-defined suffixes in C++1y.
1072   if (s != ThisTokEnd || isImaginary) {
1073     // FIXME: Don't bother expanding UCNs if !tok.hasUCN().
1074     expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin));
1075     if (isValidUDSuffix(LangOpts, UDSuffixBuf)) {
1076       if (!isImaginary) {
1077         // Any suffix pieces we might have parsed are actually part of the
1078         // ud-suffix.
1079         isLong = false;
1080         isUnsigned = false;
1081         isLongLong = false;
1082         isSizeT = false;
1083         isFloat = false;
1084         isFloat16 = false;
1085         isHalf = false;
1086         isImaginary = false;
1087         isBitInt = false;
1088         MicrosoftInteger = 0;
1089         saw_fixed_point_suffix = false;
1090         isFract = false;
1091         isAccum = false;
1092       }
1093 
1094       saw_ud_suffix = true;
1095       return;
1096     }
1097 
1098     if (s != ThisTokEnd) {
1099       // Report an error if there are any.
1100       Diags.Report(Lexer::AdvanceToTokenCharacter(
1101                        TokLoc, SuffixBegin - ThisTokBegin, SM, LangOpts),
1102                    diag::err_invalid_suffix_constant)
1103           << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)
1104           << (isFixedPointConstant ? 2 : isFPConstant);
1105       hadError = true;
1106     }
1107   }
1108 
1109   if (!hadError && saw_fixed_point_suffix) {
1110     assert(isFract || isAccum);
1111   }
1112 }
1113 
1114 /// ParseDecimalOrOctalCommon - This method is called for decimal or octal
1115 /// numbers. It issues an error for illegal digits, and handles floating point
1116 /// parsing. If it detects a floating point number, the radix is set to 10.
ParseDecimalOrOctalCommon(SourceLocation TokLoc)1117 void NumericLiteralParser::ParseDecimalOrOctalCommon(SourceLocation TokLoc){
1118   assert((radix == 8 || radix == 10) && "Unexpected radix");
1119 
1120   // If we have a hex digit other than 'e' (which denotes a FP exponent) then
1121   // the code is using an incorrect base.
1122   if (isHexDigit(*s) && *s != 'e' && *s != 'E' &&
1123       !isValidUDSuffix(LangOpts, StringRef(s, ThisTokEnd - s))) {
1124     Diags.Report(
1125         Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM, LangOpts),
1126         diag::err_invalid_digit)
1127         << StringRef(s, 1) << (radix == 8 ? 1 : 0);
1128     hadError = true;
1129     return;
1130   }
1131 
1132   if (*s == '.') {
1133     checkSeparator(TokLoc, s, CSK_AfterDigits);
1134     s++;
1135     radix = 10;
1136     saw_period = true;
1137     checkSeparator(TokLoc, s, CSK_BeforeDigits);
1138     s = SkipDigits(s); // Skip suffix.
1139   }
1140   if (*s == 'e' || *s == 'E') { // exponent
1141     checkSeparator(TokLoc, s, CSK_AfterDigits);
1142     const char *Exponent = s;
1143     s++;
1144     radix = 10;
1145     saw_exponent = true;
1146     if (s != ThisTokEnd && (*s == '+' || *s == '-'))  s++; // sign
1147     const char *first_non_digit = SkipDigits(s);
1148     if (containsDigits(s, first_non_digit)) {
1149       checkSeparator(TokLoc, s, CSK_BeforeDigits);
1150       s = first_non_digit;
1151     } else {
1152       if (!hadError) {
1153         Diags.Report(Lexer::AdvanceToTokenCharacter(
1154                          TokLoc, Exponent - ThisTokBegin, SM, LangOpts),
1155                      diag::err_exponent_has_no_digits);
1156         hadError = true;
1157       }
1158       return;
1159     }
1160   }
1161 }
1162 
1163 /// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
1164 /// suffixes as ud-suffixes, because the diagnostic experience is better if we
1165 /// treat it as an invalid suffix.
isValidUDSuffix(const LangOptions & LangOpts,StringRef Suffix)1166 bool NumericLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
1167                                            StringRef Suffix) {
1168   if (!LangOpts.CPlusPlus11 || Suffix.empty())
1169     return false;
1170 
1171   // By C++11 [lex.ext]p10, ud-suffixes starting with an '_' are always valid.
1172   if (Suffix[0] == '_')
1173     return true;
1174 
1175   // In C++11, there are no library suffixes.
1176   if (!LangOpts.CPlusPlus14)
1177     return false;
1178 
1179   // In C++14, "s", "h", "min", "ms", "us", and "ns" are used in the library.
1180   // Per tweaked N3660, "il", "i", and "if" are also used in the library.
1181   // In C++2a "d" and "y" are used in the library.
1182   return llvm::StringSwitch<bool>(Suffix)
1183       .Cases("h", "min", "s", true)
1184       .Cases("ms", "us", "ns", true)
1185       .Cases("il", "i", "if", true)
1186       .Cases("d", "y", LangOpts.CPlusPlus20)
1187       .Default(false);
1188 }
1189 
checkSeparator(SourceLocation TokLoc,const char * Pos,CheckSeparatorKind IsAfterDigits)1190 void NumericLiteralParser::checkSeparator(SourceLocation TokLoc,
1191                                           const char *Pos,
1192                                           CheckSeparatorKind IsAfterDigits) {
1193   if (IsAfterDigits == CSK_AfterDigits) {
1194     if (Pos == ThisTokBegin)
1195       return;
1196     --Pos;
1197   } else if (Pos == ThisTokEnd)
1198     return;
1199 
1200   if (isDigitSeparator(*Pos)) {
1201     Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, Pos - ThisTokBegin, SM,
1202                                                 LangOpts),
1203                  diag::err_digit_separator_not_between_digits)
1204         << IsAfterDigits;
1205     hadError = true;
1206   }
1207 }
1208 
1209 /// ParseNumberStartingWithZero - This method is called when the first character
1210 /// of the number is found to be a zero.  This means it is either an octal
1211 /// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
1212 /// a floating point number (01239.123e4).  Eat the prefix, determining the
1213 /// radix etc.
ParseNumberStartingWithZero(SourceLocation TokLoc)1214 void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
1215   assert(s[0] == '0' && "Invalid method call");
1216   s++;
1217 
1218   int c1 = s[0];
1219 
1220   // Handle a hex number like 0x1234.
1221   if ((c1 == 'x' || c1 == 'X') && (isHexDigit(s[1]) || s[1] == '.')) {
1222     s++;
1223     assert(s < ThisTokEnd && "didn't maximally munch?");
1224     radix = 16;
1225     DigitsBegin = s;
1226     s = SkipHexDigits(s);
1227     bool HasSignificandDigits = containsDigits(DigitsBegin, s);
1228     if (s == ThisTokEnd) {
1229       // Done.
1230     } else if (*s == '.') {
1231       s++;
1232       saw_period = true;
1233       const char *floatDigitsBegin = s;
1234       s = SkipHexDigits(s);
1235       if (containsDigits(floatDigitsBegin, s))
1236         HasSignificandDigits = true;
1237       if (HasSignificandDigits)
1238         checkSeparator(TokLoc, floatDigitsBegin, CSK_BeforeDigits);
1239     }
1240 
1241     if (!HasSignificandDigits) {
1242       Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
1243                                                   LangOpts),
1244                    diag::err_hex_constant_requires)
1245           << LangOpts.CPlusPlus << 1;
1246       hadError = true;
1247       return;
1248     }
1249 
1250     // A binary exponent can appear with or with a '.'. If dotted, the
1251     // binary exponent is required.
1252     if (*s == 'p' || *s == 'P') {
1253       checkSeparator(TokLoc, s, CSK_AfterDigits);
1254       const char *Exponent = s;
1255       s++;
1256       saw_exponent = true;
1257       if (s != ThisTokEnd && (*s == '+' || *s == '-'))  s++; // sign
1258       const char *first_non_digit = SkipDigits(s);
1259       if (!containsDigits(s, first_non_digit)) {
1260         if (!hadError) {
1261           Diags.Report(Lexer::AdvanceToTokenCharacter(
1262                            TokLoc, Exponent - ThisTokBegin, SM, LangOpts),
1263                        diag::err_exponent_has_no_digits);
1264           hadError = true;
1265         }
1266         return;
1267       }
1268       checkSeparator(TokLoc, s, CSK_BeforeDigits);
1269       s = first_non_digit;
1270 
1271       if (!LangOpts.HexFloats)
1272         Diags.Report(TokLoc, LangOpts.CPlusPlus
1273                                  ? diag::ext_hex_literal_invalid
1274                                  : diag::ext_hex_constant_invalid);
1275       else if (LangOpts.CPlusPlus17)
1276         Diags.Report(TokLoc, diag::warn_cxx17_hex_literal);
1277     } else if (saw_period) {
1278       Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
1279                                                   LangOpts),
1280                    diag::err_hex_constant_requires)
1281           << LangOpts.CPlusPlus << 0;
1282       hadError = true;
1283     }
1284     return;
1285   }
1286 
1287   // Handle simple binary numbers 0b01010
1288   if ((c1 == 'b' || c1 == 'B') && (s[1] == '0' || s[1] == '1')) {
1289     // 0b101010 is a C++1y / GCC extension.
1290     Diags.Report(TokLoc, LangOpts.CPlusPlus14
1291                              ? diag::warn_cxx11_compat_binary_literal
1292                          : LangOpts.CPlusPlus ? diag::ext_binary_literal_cxx14
1293                                               : diag::ext_binary_literal);
1294     ++s;
1295     assert(s < ThisTokEnd && "didn't maximally munch?");
1296     radix = 2;
1297     DigitsBegin = s;
1298     s = SkipBinaryDigits(s);
1299     if (s == ThisTokEnd) {
1300       // Done.
1301     } else if (isHexDigit(*s) &&
1302                !isValidUDSuffix(LangOpts, StringRef(s, ThisTokEnd - s))) {
1303       Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
1304                                                   LangOpts),
1305                    diag::err_invalid_digit)
1306           << StringRef(s, 1) << 2;
1307       hadError = true;
1308     }
1309     // Other suffixes will be diagnosed by the caller.
1310     return;
1311   }
1312 
1313   // For now, the radix is set to 8. If we discover that we have a
1314   // floating point constant, the radix will change to 10. Octal floating
1315   // point constants are not permitted (only decimal and hexadecimal).
1316   radix = 8;
1317   const char *PossibleNewDigitStart = s;
1318   s = SkipOctalDigits(s);
1319   // When the value is 0 followed by a suffix (like 0wb), we want to leave 0
1320   // as the start of the digits. So if skipping octal digits does not skip
1321   // anything, we leave the digit start where it was.
1322   if (s != PossibleNewDigitStart)
1323     DigitsBegin = PossibleNewDigitStart;
1324 
1325   if (s == ThisTokEnd)
1326     return; // Done, simple octal number like 01234
1327 
1328   // If we have some other non-octal digit that *is* a decimal digit, see if
1329   // this is part of a floating point number like 094.123 or 09e1.
1330   if (isDigit(*s)) {
1331     const char *EndDecimal = SkipDigits(s);
1332     if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') {
1333       s = EndDecimal;
1334       radix = 10;
1335     }
1336   }
1337 
1338   ParseDecimalOrOctalCommon(TokLoc);
1339 }
1340 
alwaysFitsInto64Bits(unsigned Radix,unsigned NumDigits)1341 static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) {
1342   switch (Radix) {
1343   case 2:
1344     return NumDigits <= 64;
1345   case 8:
1346     return NumDigits <= 64 / 3; // Digits are groups of 3 bits.
1347   case 10:
1348     return NumDigits <= 19; // floor(log10(2^64))
1349   case 16:
1350     return NumDigits <= 64 / 4; // Digits are groups of 4 bits.
1351   default:
1352     llvm_unreachable("impossible Radix");
1353   }
1354 }
1355 
1356 /// GetIntegerValue - Convert this numeric literal value to an APInt that
1357 /// matches Val's input width.  If there is an overflow, set Val to the low bits
1358 /// of the result and return true.  Otherwise, return false.
GetIntegerValue(llvm::APInt & Val)1359 bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
1360   // Fast path: Compute a conservative bound on the maximum number of
1361   // bits per digit in this radix. If we can't possibly overflow a
1362   // uint64 based on that bound then do the simple conversion to
1363   // integer. This avoids the expensive overflow checking below, and
1364   // handles the common cases that matter (small decimal integers and
1365   // hex/octal values which don't overflow).
1366   const unsigned NumDigits = SuffixBegin - DigitsBegin;
1367   if (alwaysFitsInto64Bits(radix, NumDigits)) {
1368     uint64_t N = 0;
1369     for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr)
1370       if (!isDigitSeparator(*Ptr))
1371         N = N * radix + llvm::hexDigitValue(*Ptr);
1372 
1373     // This will truncate the value to Val's input width. Simply check
1374     // for overflow by comparing.
1375     Val = N;
1376     return Val.getZExtValue() != N;
1377   }
1378 
1379   Val = 0;
1380   const char *Ptr = DigitsBegin;
1381 
1382   llvm::APInt RadixVal(Val.getBitWidth(), radix);
1383   llvm::APInt CharVal(Val.getBitWidth(), 0);
1384   llvm::APInt OldVal = Val;
1385 
1386   bool OverflowOccurred = false;
1387   while (Ptr < SuffixBegin) {
1388     if (isDigitSeparator(*Ptr)) {
1389       ++Ptr;
1390       continue;
1391     }
1392 
1393     unsigned C = llvm::hexDigitValue(*Ptr++);
1394 
1395     // If this letter is out of bound for this radix, reject it.
1396     assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1397 
1398     CharVal = C;
1399 
1400     // Add the digit to the value in the appropriate radix.  If adding in digits
1401     // made the value smaller, then this overflowed.
1402     OldVal = Val;
1403 
1404     // Multiply by radix, did overflow occur on the multiply?
1405     Val *= RadixVal;
1406     OverflowOccurred |= Val.udiv(RadixVal) != OldVal;
1407 
1408     // Add value, did overflow occur on the value?
1409     //   (a + b) ult b  <=> overflow
1410     Val += CharVal;
1411     OverflowOccurred |= Val.ult(CharVal);
1412   }
1413   return OverflowOccurred;
1414 }
1415 
1416 llvm::APFloat::opStatus
GetFloatValue(llvm::APFloat & Result)1417 NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) {
1418   using llvm::APFloat;
1419 
1420   unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin);
1421 
1422   llvm::SmallString<16> Buffer;
1423   StringRef Str(ThisTokBegin, n);
1424   if (Str.contains('\'')) {
1425     Buffer.reserve(n);
1426     std::remove_copy_if(Str.begin(), Str.end(), std::back_inserter(Buffer),
1427                         &isDigitSeparator);
1428     Str = Buffer;
1429   }
1430 
1431   auto StatusOrErr =
1432       Result.convertFromString(Str, APFloat::rmNearestTiesToEven);
1433   assert(StatusOrErr && "Invalid floating point representation");
1434   return !errorToBool(StatusOrErr.takeError()) ? *StatusOrErr
1435                                                : APFloat::opInvalidOp;
1436 }
1437 
IsExponentPart(char c)1438 static inline bool IsExponentPart(char c) {
1439   return c == 'p' || c == 'P' || c == 'e' || c == 'E';
1440 }
1441 
GetFixedPointValue(llvm::APInt & StoreVal,unsigned Scale)1442 bool NumericLiteralParser::GetFixedPointValue(llvm::APInt &StoreVal, unsigned Scale) {
1443   assert(radix == 16 || radix == 10);
1444 
1445   // Find how many digits are needed to store the whole literal.
1446   unsigned NumDigits = SuffixBegin - DigitsBegin;
1447   if (saw_period) --NumDigits;
1448 
1449   // Initial scan of the exponent if it exists
1450   bool ExpOverflowOccurred = false;
1451   bool NegativeExponent = false;
1452   const char *ExponentBegin;
1453   uint64_t Exponent = 0;
1454   int64_t BaseShift = 0;
1455   if (saw_exponent) {
1456     const char *Ptr = DigitsBegin;
1457 
1458     while (!IsExponentPart(*Ptr)) ++Ptr;
1459     ExponentBegin = Ptr;
1460     ++Ptr;
1461     NegativeExponent = *Ptr == '-';
1462     if (NegativeExponent) ++Ptr;
1463 
1464     unsigned NumExpDigits = SuffixBegin - Ptr;
1465     if (alwaysFitsInto64Bits(radix, NumExpDigits)) {
1466       llvm::StringRef ExpStr(Ptr, NumExpDigits);
1467       llvm::APInt ExpInt(/*numBits=*/64, ExpStr, /*radix=*/10);
1468       Exponent = ExpInt.getZExtValue();
1469     } else {
1470       ExpOverflowOccurred = true;
1471     }
1472 
1473     if (NegativeExponent) BaseShift -= Exponent;
1474     else BaseShift += Exponent;
1475   }
1476 
1477   // Number of bits needed for decimal literal is
1478   //   ceil(NumDigits * log2(10))       Integral part
1479   // + Scale                            Fractional part
1480   // + ceil(Exponent * log2(10))        Exponent
1481   // --------------------------------------------------
1482   //   ceil((NumDigits + Exponent) * log2(10)) + Scale
1483   //
1484   // But for simplicity in handling integers, we can round up log2(10) to 4,
1485   // making:
1486   // 4 * (NumDigits + Exponent) + Scale
1487   //
1488   // Number of digits needed for hexadecimal literal is
1489   //   4 * NumDigits                    Integral part
1490   // + Scale                            Fractional part
1491   // + Exponent                         Exponent
1492   // --------------------------------------------------
1493   //   (4 * NumDigits) + Scale + Exponent
1494   uint64_t NumBitsNeeded;
1495   if (radix == 10)
1496     NumBitsNeeded = 4 * (NumDigits + Exponent) + Scale;
1497   else
1498     NumBitsNeeded = 4 * NumDigits + Exponent + Scale;
1499 
1500   if (NumBitsNeeded > std::numeric_limits<unsigned>::max())
1501     ExpOverflowOccurred = true;
1502   llvm::APInt Val(static_cast<unsigned>(NumBitsNeeded), 0, /*isSigned=*/false);
1503 
1504   bool FoundDecimal = false;
1505 
1506   int64_t FractBaseShift = 0;
1507   const char *End = saw_exponent ? ExponentBegin : SuffixBegin;
1508   for (const char *Ptr = DigitsBegin; Ptr < End; ++Ptr) {
1509     if (*Ptr == '.') {
1510       FoundDecimal = true;
1511       continue;
1512     }
1513 
1514     // Normal reading of an integer
1515     unsigned C = llvm::hexDigitValue(*Ptr);
1516     assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1517 
1518     Val *= radix;
1519     Val += C;
1520 
1521     if (FoundDecimal)
1522       // Keep track of how much we will need to adjust this value by from the
1523       // number of digits past the radix point.
1524       --FractBaseShift;
1525   }
1526 
1527   // For a radix of 16, we will be multiplying by 2 instead of 16.
1528   if (radix == 16) FractBaseShift *= 4;
1529   BaseShift += FractBaseShift;
1530 
1531   Val <<= Scale;
1532 
1533   uint64_t Base = (radix == 16) ? 2 : 10;
1534   if (BaseShift > 0) {
1535     for (int64_t i = 0; i < BaseShift; ++i) {
1536       Val *= Base;
1537     }
1538   } else if (BaseShift < 0) {
1539     for (int64_t i = BaseShift; i < 0 && !Val.isZero(); ++i)
1540       Val = Val.udiv(Base);
1541   }
1542 
1543   bool IntOverflowOccurred = false;
1544   auto MaxVal = llvm::APInt::getMaxValue(StoreVal.getBitWidth());
1545   if (Val.getBitWidth() > StoreVal.getBitWidth()) {
1546     IntOverflowOccurred |= Val.ugt(MaxVal.zext(Val.getBitWidth()));
1547     StoreVal = Val.trunc(StoreVal.getBitWidth());
1548   } else if (Val.getBitWidth() < StoreVal.getBitWidth()) {
1549     IntOverflowOccurred |= Val.zext(MaxVal.getBitWidth()).ugt(MaxVal);
1550     StoreVal = Val.zext(StoreVal.getBitWidth());
1551   } else {
1552     StoreVal = Val;
1553   }
1554 
1555   return IntOverflowOccurred || ExpOverflowOccurred;
1556 }
1557 
1558 /// \verbatim
1559 ///       user-defined-character-literal: [C++11 lex.ext]
1560 ///         character-literal ud-suffix
1561 ///       ud-suffix:
1562 ///         identifier
1563 ///       character-literal: [C++11 lex.ccon]
1564 ///         ' c-char-sequence '
1565 ///         u' c-char-sequence '
1566 ///         U' c-char-sequence '
1567 ///         L' c-char-sequence '
1568 ///         u8' c-char-sequence ' [C++1z lex.ccon]
1569 ///       c-char-sequence:
1570 ///         c-char
1571 ///         c-char-sequence c-char
1572 ///       c-char:
1573 ///         any member of the source character set except the single-quote ',
1574 ///           backslash \, or new-line character
1575 ///         escape-sequence
1576 ///         universal-character-name
1577 ///       escape-sequence:
1578 ///         simple-escape-sequence
1579 ///         octal-escape-sequence
1580 ///         hexadecimal-escape-sequence
1581 ///       simple-escape-sequence:
1582 ///         one of \' \" \? \\ \a \b \f \n \r \t \v
1583 ///       octal-escape-sequence:
1584 ///         \ octal-digit
1585 ///         \ octal-digit octal-digit
1586 ///         \ octal-digit octal-digit octal-digit
1587 ///       hexadecimal-escape-sequence:
1588 ///         \x hexadecimal-digit
1589 ///         hexadecimal-escape-sequence hexadecimal-digit
1590 ///       universal-character-name: [C++11 lex.charset]
1591 ///         \u hex-quad
1592 ///         \U hex-quad hex-quad
1593 ///       hex-quad:
1594 ///         hex-digit hex-digit hex-digit hex-digit
1595 /// \endverbatim
1596 ///
CharLiteralParser(const char * begin,const char * end,SourceLocation Loc,Preprocessor & PP,tok::TokenKind kind)1597 CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
1598                                      SourceLocation Loc, Preprocessor &PP,
1599                                      tok::TokenKind kind) {
1600   // At this point we know that the character matches the regex "(L|u|U)?'.*'".
1601   HadError = false;
1602 
1603   Kind = kind;
1604 
1605   const char *TokBegin = begin;
1606 
1607   // Skip over wide character determinant.
1608   if (Kind != tok::char_constant)
1609     ++begin;
1610   if (Kind == tok::utf8_char_constant)
1611     ++begin;
1612 
1613   // Skip over the entry quote.
1614   if (begin[0] != '\'') {
1615     PP.Diag(Loc, diag::err_lexing_char);
1616     HadError = true;
1617     return;
1618   }
1619 
1620   ++begin;
1621 
1622   // Remove an optional ud-suffix.
1623   if (end[-1] != '\'') {
1624     const char *UDSuffixEnd = end;
1625     do {
1626       --end;
1627     } while (end[-1] != '\'');
1628     // FIXME: Don't bother with this if !tok.hasUCN().
1629     expandUCNs(UDSuffixBuf, StringRef(end, UDSuffixEnd - end));
1630     UDSuffixOffset = end - TokBegin;
1631   }
1632 
1633   // Trim the ending quote.
1634   assert(end != begin && "Invalid token lexed");
1635   --end;
1636 
1637   // FIXME: The "Value" is an uint64_t so we can handle char literals of
1638   // up to 64-bits.
1639   // FIXME: This extensively assumes that 'char' is 8-bits.
1640   assert(PP.getTargetInfo().getCharWidth() == 8 &&
1641          "Assumes char is 8 bits");
1642   assert(PP.getTargetInfo().getIntWidth() <= 64 &&
1643          (PP.getTargetInfo().getIntWidth() & 7) == 0 &&
1644          "Assumes sizeof(int) on target is <= 64 and a multiple of char");
1645   assert(PP.getTargetInfo().getWCharWidth() <= 64 &&
1646          "Assumes sizeof(wchar) on target is <= 64");
1647 
1648   SmallVector<uint32_t, 4> codepoint_buffer;
1649   codepoint_buffer.resize(end - begin);
1650   uint32_t *buffer_begin = &codepoint_buffer.front();
1651   uint32_t *buffer_end = buffer_begin + codepoint_buffer.size();
1652 
1653   // Unicode escapes representing characters that cannot be correctly
1654   // represented in a single code unit are disallowed in character literals
1655   // by this implementation.
1656   uint32_t largest_character_for_kind;
1657   if (tok::wide_char_constant == Kind) {
1658     largest_character_for_kind =
1659         0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth());
1660   } else if (tok::utf8_char_constant == Kind) {
1661     largest_character_for_kind = 0x7F;
1662   } else if (tok::utf16_char_constant == Kind) {
1663     largest_character_for_kind = 0xFFFF;
1664   } else if (tok::utf32_char_constant == Kind) {
1665     largest_character_for_kind = 0x10FFFF;
1666   } else {
1667     largest_character_for_kind = 0x7Fu;
1668   }
1669 
1670   while (begin != end) {
1671     // Is this a span of non-escape characters?
1672     if (begin[0] != '\\') {
1673       char const *start = begin;
1674       do {
1675         ++begin;
1676       } while (begin != end && *begin != '\\');
1677 
1678       char const *tmp_in_start = start;
1679       uint32_t *tmp_out_start = buffer_begin;
1680       llvm::ConversionResult res =
1681           llvm::ConvertUTF8toUTF32(reinterpret_cast<llvm::UTF8 const **>(&start),
1682                              reinterpret_cast<llvm::UTF8 const *>(begin),
1683                              &buffer_begin, buffer_end, llvm::strictConversion);
1684       if (res != llvm::conversionOK) {
1685         // If we see bad encoding for unprefixed character literals, warn and
1686         // simply copy the byte values, for compatibility with gcc and
1687         // older versions of clang.
1688         bool NoErrorOnBadEncoding = isOrdinary();
1689         unsigned Msg = diag::err_bad_character_encoding;
1690         if (NoErrorOnBadEncoding)
1691           Msg = diag::warn_bad_character_encoding;
1692         PP.Diag(Loc, Msg);
1693         if (NoErrorOnBadEncoding) {
1694           start = tmp_in_start;
1695           buffer_begin = tmp_out_start;
1696           for (; start != begin; ++start, ++buffer_begin)
1697             *buffer_begin = static_cast<uint8_t>(*start);
1698         } else {
1699           HadError = true;
1700         }
1701       } else {
1702         for (; tmp_out_start < buffer_begin; ++tmp_out_start) {
1703           if (*tmp_out_start > largest_character_for_kind) {
1704             HadError = true;
1705             PP.Diag(Loc, diag::err_character_too_large);
1706           }
1707         }
1708       }
1709 
1710       continue;
1711     }
1712     // Is this a Universal Character Name escape?
1713     if (begin[1] == 'u' || begin[1] == 'U' || begin[1] == 'N') {
1714       unsigned short UcnLen = 0;
1715       if (!ProcessUCNEscape(TokBegin, begin, end, *buffer_begin, UcnLen,
1716                             FullSourceLoc(Loc, PP.getSourceManager()),
1717                             &PP.getDiagnostics(), PP.getLangOpts(), true)) {
1718         HadError = true;
1719       } else if (*buffer_begin > largest_character_for_kind) {
1720         HadError = true;
1721         PP.Diag(Loc, diag::err_character_too_large);
1722       }
1723 
1724       ++buffer_begin;
1725       continue;
1726     }
1727     unsigned CharWidth = getCharWidth(Kind, PP.getTargetInfo());
1728     uint64_t result =
1729       ProcessCharEscape(TokBegin, begin, end, HadError,
1730                         FullSourceLoc(Loc,PP.getSourceManager()),
1731                         CharWidth, &PP.getDiagnostics(), PP.getLangOpts());
1732     *buffer_begin++ = result;
1733   }
1734 
1735   unsigned NumCharsSoFar = buffer_begin - &codepoint_buffer.front();
1736 
1737   if (NumCharsSoFar > 1) {
1738     if (isOrdinary() && NumCharsSoFar == 4)
1739       PP.Diag(Loc, diag::warn_four_char_character_literal);
1740     else if (isOrdinary())
1741       PP.Diag(Loc, diag::warn_multichar_character_literal);
1742     else {
1743       PP.Diag(Loc, diag::err_multichar_character_literal) << (isWide() ? 0 : 1);
1744       HadError = true;
1745     }
1746     IsMultiChar = true;
1747   } else {
1748     IsMultiChar = false;
1749   }
1750 
1751   llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0);
1752 
1753   // Narrow character literals act as though their value is concatenated
1754   // in this implementation, but warn on overflow.
1755   bool multi_char_too_long = false;
1756   if (isOrdinary() && isMultiChar()) {
1757     LitVal = 0;
1758     for (size_t i = 0; i < NumCharsSoFar; ++i) {
1759       // check for enough leading zeros to shift into
1760       multi_char_too_long |= (LitVal.countLeadingZeros() < 8);
1761       LitVal <<= 8;
1762       LitVal = LitVal + (codepoint_buffer[i] & 0xFF);
1763     }
1764   } else if (NumCharsSoFar > 0) {
1765     // otherwise just take the last character
1766     LitVal = buffer_begin[-1];
1767   }
1768 
1769   if (!HadError && multi_char_too_long) {
1770     PP.Diag(Loc, diag::warn_char_constant_too_large);
1771   }
1772 
1773   // Transfer the value from APInt to uint64_t
1774   Value = LitVal.getZExtValue();
1775 
1776   // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
1777   // if 'char' is signed for this target (C99 6.4.4.4p10).  Note that multiple
1778   // character constants are not sign extended in the this implementation:
1779   // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
1780   if (isOrdinary() && NumCharsSoFar == 1 && (Value & 128) &&
1781       PP.getLangOpts().CharIsSigned)
1782     Value = (signed char)Value;
1783 }
1784 
1785 /// \verbatim
1786 ///       string-literal: [C++0x lex.string]
1787 ///         encoding-prefix " [s-char-sequence] "
1788 ///         encoding-prefix R raw-string
1789 ///       encoding-prefix:
1790 ///         u8
1791 ///         u
1792 ///         U
1793 ///         L
1794 ///       s-char-sequence:
1795 ///         s-char
1796 ///         s-char-sequence s-char
1797 ///       s-char:
1798 ///         any member of the source character set except the double-quote ",
1799 ///           backslash \, or new-line character
1800 ///         escape-sequence
1801 ///         universal-character-name
1802 ///       raw-string:
1803 ///         " d-char-sequence ( r-char-sequence ) d-char-sequence "
1804 ///       r-char-sequence:
1805 ///         r-char
1806 ///         r-char-sequence r-char
1807 ///       r-char:
1808 ///         any member of the source character set, except a right parenthesis )
1809 ///           followed by the initial d-char-sequence (which may be empty)
1810 ///           followed by a double quote ".
1811 ///       d-char-sequence:
1812 ///         d-char
1813 ///         d-char-sequence d-char
1814 ///       d-char:
1815 ///         any member of the basic source character set except:
1816 ///           space, the left parenthesis (, the right parenthesis ),
1817 ///           the backslash \, and the control characters representing horizontal
1818 ///           tab, vertical tab, form feed, and newline.
1819 ///       escape-sequence: [C++0x lex.ccon]
1820 ///         simple-escape-sequence
1821 ///         octal-escape-sequence
1822 ///         hexadecimal-escape-sequence
1823 ///       simple-escape-sequence:
1824 ///         one of \' \" \? \\ \a \b \f \n \r \t \v
1825 ///       octal-escape-sequence:
1826 ///         \ octal-digit
1827 ///         \ octal-digit octal-digit
1828 ///         \ octal-digit octal-digit octal-digit
1829 ///       hexadecimal-escape-sequence:
1830 ///         \x hexadecimal-digit
1831 ///         hexadecimal-escape-sequence hexadecimal-digit
1832 ///       universal-character-name:
1833 ///         \u hex-quad
1834 ///         \U hex-quad hex-quad
1835 ///       hex-quad:
1836 ///         hex-digit hex-digit hex-digit hex-digit
1837 /// \endverbatim
1838 ///
1839 StringLiteralParser::
StringLiteralParser(ArrayRef<Token> StringToks,Preprocessor & PP)1840 StringLiteralParser(ArrayRef<Token> StringToks,
1841                     Preprocessor &PP)
1842   : SM(PP.getSourceManager()), Features(PP.getLangOpts()),
1843     Target(PP.getTargetInfo()), Diags(&PP.getDiagnostics()),
1844     MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown),
1845     ResultPtr(ResultBuf.data()), hadError(false), Pascal(false) {
1846   init(StringToks);
1847 }
1848 
init(ArrayRef<Token> StringToks)1849 void StringLiteralParser::init(ArrayRef<Token> StringToks){
1850   // The literal token may have come from an invalid source location (e.g. due
1851   // to a PCH error), in which case the token length will be 0.
1852   if (StringToks.empty() || StringToks[0].getLength() < 2)
1853     return DiagnoseLexingError(SourceLocation());
1854 
1855   // Scan all of the string portions, remember the max individual token length,
1856   // computing a bound on the concatenated string length, and see whether any
1857   // piece is a wide-string.  If any of the string portions is a wide-string
1858   // literal, the result is a wide-string literal [C99 6.4.5p4].
1859   assert(!StringToks.empty() && "expected at least one token");
1860   MaxTokenLength = StringToks[0].getLength();
1861   assert(StringToks[0].getLength() >= 2 && "literal token is invalid!");
1862   SizeBound = StringToks[0].getLength()-2;  // -2 for "".
1863   Kind = StringToks[0].getKind();
1864 
1865   hadError = false;
1866 
1867   // Implement Translation Phase #6: concatenation of string literals
1868   /// (C99 5.1.1.2p1).  The common case is only one string fragment.
1869   for (unsigned i = 1; i != StringToks.size(); ++i) {
1870     if (StringToks[i].getLength() < 2)
1871       return DiagnoseLexingError(StringToks[i].getLocation());
1872 
1873     // The string could be shorter than this if it needs cleaning, but this is a
1874     // reasonable bound, which is all we need.
1875     assert(StringToks[i].getLength() >= 2 && "literal token is invalid!");
1876     SizeBound += StringToks[i].getLength()-2;  // -2 for "".
1877 
1878     // Remember maximum string piece length.
1879     if (StringToks[i].getLength() > MaxTokenLength)
1880       MaxTokenLength = StringToks[i].getLength();
1881 
1882     // Remember if we see any wide or utf-8/16/32 strings.
1883     // Also check for illegal concatenations.
1884     if (StringToks[i].isNot(Kind) && StringToks[i].isNot(tok::string_literal)) {
1885       if (isOrdinary()) {
1886         Kind = StringToks[i].getKind();
1887       } else {
1888         if (Diags)
1889           Diags->Report(StringToks[i].getLocation(),
1890                         diag::err_unsupported_string_concat);
1891         hadError = true;
1892       }
1893     }
1894   }
1895 
1896   // Include space for the null terminator.
1897   ++SizeBound;
1898 
1899   // TODO: K&R warning: "traditional C rejects string constant concatenation"
1900 
1901   // Get the width in bytes of char/wchar_t/char16_t/char32_t
1902   CharByteWidth = getCharWidth(Kind, Target);
1903   assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple");
1904   CharByteWidth /= 8;
1905 
1906   // The output buffer size needs to be large enough to hold wide characters.
1907   // This is a worst-case assumption which basically corresponds to L"" "long".
1908   SizeBound *= CharByteWidth;
1909 
1910   // Size the temporary buffer to hold the result string data.
1911   ResultBuf.resize(SizeBound);
1912 
1913   // Likewise, but for each string piece.
1914   SmallString<512> TokenBuf;
1915   TokenBuf.resize(MaxTokenLength);
1916 
1917   // Loop over all the strings, getting their spelling, and expanding them to
1918   // wide strings as appropriate.
1919   ResultPtr = &ResultBuf[0];   // Next byte to fill in.
1920 
1921   Pascal = false;
1922 
1923   SourceLocation UDSuffixTokLoc;
1924 
1925   for (unsigned i = 0, e = StringToks.size(); i != e; ++i) {
1926     const char *ThisTokBuf = &TokenBuf[0];
1927     // Get the spelling of the token, which eliminates trigraphs, etc.  We know
1928     // that ThisTokBuf points to a buffer that is big enough for the whole token
1929     // and 'spelled' tokens can only shrink.
1930     bool StringInvalid = false;
1931     unsigned ThisTokLen =
1932       Lexer::getSpelling(StringToks[i], ThisTokBuf, SM, Features,
1933                          &StringInvalid);
1934     if (StringInvalid)
1935       return DiagnoseLexingError(StringToks[i].getLocation());
1936 
1937     const char *ThisTokBegin = ThisTokBuf;
1938     const char *ThisTokEnd = ThisTokBuf+ThisTokLen;
1939 
1940     // Remove an optional ud-suffix.
1941     if (ThisTokEnd[-1] != '"') {
1942       const char *UDSuffixEnd = ThisTokEnd;
1943       do {
1944         --ThisTokEnd;
1945       } while (ThisTokEnd[-1] != '"');
1946 
1947       StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd);
1948 
1949       if (UDSuffixBuf.empty()) {
1950         if (StringToks[i].hasUCN())
1951           expandUCNs(UDSuffixBuf, UDSuffix);
1952         else
1953           UDSuffixBuf.assign(UDSuffix);
1954         UDSuffixToken = i;
1955         UDSuffixOffset = ThisTokEnd - ThisTokBuf;
1956         UDSuffixTokLoc = StringToks[i].getLocation();
1957       } else {
1958         SmallString<32> ExpandedUDSuffix;
1959         if (StringToks[i].hasUCN()) {
1960           expandUCNs(ExpandedUDSuffix, UDSuffix);
1961           UDSuffix = ExpandedUDSuffix;
1962         }
1963 
1964         // C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the
1965         // result of a concatenation involving at least one user-defined-string-
1966         // literal, all the participating user-defined-string-literals shall
1967         // have the same ud-suffix.
1968         if (UDSuffixBuf != UDSuffix) {
1969           if (Diags) {
1970             SourceLocation TokLoc = StringToks[i].getLocation();
1971             Diags->Report(TokLoc, diag::err_string_concat_mixed_suffix)
1972               << UDSuffixBuf << UDSuffix
1973               << SourceRange(UDSuffixTokLoc, UDSuffixTokLoc)
1974               << SourceRange(TokLoc, TokLoc);
1975           }
1976           hadError = true;
1977         }
1978       }
1979     }
1980 
1981     // Strip the end quote.
1982     --ThisTokEnd;
1983 
1984     // TODO: Input character set mapping support.
1985 
1986     // Skip marker for wide or unicode strings.
1987     if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') {
1988       ++ThisTokBuf;
1989       // Skip 8 of u8 marker for utf8 strings.
1990       if (ThisTokBuf[0] == '8')
1991         ++ThisTokBuf;
1992     }
1993 
1994     // Check for raw string
1995     if (ThisTokBuf[0] == 'R') {
1996       if (ThisTokBuf[1] != '"') {
1997         // The file may have come from PCH and then changed after loading the
1998         // PCH; Fail gracefully.
1999         return DiagnoseLexingError(StringToks[i].getLocation());
2000       }
2001       ThisTokBuf += 2; // skip R"
2002 
2003       // C++11 [lex.string]p2: A `d-char-sequence` shall consist of at most 16
2004       // characters.
2005       constexpr unsigned MaxRawStrDelimLen = 16;
2006 
2007       const char *Prefix = ThisTokBuf;
2008       while (static_cast<unsigned>(ThisTokBuf - Prefix) < MaxRawStrDelimLen &&
2009              ThisTokBuf[0] != '(')
2010         ++ThisTokBuf;
2011       if (ThisTokBuf[0] != '(')
2012         return DiagnoseLexingError(StringToks[i].getLocation());
2013       ++ThisTokBuf; // skip '('
2014 
2015       // Remove same number of characters from the end
2016       ThisTokEnd -= ThisTokBuf - Prefix;
2017       if (ThisTokEnd < ThisTokBuf)
2018         return DiagnoseLexingError(StringToks[i].getLocation());
2019 
2020       // C++14 [lex.string]p4: A source-file new-line in a raw string literal
2021       // results in a new-line in the resulting execution string-literal.
2022       StringRef RemainingTokenSpan(ThisTokBuf, ThisTokEnd - ThisTokBuf);
2023       while (!RemainingTokenSpan.empty()) {
2024         // Split the string literal on \r\n boundaries.
2025         size_t CRLFPos = RemainingTokenSpan.find("\r\n");
2026         StringRef BeforeCRLF = RemainingTokenSpan.substr(0, CRLFPos);
2027         StringRef AfterCRLF = RemainingTokenSpan.substr(CRLFPos);
2028 
2029         // Copy everything before the \r\n sequence into the string literal.
2030         if (CopyStringFragment(StringToks[i], ThisTokBegin, BeforeCRLF))
2031           hadError = true;
2032 
2033         // Point into the \n inside the \r\n sequence and operate on the
2034         // remaining portion of the literal.
2035         RemainingTokenSpan = AfterCRLF.substr(1);
2036       }
2037     } else {
2038       if (ThisTokBuf[0] != '"') {
2039         // The file may have come from PCH and then changed after loading the
2040         // PCH; Fail gracefully.
2041         return DiagnoseLexingError(StringToks[i].getLocation());
2042       }
2043       ++ThisTokBuf; // skip "
2044 
2045       // Check if this is a pascal string
2046       if (Features.PascalStrings && ThisTokBuf + 1 != ThisTokEnd &&
2047           ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') {
2048 
2049         // If the \p sequence is found in the first token, we have a pascal string
2050         // Otherwise, if we already have a pascal string, ignore the first \p
2051         if (i == 0) {
2052           ++ThisTokBuf;
2053           Pascal = true;
2054         } else if (Pascal)
2055           ThisTokBuf += 2;
2056       }
2057 
2058       while (ThisTokBuf != ThisTokEnd) {
2059         // Is this a span of non-escape characters?
2060         if (ThisTokBuf[0] != '\\') {
2061           const char *InStart = ThisTokBuf;
2062           do {
2063             ++ThisTokBuf;
2064           } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
2065 
2066           // Copy the character span over.
2067           if (CopyStringFragment(StringToks[i], ThisTokBegin,
2068                                  StringRef(InStart, ThisTokBuf - InStart)))
2069             hadError = true;
2070           continue;
2071         }
2072         // Is this a Universal Character Name escape?
2073         if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U' ||
2074             ThisTokBuf[1] == 'N') {
2075           EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd,
2076                           ResultPtr, hadError,
2077                           FullSourceLoc(StringToks[i].getLocation(), SM),
2078                           CharByteWidth, Diags, Features);
2079           continue;
2080         }
2081         // Otherwise, this is a non-UCN escape character.  Process it.
2082         unsigned ResultChar =
2083           ProcessCharEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, hadError,
2084                             FullSourceLoc(StringToks[i].getLocation(), SM),
2085                             CharByteWidth*8, Diags, Features);
2086 
2087         if (CharByteWidth == 4) {
2088           // FIXME: Make the type of the result buffer correct instead of
2089           // using reinterpret_cast.
2090           llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultPtr);
2091           *ResultWidePtr = ResultChar;
2092           ResultPtr += 4;
2093         } else if (CharByteWidth == 2) {
2094           // FIXME: Make the type of the result buffer correct instead of
2095           // using reinterpret_cast.
2096           llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultPtr);
2097           *ResultWidePtr = ResultChar & 0xFFFF;
2098           ResultPtr += 2;
2099         } else {
2100           assert(CharByteWidth == 1 && "Unexpected char width");
2101           *ResultPtr++ = ResultChar & 0xFF;
2102         }
2103       }
2104     }
2105   }
2106 
2107   if (Pascal) {
2108     if (CharByteWidth == 4) {
2109       // FIXME: Make the type of the result buffer correct instead of
2110       // using reinterpret_cast.
2111       llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultBuf.data());
2112       ResultWidePtr[0] = GetNumStringChars() - 1;
2113     } else if (CharByteWidth == 2) {
2114       // FIXME: Make the type of the result buffer correct instead of
2115       // using reinterpret_cast.
2116       llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultBuf.data());
2117       ResultWidePtr[0] = GetNumStringChars() - 1;
2118     } else {
2119       assert(CharByteWidth == 1 && "Unexpected char width");
2120       ResultBuf[0] = GetNumStringChars() - 1;
2121     }
2122 
2123     // Verify that pascal strings aren't too large.
2124     if (GetStringLength() > 256) {
2125       if (Diags)
2126         Diags->Report(StringToks.front().getLocation(),
2127                       diag::err_pascal_string_too_long)
2128           << SourceRange(StringToks.front().getLocation(),
2129                          StringToks.back().getLocation());
2130       hadError = true;
2131       return;
2132     }
2133   } else if (Diags) {
2134     // Complain if this string literal has too many characters.
2135     unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509;
2136 
2137     if (GetNumStringChars() > MaxChars)
2138       Diags->Report(StringToks.front().getLocation(),
2139                     diag::ext_string_too_long)
2140         << GetNumStringChars() << MaxChars
2141         << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0)
2142         << SourceRange(StringToks.front().getLocation(),
2143                        StringToks.back().getLocation());
2144   }
2145 }
2146 
resyncUTF8(const char * Err,const char * End)2147 static const char *resyncUTF8(const char *Err, const char *End) {
2148   if (Err == End)
2149     return End;
2150   End = Err + std::min<unsigned>(llvm::getNumBytesForUTF8(*Err), End-Err);
2151   while (++Err != End && (*Err & 0xC0) == 0x80)
2152     ;
2153   return Err;
2154 }
2155 
2156 /// This function copies from Fragment, which is a sequence of bytes
2157 /// within Tok's contents (which begin at TokBegin) into ResultPtr.
2158 /// Performs widening for multi-byte characters.
CopyStringFragment(const Token & Tok,const char * TokBegin,StringRef Fragment)2159 bool StringLiteralParser::CopyStringFragment(const Token &Tok,
2160                                              const char *TokBegin,
2161                                              StringRef Fragment) {
2162   const llvm::UTF8 *ErrorPtrTmp;
2163   if (ConvertUTF8toWide(CharByteWidth, Fragment, ResultPtr, ErrorPtrTmp))
2164     return false;
2165 
2166   // If we see bad encoding for unprefixed string literals, warn and
2167   // simply copy the byte values, for compatibility with gcc and older
2168   // versions of clang.
2169   bool NoErrorOnBadEncoding = isOrdinary();
2170   if (NoErrorOnBadEncoding) {
2171     memcpy(ResultPtr, Fragment.data(), Fragment.size());
2172     ResultPtr += Fragment.size();
2173   }
2174 
2175   if (Diags) {
2176     const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
2177 
2178     FullSourceLoc SourceLoc(Tok.getLocation(), SM);
2179     const DiagnosticBuilder &Builder =
2180       Diag(Diags, Features, SourceLoc, TokBegin,
2181            ErrorPtr, resyncUTF8(ErrorPtr, Fragment.end()),
2182            NoErrorOnBadEncoding ? diag::warn_bad_string_encoding
2183                                 : diag::err_bad_string_encoding);
2184 
2185     const char *NextStart = resyncUTF8(ErrorPtr, Fragment.end());
2186     StringRef NextFragment(NextStart, Fragment.end()-NextStart);
2187 
2188     // Decode into a dummy buffer.
2189     SmallString<512> Dummy;
2190     Dummy.reserve(Fragment.size() * CharByteWidth);
2191     char *Ptr = Dummy.data();
2192 
2193     while (!ConvertUTF8toWide(CharByteWidth, NextFragment, Ptr, ErrorPtrTmp)) {
2194       const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
2195       NextStart = resyncUTF8(ErrorPtr, Fragment.end());
2196       Builder << MakeCharSourceRange(Features, SourceLoc, TokBegin,
2197                                      ErrorPtr, NextStart);
2198       NextFragment = StringRef(NextStart, Fragment.end()-NextStart);
2199     }
2200   }
2201   return !NoErrorOnBadEncoding;
2202 }
2203 
DiagnoseLexingError(SourceLocation Loc)2204 void StringLiteralParser::DiagnoseLexingError(SourceLocation Loc) {
2205   hadError = true;
2206   if (Diags)
2207     Diags->Report(Loc, diag::err_lexing_string);
2208 }
2209 
2210 /// getOffsetOfStringByte - This function returns the offset of the
2211 /// specified byte of the string data represented by Token.  This handles
2212 /// advancing over escape sequences in the string.
getOffsetOfStringByte(const Token & Tok,unsigned ByteNo) const2213 unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
2214                                                     unsigned ByteNo) const {
2215   // Get the spelling of the token.
2216   SmallString<32> SpellingBuffer;
2217   SpellingBuffer.resize(Tok.getLength());
2218 
2219   bool StringInvalid = false;
2220   const char *SpellingPtr = &SpellingBuffer[0];
2221   unsigned TokLen = Lexer::getSpelling(Tok, SpellingPtr, SM, Features,
2222                                        &StringInvalid);
2223   if (StringInvalid)
2224     return 0;
2225 
2226   const char *SpellingStart = SpellingPtr;
2227   const char *SpellingEnd = SpellingPtr+TokLen;
2228 
2229   // Handle UTF-8 strings just like narrow strings.
2230   if (SpellingPtr[0] == 'u' && SpellingPtr[1] == '8')
2231     SpellingPtr += 2;
2232 
2233   assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' &&
2234          SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet");
2235 
2236   // For raw string literals, this is easy.
2237   if (SpellingPtr[0] == 'R') {
2238     assert(SpellingPtr[1] == '"' && "Should be a raw string literal!");
2239     // Skip 'R"'.
2240     SpellingPtr += 2;
2241     while (*SpellingPtr != '(') {
2242       ++SpellingPtr;
2243       assert(SpellingPtr < SpellingEnd && "Missing ( for raw string literal");
2244     }
2245     // Skip '('.
2246     ++SpellingPtr;
2247     return SpellingPtr - SpellingStart + ByteNo;
2248   }
2249 
2250   // Skip over the leading quote
2251   assert(SpellingPtr[0] == '"' && "Should be a string literal!");
2252   ++SpellingPtr;
2253 
2254   // Skip over bytes until we find the offset we're looking for.
2255   while (ByteNo) {
2256     assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!");
2257 
2258     // Step over non-escapes simply.
2259     if (*SpellingPtr != '\\') {
2260       ++SpellingPtr;
2261       --ByteNo;
2262       continue;
2263     }
2264 
2265     // Otherwise, this is an escape character.  Advance over it.
2266     bool HadError = false;
2267     if (SpellingPtr[1] == 'u' || SpellingPtr[1] == 'U' ||
2268         SpellingPtr[1] == 'N') {
2269       const char *EscapePtr = SpellingPtr;
2270       unsigned Len = MeasureUCNEscape(SpellingStart, SpellingPtr, SpellingEnd,
2271                                       1, Features, HadError);
2272       if (Len > ByteNo) {
2273         // ByteNo is somewhere within the escape sequence.
2274         SpellingPtr = EscapePtr;
2275         break;
2276       }
2277       ByteNo -= Len;
2278     } else {
2279       ProcessCharEscape(SpellingStart, SpellingPtr, SpellingEnd, HadError,
2280                         FullSourceLoc(Tok.getLocation(), SM),
2281                         CharByteWidth*8, Diags, Features);
2282       --ByteNo;
2283     }
2284     assert(!HadError && "This method isn't valid on erroneous strings");
2285   }
2286 
2287   return SpellingPtr-SpellingStart;
2288 }
2289 
2290 /// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
2291 /// suffixes as ud-suffixes, because the diagnostic experience is better if we
2292 /// treat it as an invalid suffix.
isValidUDSuffix(const LangOptions & LangOpts,StringRef Suffix)2293 bool StringLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
2294                                           StringRef Suffix) {
2295   return NumericLiteralParser::isValidUDSuffix(LangOpts, Suffix) ||
2296          Suffix == "sv";
2297 }
2298