1 //===--- CommentLexer.cpp -------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/AST/CommentLexer.h"
10 #include "clang/AST/CommentCommandTraits.h"
11 #include "clang/AST/CommentDiagnostic.h"
12 #include "clang/Basic/CharInfo.h"
13 #include "llvm/ADT/StringExtras.h"
14 #include "llvm/ADT/StringSwitch.h"
15 #include "llvm/Support/ConvertUTF.h"
16 #include "llvm/Support/ErrorHandling.h"
17 
18 namespace clang {
19 namespace comments {
20 
21 void Token::dump(const Lexer &L, const SourceManager &SM) const {
22   llvm::errs() << "comments::Token Kind=" << Kind << " ";
23   Loc.print(llvm::errs(), SM);
24   llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
25 }
26 
27 static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
28   return isLetter(C);
29 }
30 
31 static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
32   return isDigit(C);
33 }
34 
35 static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
36   return isHexDigit(C);
37 }
38 
39 static inline StringRef convertCodePointToUTF8(
40                                       llvm::BumpPtrAllocator &Allocator,
41                                       unsigned CodePoint) {
42   char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
43   char *ResolvedPtr = Resolved;
44   if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
45     return StringRef(Resolved, ResolvedPtr - Resolved);
46   else
47     return StringRef();
48 }
49 
50 namespace {
51 
52 #include "clang/AST/CommentHTMLTags.inc"
53 #include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
54 
55 } // end anonymous namespace
56 
57 StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
58   // Fast path, first check a few most widely used named character references.
59   return llvm::StringSwitch<StringRef>(Name)
60       .Case("amp", "&")
61       .Case("lt", "<")
62       .Case("gt", ">")
63       .Case("quot", "\"")
64       .Case("apos", "\'")
65       // Slow path.
66       .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
67 }
68 
69 StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
70   unsigned CodePoint = 0;
71   for (unsigned i = 0, e = Name.size(); i != e; ++i) {
72     assert(isHTMLDecimalCharacterReferenceCharacter(Name[i]));
73     CodePoint *= 10;
74     CodePoint += Name[i] - '0';
75   }
76   return convertCodePointToUTF8(Allocator, CodePoint);
77 }
78 
79 StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
80   unsigned CodePoint = 0;
81   for (unsigned i = 0, e = Name.size(); i != e; ++i) {
82     CodePoint *= 16;
83     const char C = Name[i];
84     assert(isHTMLHexCharacterReferenceCharacter(C));
85     CodePoint += llvm::hexDigitValue(C);
86   }
87   return convertCodePointToUTF8(Allocator, CodePoint);
88 }
89 
90 void Lexer::skipLineStartingDecorations() {
91   // This function should be called only for C comments
92   assert(CommentState == LCS_InsideCComment);
93 
94   if (BufferPtr == CommentEnd)
95     return;
96 
97   const char *NewBufferPtr = BufferPtr;
98   while (isHorizontalWhitespace(*NewBufferPtr))
99     if (++NewBufferPtr == CommentEnd)
100       return;
101   if (*NewBufferPtr == '*')
102     BufferPtr = NewBufferPtr + 1;
103 }
104 
105 namespace {
106 /// Returns pointer to the first newline character in the string.
107 const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
108   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
109     if (isVerticalWhitespace(*BufferPtr))
110       return BufferPtr;
111   }
112   return BufferEnd;
113 }
114 
115 const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
116   if (BufferPtr == BufferEnd)
117     return BufferPtr;
118 
119   if (*BufferPtr == '\n')
120     BufferPtr++;
121   else {
122     assert(*BufferPtr == '\r');
123     BufferPtr++;
124     if (BufferPtr != BufferEnd && *BufferPtr == '\n')
125       BufferPtr++;
126   }
127   return BufferPtr;
128 }
129 
130 const char *skipNamedCharacterReference(const char *BufferPtr,
131                                         const char *BufferEnd) {
132   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
133     if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr))
134       return BufferPtr;
135   }
136   return BufferEnd;
137 }
138 
139 const char *skipDecimalCharacterReference(const char *BufferPtr,
140                                           const char *BufferEnd) {
141   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
142     if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr))
143       return BufferPtr;
144   }
145   return BufferEnd;
146 }
147 
148 const char *skipHexCharacterReference(const char *BufferPtr,
149                                       const char *BufferEnd) {
150   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
151     if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
152       return BufferPtr;
153   }
154   return BufferEnd;
155 }
156 
157 bool isHTMLIdentifierStartingCharacter(char C) {
158   return isLetter(C);
159 }
160 
161 bool isHTMLIdentifierCharacter(char C) {
162   return isAlphanumeric(C);
163 }
164 
165 const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
166   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
167     if (!isHTMLIdentifierCharacter(*BufferPtr))
168       return BufferPtr;
169   }
170   return BufferEnd;
171 }
172 
173 /// Skip HTML string quoted in single or double quotes.  Escaping quotes inside
174 /// string allowed.
175 ///
176 /// Returns pointer to closing quote.
177 const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
178 {
179   const char Quote = *BufferPtr;
180   assert(Quote == '\"' || Quote == '\'');
181 
182   BufferPtr++;
183   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
184     const char C = *BufferPtr;
185     if (C == Quote && BufferPtr[-1] != '\\')
186       return BufferPtr;
187   }
188   return BufferEnd;
189 }
190 
191 const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
192   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
193     if (!isWhitespace(*BufferPtr))
194       return BufferPtr;
195   }
196   return BufferEnd;
197 }
198 
199 bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
200   return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
201 }
202 
203 bool isCommandNameStartCharacter(char C) {
204   return isLetter(C);
205 }
206 
207 bool isCommandNameCharacter(char C) {
208   return isAlphanumeric(C);
209 }
210 
211 const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
212   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
213     if (!isCommandNameCharacter(*BufferPtr))
214       return BufferPtr;
215   }
216   return BufferEnd;
217 }
218 
219 /// Return the one past end pointer for BCPL comments.
220 /// Handles newlines escaped with backslash or trigraph for backslahs.
221 const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
222   const char *CurPtr = BufferPtr;
223   while (CurPtr != BufferEnd) {
224     while (!isVerticalWhitespace(*CurPtr)) {
225       CurPtr++;
226       if (CurPtr == BufferEnd)
227         return BufferEnd;
228     }
229     // We found a newline, check if it is escaped.
230     const char *EscapePtr = CurPtr - 1;
231     while(isHorizontalWhitespace(*EscapePtr))
232       EscapePtr--;
233 
234     if (*EscapePtr == '\\' ||
235         (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' &&
236          EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) {
237       // We found an escaped newline.
238       CurPtr = skipNewline(CurPtr, BufferEnd);
239     } else
240       return CurPtr; // Not an escaped newline.
241   }
242   return BufferEnd;
243 }
244 
245 /// Return the one past end pointer for C comments.
246 /// Very dumb, does not handle escaped newlines or trigraphs.
247 const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
248   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
249     if (*BufferPtr == '*') {
250       assert(BufferPtr + 1 != BufferEnd);
251       if (*(BufferPtr + 1) == '/')
252         return BufferPtr;
253     }
254   }
255   llvm_unreachable("buffer end hit before '*/' was seen");
256 }
257 
258 } // end anonymous namespace
259 
260 void Lexer::formTokenWithChars(Token &Result, const char *TokEnd,
261                                tok::TokenKind Kind) {
262   const unsigned TokLen = TokEnd - BufferPtr;
263   Result.setLocation(getSourceLocation(BufferPtr));
264   Result.setKind(Kind);
265   Result.setLength(TokLen);
266 #ifndef NDEBUG
267   Result.TextPtr = "<UNSET>";
268   Result.IntVal = 7;
269 #endif
270   BufferPtr = TokEnd;
271 }
272 
273 const char *Lexer::skipTextToken() {
274   const char *TokenPtr = BufferPtr;
275   assert(TokenPtr < CommentEnd);
276   StringRef TokStartSymbols = ParseCommands ? "\n\r\\@\"&<" : "\n\r";
277 
278 again:
279   size_t End =
280       StringRef(TokenPtr, CommentEnd - TokenPtr).find_first_of(TokStartSymbols);
281   if (End == StringRef::npos)
282     return CommentEnd;
283 
284   // Doxygen doesn't recognize any commands in a one-line double quotation.
285   // If we don't find an ending quotation mark, we pretend it never began.
286   if (*(TokenPtr + End) == '\"') {
287     TokenPtr += End + 1;
288     End = StringRef(TokenPtr, CommentEnd - TokenPtr).find_first_of("\n\r\"");
289     if (End != StringRef::npos && *(TokenPtr + End) == '\"')
290       TokenPtr += End + 1;
291     goto again;
292   }
293   return TokenPtr + End;
294 }
295 
296 void Lexer::lexCommentText(Token &T) {
297   assert(CommentState == LCS_InsideBCPLComment ||
298          CommentState == LCS_InsideCComment);
299 
300   // Handles lexing non-command text, i.e. text and newline.
301   auto HandleNonCommandToken = [&]() -> void {
302     assert(State == LS_Normal);
303 
304     const char *TokenPtr = BufferPtr;
305     assert(TokenPtr < CommentEnd);
306     switch (*TokenPtr) {
307       case '\n':
308       case '\r':
309           TokenPtr = skipNewline(TokenPtr, CommentEnd);
310           formTokenWithChars(T, TokenPtr, tok::newline);
311 
312           if (CommentState == LCS_InsideCComment)
313             skipLineStartingDecorations();
314           return;
315 
316       default:
317         return formTextToken(T, skipTextToken());
318     }
319   };
320 
321   if (!ParseCommands)
322     return HandleNonCommandToken();
323 
324   switch (State) {
325   case LS_Normal:
326     break;
327   case LS_VerbatimBlockFirstLine:
328     lexVerbatimBlockFirstLine(T);
329     return;
330   case LS_VerbatimBlockBody:
331     lexVerbatimBlockBody(T);
332     return;
333   case LS_VerbatimLineText:
334     lexVerbatimLineText(T);
335     return;
336   case LS_HTMLStartTag:
337     lexHTMLStartTag(T);
338     return;
339   case LS_HTMLEndTag:
340     lexHTMLEndTag(T);
341     return;
342   }
343 
344   assert(State == LS_Normal);
345   const char *TokenPtr = BufferPtr;
346   assert(TokenPtr < CommentEnd);
347   switch(*TokenPtr) {
348     case '\\':
349     case '@': {
350       // Commands that start with a backslash and commands that start with
351       // 'at' have equivalent semantics.  But we keep information about the
352       // exact syntax in AST for comments.
353       tok::TokenKind CommandKind =
354           (*TokenPtr == '@') ? tok::at_command : tok::backslash_command;
355       TokenPtr++;
356       if (TokenPtr == CommentEnd) {
357         formTextToken(T, TokenPtr);
358         return;
359       }
360       char C = *TokenPtr;
361       switch (C) {
362       default:
363         break;
364 
365       case '\\': case '@': case '&': case '$':
366       case '#':  case '<': case '>': case '%':
367       case '\"': case '.': case ':':
368         // This is one of \\ \@ \& \$ etc escape sequences.
369         TokenPtr++;
370         if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') {
371           // This is the \:: escape sequence.
372           TokenPtr++;
373         }
374         StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
375         formTokenWithChars(T, TokenPtr, tok::text);
376         T.setText(UnescapedText);
377         return;
378       }
379 
380       // Don't make zero-length commands.
381       if (!isCommandNameStartCharacter(*TokenPtr)) {
382         formTextToken(T, TokenPtr);
383         return;
384       }
385 
386       TokenPtr = skipCommandName(TokenPtr, CommentEnd);
387       unsigned Length = TokenPtr - (BufferPtr + 1);
388 
389       // Hardcoded support for lexing LaTeX formula commands
390       // \f$ \f( \f) \f[ \f] \f{ \f} as a single command.
391       if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
392         C = *TokenPtr;
393         if (C == '$' || C == '(' || C == ')' || C == '[' || C == ']' ||
394             C == '{' || C == '}') {
395           TokenPtr++;
396           Length++;
397         }
398       }
399 
400       StringRef CommandName(BufferPtr + 1, Length);
401 
402       const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
403       if (!Info) {
404         if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
405           StringRef CorrectedName = Info->Name;
406           SourceLocation Loc = getSourceLocation(BufferPtr);
407           SourceLocation EndLoc = getSourceLocation(TokenPtr);
408           SourceRange FullRange = SourceRange(Loc, EndLoc);
409           SourceRange CommandRange(Loc.getLocWithOffset(1), EndLoc);
410           Diag(Loc, diag::warn_correct_comment_command_name)
411             << FullRange << CommandName << CorrectedName
412             << FixItHint::CreateReplacement(CommandRange, CorrectedName);
413         } else {
414           formTokenWithChars(T, TokenPtr, tok::unknown_command);
415           T.setUnknownCommandName(CommandName);
416           Diag(T.getLocation(), diag::warn_unknown_comment_command_name)
417               << SourceRange(T.getLocation(), T.getEndLocation());
418           return;
419         }
420       }
421       if (Info->IsVerbatimBlockCommand) {
422         setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
423         return;
424       }
425       if (Info->IsVerbatimLineCommand) {
426         setupAndLexVerbatimLine(T, TokenPtr, Info);
427         return;
428       }
429       formTokenWithChars(T, TokenPtr, CommandKind);
430       T.setCommandID(Info->getID());
431       return;
432     }
433 
434     case '&':
435       lexHTMLCharacterReference(T);
436       return;
437 
438     case '<': {
439       TokenPtr++;
440       if (TokenPtr == CommentEnd) {
441         formTextToken(T, TokenPtr);
442         return;
443       }
444       const char C = *TokenPtr;
445       if (isHTMLIdentifierStartingCharacter(C))
446         setupAndLexHTMLStartTag(T);
447       else if (C == '/')
448         setupAndLexHTMLEndTag(T);
449       else
450         formTextToken(T, TokenPtr);
451       return;
452     }
453 
454     default:
455       return HandleNonCommandToken();
456   }
457 }
458 
459 void Lexer::setupAndLexVerbatimBlock(Token &T,
460                                      const char *TextBegin,
461                                      char Marker, const CommandInfo *Info) {
462   assert(Info->IsVerbatimBlockCommand);
463 
464   VerbatimBlockEndCommandName.clear();
465   VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
466   VerbatimBlockEndCommandName.append(Info->EndCommandName);
467 
468   formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
469   T.setVerbatimBlockID(Info->getID());
470 
471   // If there is a newline following the verbatim opening command, skip the
472   // newline so that we don't create an tok::verbatim_block_line with empty
473   // text content.
474   if (BufferPtr != CommentEnd &&
475       isVerticalWhitespace(*BufferPtr)) {
476     BufferPtr = skipNewline(BufferPtr, CommentEnd);
477     State = LS_VerbatimBlockBody;
478     return;
479   }
480 
481   State = LS_VerbatimBlockFirstLine;
482 }
483 
484 void Lexer::lexVerbatimBlockFirstLine(Token &T) {
485 again:
486   assert(BufferPtr < CommentEnd);
487 
488   // FIXME: It would be better to scan the text once, finding either the block
489   // end command or newline.
490   //
491   // Extract current line.
492   const char *Newline = findNewline(BufferPtr, CommentEnd);
493   StringRef Line(BufferPtr, Newline - BufferPtr);
494 
495   // Look for end command in current line.
496   size_t Pos = Line.find(VerbatimBlockEndCommandName);
497   const char *TextEnd;
498   const char *NextLine;
499   if (Pos == StringRef::npos) {
500     // Current line is completely verbatim.
501     TextEnd = Newline;
502     NextLine = skipNewline(Newline, CommentEnd);
503   } else if (Pos == 0) {
504     // Current line contains just an end command.
505     const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
506     StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
507     formTokenWithChars(T, End, tok::verbatim_block_end);
508     T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
509     State = LS_Normal;
510     return;
511   } else {
512     // There is some text, followed by end command.  Extract text first.
513     TextEnd = BufferPtr + Pos;
514     NextLine = TextEnd;
515     // If there is only whitespace before end command, skip whitespace.
516     if (isWhitespace(BufferPtr, TextEnd)) {
517       BufferPtr = TextEnd;
518       goto again;
519     }
520   }
521 
522   StringRef Text(BufferPtr, TextEnd - BufferPtr);
523   formTokenWithChars(T, NextLine, tok::verbatim_block_line);
524   T.setVerbatimBlockText(Text);
525 
526   State = LS_VerbatimBlockBody;
527 }
528 
529 void Lexer::lexVerbatimBlockBody(Token &T) {
530   assert(State == LS_VerbatimBlockBody);
531 
532   if (CommentState == LCS_InsideCComment)
533     skipLineStartingDecorations();
534 
535   if (BufferPtr == CommentEnd) {
536     formTokenWithChars(T, BufferPtr, tok::verbatim_block_line);
537     T.setVerbatimBlockText("");
538     return;
539   }
540 
541   lexVerbatimBlockFirstLine(T);
542 }
543 
544 void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
545                                     const CommandInfo *Info) {
546   assert(Info->IsVerbatimLineCommand);
547   formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
548   T.setVerbatimLineID(Info->getID());
549 
550   State = LS_VerbatimLineText;
551 }
552 
553 void Lexer::lexVerbatimLineText(Token &T) {
554   assert(State == LS_VerbatimLineText);
555 
556   // Extract current line.
557   const char *Newline = findNewline(BufferPtr, CommentEnd);
558   StringRef Text(BufferPtr, Newline - BufferPtr);
559   formTokenWithChars(T, Newline, tok::verbatim_line_text);
560   T.setVerbatimLineText(Text);
561 
562   State = LS_Normal;
563 }
564 
565 void Lexer::lexHTMLCharacterReference(Token &T) {
566   const char *TokenPtr = BufferPtr;
567   assert(*TokenPtr == '&');
568   TokenPtr++;
569   if (TokenPtr == CommentEnd) {
570     formTextToken(T, TokenPtr);
571     return;
572   }
573   const char *NamePtr;
574   bool isNamed = false;
575   bool isDecimal = false;
576   char C = *TokenPtr;
577   if (isHTMLNamedCharacterReferenceCharacter(C)) {
578     NamePtr = TokenPtr;
579     TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
580     isNamed = true;
581   } else if (C == '#') {
582     TokenPtr++;
583     if (TokenPtr == CommentEnd) {
584       formTextToken(T, TokenPtr);
585       return;
586     }
587     C = *TokenPtr;
588     if (isHTMLDecimalCharacterReferenceCharacter(C)) {
589       NamePtr = TokenPtr;
590       TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
591       isDecimal = true;
592     } else if (C == 'x' || C == 'X') {
593       TokenPtr++;
594       NamePtr = TokenPtr;
595       TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
596     } else {
597       formTextToken(T, TokenPtr);
598       return;
599     }
600   } else {
601     formTextToken(T, TokenPtr);
602     return;
603   }
604   if (NamePtr == TokenPtr || TokenPtr == CommentEnd ||
605       *TokenPtr != ';') {
606     formTextToken(T, TokenPtr);
607     return;
608   }
609   StringRef Name(NamePtr, TokenPtr - NamePtr);
610   TokenPtr++; // Skip semicolon.
611   StringRef Resolved;
612   if (isNamed)
613     Resolved = resolveHTMLNamedCharacterReference(Name);
614   else if (isDecimal)
615     Resolved = resolveHTMLDecimalCharacterReference(Name);
616   else
617     Resolved = resolveHTMLHexCharacterReference(Name);
618 
619   if (Resolved.empty()) {
620     formTextToken(T, TokenPtr);
621     return;
622   }
623   formTokenWithChars(T, TokenPtr, tok::text);
624   T.setText(Resolved);
625 }
626 
627 void Lexer::setupAndLexHTMLStartTag(Token &T) {
628   assert(BufferPtr[0] == '<' &&
629          isHTMLIdentifierStartingCharacter(BufferPtr[1]));
630   const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
631   StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
632   if (!isHTMLTagName(Name)) {
633     formTextToken(T, TagNameEnd);
634     return;
635   }
636 
637   formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
638   T.setHTMLTagStartName(Name);
639 
640   BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
641 
642   const char C = *BufferPtr;
643   if (BufferPtr != CommentEnd &&
644       (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
645     State = LS_HTMLStartTag;
646 }
647 
648 void Lexer::lexHTMLStartTag(Token &T) {
649   assert(State == LS_HTMLStartTag);
650 
651   const char *TokenPtr = BufferPtr;
652   char C = *TokenPtr;
653   if (isHTMLIdentifierCharacter(C)) {
654     TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
655     StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
656     formTokenWithChars(T, TokenPtr, tok::html_ident);
657     T.setHTMLIdent(Ident);
658   } else {
659     switch (C) {
660     case '=':
661       TokenPtr++;
662       formTokenWithChars(T, TokenPtr, tok::html_equals);
663       break;
664     case '\"':
665     case '\'': {
666       const char *OpenQuote = TokenPtr;
667       TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
668       const char *ClosingQuote = TokenPtr;
669       if (TokenPtr != CommentEnd) // Skip closing quote.
670         TokenPtr++;
671       formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
672       T.setHTMLQuotedString(StringRef(OpenQuote + 1,
673                                       ClosingQuote - (OpenQuote + 1)));
674       break;
675     }
676     case '>':
677       TokenPtr++;
678       formTokenWithChars(T, TokenPtr, tok::html_greater);
679       State = LS_Normal;
680       return;
681     case '/':
682       TokenPtr++;
683       if (TokenPtr != CommentEnd && *TokenPtr == '>') {
684         TokenPtr++;
685         formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
686       } else
687         formTextToken(T, TokenPtr);
688 
689       State = LS_Normal;
690       return;
691     }
692   }
693 
694   // Now look ahead and return to normal state if we don't see any HTML tokens
695   // ahead.
696   BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
697   if (BufferPtr == CommentEnd) {
698     State = LS_Normal;
699     return;
700   }
701 
702   C = *BufferPtr;
703   if (!isHTMLIdentifierStartingCharacter(C) &&
704       C != '=' && C != '\"' && C != '\'' && C != '>') {
705     State = LS_Normal;
706     return;
707   }
708 }
709 
710 void Lexer::setupAndLexHTMLEndTag(Token &T) {
711   assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
712 
713   const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
714   const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
715   StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
716   if (!isHTMLTagName(Name)) {
717     formTextToken(T, TagNameEnd);
718     return;
719   }
720 
721   const char *End = skipWhitespace(TagNameEnd, CommentEnd);
722 
723   formTokenWithChars(T, End, tok::html_end_tag);
724   T.setHTMLTagEndName(Name);
725 
726   if (BufferPtr != CommentEnd && *BufferPtr == '>')
727     State = LS_HTMLEndTag;
728 }
729 
730 void Lexer::lexHTMLEndTag(Token &T) {
731   assert(BufferPtr != CommentEnd && *BufferPtr == '>');
732 
733   formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
734   State = LS_Normal;
735 }
736 
737 Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
738              const CommandTraits &Traits, SourceLocation FileLoc,
739              const char *BufferStart, const char *BufferEnd, bool ParseCommands)
740     : Allocator(Allocator), Diags(Diags), Traits(Traits),
741       BufferStart(BufferStart), BufferEnd(BufferEnd), BufferPtr(BufferStart),
742       FileLoc(FileLoc), ParseCommands(ParseCommands),
743       CommentState(LCS_BeforeComment), State(LS_Normal) {}
744 
745 void Lexer::lex(Token &T) {
746 again:
747   switch (CommentState) {
748   case LCS_BeforeComment:
749     if (BufferPtr == BufferEnd) {
750       formTokenWithChars(T, BufferPtr, tok::eof);
751       return;
752     }
753 
754     assert(*BufferPtr == '/');
755     BufferPtr++; // Skip first slash.
756     switch(*BufferPtr) {
757     case '/': { // BCPL comment.
758       BufferPtr++; // Skip second slash.
759 
760       if (BufferPtr != BufferEnd) {
761         // Skip Doxygen magic marker, if it is present.
762         // It might be missing because of a typo //< or /*<, or because we
763         // merged this non-Doxygen comment into a bunch of Doxygen comments
764         // around it: /** ... */ /* ... */ /** ... */
765         const char C = *BufferPtr;
766         if (C == '/' || C == '!')
767           BufferPtr++;
768       }
769 
770       // Skip less-than symbol that marks trailing comments.
771       // Skip it even if the comment is not a Doxygen one, because //< and /*<
772       // are frequent typos.
773       if (BufferPtr != BufferEnd && *BufferPtr == '<')
774         BufferPtr++;
775 
776       CommentState = LCS_InsideBCPLComment;
777       if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
778         State = LS_Normal;
779       CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
780       goto again;
781     }
782     case '*': { // C comment.
783       BufferPtr++; // Skip star.
784 
785       // Skip Doxygen magic marker.
786       const char C = *BufferPtr;
787       if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!')
788         BufferPtr++;
789 
790       // Skip less-than symbol that marks trailing comments.
791       if (BufferPtr != BufferEnd && *BufferPtr == '<')
792         BufferPtr++;
793 
794       CommentState = LCS_InsideCComment;
795       State = LS_Normal;
796       CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
797       goto again;
798     }
799     default:
800       llvm_unreachable("second character of comment should be '/' or '*'");
801     }
802 
803   case LCS_BetweenComments: {
804     // Consecutive comments are extracted only if there is only whitespace
805     // between them.  So we can search for the start of the next comment.
806     const char *EndWhitespace = BufferPtr;
807     while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
808       EndWhitespace++;
809 
810     // Turn any whitespace between comments (and there is only whitespace
811     // between them -- guaranteed by comment extraction) into a newline.  We
812     // have two newlines between C comments in total (first one was synthesized
813     // after a comment).
814     formTokenWithChars(T, EndWhitespace, tok::newline);
815 
816     CommentState = LCS_BeforeComment;
817     break;
818   }
819 
820   case LCS_InsideBCPLComment:
821   case LCS_InsideCComment:
822     if (BufferPtr != CommentEnd) {
823       lexCommentText(T);
824       break;
825     } else {
826       // Skip C comment closing sequence.
827       if (CommentState == LCS_InsideCComment) {
828         assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
829         BufferPtr += 2;
830         assert(BufferPtr <= BufferEnd);
831 
832         // Synthenize newline just after the C comment, regardless if there is
833         // actually a newline.
834         formTokenWithChars(T, BufferPtr, tok::newline);
835 
836         CommentState = LCS_BetweenComments;
837         break;
838       } else {
839         // Don't synthesized a newline after BCPL comment.
840         CommentState = LCS_BetweenComments;
841         goto again;
842       }
843     }
844   }
845 }
846 
847 StringRef Lexer::getSpelling(const Token &Tok,
848                              const SourceManager &SourceMgr) const {
849   SourceLocation Loc = Tok.getLocation();
850   std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
851 
852   bool InvalidTemp = false;
853   StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
854   if (InvalidTemp)
855     return StringRef();
856 
857   const char *Begin = File.data() + LocInfo.second;
858   return StringRef(Begin, Tok.getLength());
859 }
860 
861 } // end namespace comments
862 } // end namespace clang
863