1 //===--- CommentParser.cpp - Doxygen comment parser -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/AST/CommentParser.h"
10 #include "clang/AST/CommentCommandTraits.h"
11 #include "clang/AST/CommentDiagnostic.h"
12 #include "clang/AST/CommentSema.h"
13 #include "clang/Basic/CharInfo.h"
14 #include "clang/Basic/SourceManager.h"
15 #include "llvm/Support/ErrorHandling.h"
16 
17 namespace clang {
18 
19 static inline bool isWhitespace(llvm::StringRef S) {
20   for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; ++I) {
21     if (!isWhitespace(*I))
22       return false;
23   }
24   return true;
25 }
26 
27 namespace comments {
28 
29 /// Re-lexes a sequence of tok::text tokens.
30 class TextTokenRetokenizer {
31   llvm::BumpPtrAllocator &Allocator;
32   Parser &P;
33 
34   /// This flag is set when there are no more tokens we can fetch from lexer.
35   bool NoMoreInterestingTokens;
36 
37   /// Token buffer: tokens we have processed and lookahead.
38   SmallVector<Token, 16> Toks;
39 
40   /// A position in \c Toks.
41   struct Position {
42     const char *BufferStart;
43     const char *BufferEnd;
44     const char *BufferPtr;
45     SourceLocation BufferStartLoc;
46     unsigned CurToken;
47   };
48 
49   /// Current position in Toks.
50   Position Pos;
51 
52   bool isEnd() const {
53     return Pos.CurToken >= Toks.size();
54   }
55 
56   /// Sets up the buffer pointers to point to current token.
57   void setupBuffer() {
58     assert(!isEnd());
59     const Token &Tok = Toks[Pos.CurToken];
60 
61     Pos.BufferStart = Tok.getText().begin();
62     Pos.BufferEnd = Tok.getText().end();
63     Pos.BufferPtr = Pos.BufferStart;
64     Pos.BufferStartLoc = Tok.getLocation();
65   }
66 
67   SourceLocation getSourceLocation() const {
68     const unsigned CharNo = Pos.BufferPtr - Pos.BufferStart;
69     return Pos.BufferStartLoc.getLocWithOffset(CharNo);
70   }
71 
72   char peek() const {
73     assert(!isEnd());
74     assert(Pos.BufferPtr != Pos.BufferEnd);
75     return *Pos.BufferPtr;
76   }
77 
78   void consumeChar() {
79     assert(!isEnd());
80     assert(Pos.BufferPtr != Pos.BufferEnd);
81     Pos.BufferPtr++;
82     if (Pos.BufferPtr == Pos.BufferEnd) {
83       Pos.CurToken++;
84       if (isEnd() && !addToken())
85         return;
86 
87       assert(!isEnd());
88       setupBuffer();
89     }
90   }
91 
92   /// Add a token.
93   /// Returns true on success, false if there are no interesting tokens to
94   /// fetch from lexer.
95   bool addToken() {
96     if (NoMoreInterestingTokens)
97       return false;
98 
99     if (P.Tok.is(tok::newline)) {
100       // If we see a single newline token between text tokens, skip it.
101       Token Newline = P.Tok;
102       P.consumeToken();
103       if (P.Tok.isNot(tok::text)) {
104         P.putBack(Newline);
105         NoMoreInterestingTokens = true;
106         return false;
107       }
108     }
109     if (P.Tok.isNot(tok::text)) {
110       NoMoreInterestingTokens = true;
111       return false;
112     }
113 
114     Toks.push_back(P.Tok);
115     P.consumeToken();
116     if (Toks.size() == 1)
117       setupBuffer();
118     return true;
119   }
120 
121   void consumeWhitespace() {
122     while (!isEnd()) {
123       if (isWhitespace(peek()))
124         consumeChar();
125       else
126         break;
127     }
128   }
129 
130   void formTokenWithChars(Token &Result,
131                           SourceLocation Loc,
132                           const char *TokBegin,
133                           unsigned TokLength,
134                           StringRef Text) {
135     Result.setLocation(Loc);
136     Result.setKind(tok::text);
137     Result.setLength(TokLength);
138 #ifndef NDEBUG
139     Result.TextPtr = "<UNSET>";
140     Result.IntVal = 7;
141 #endif
142     Result.setText(Text);
143   }
144 
145 public:
146   TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P):
147       Allocator(Allocator), P(P), NoMoreInterestingTokens(false) {
148     Pos.CurToken = 0;
149     addToken();
150   }
151 
152   /// Extract a word -- sequence of non-whitespace characters.
153   bool lexWord(Token &Tok) {
154     if (isEnd())
155       return false;
156 
157     Position SavedPos = Pos;
158 
159     consumeWhitespace();
160     SmallString<32> WordText;
161     const char *WordBegin = Pos.BufferPtr;
162     SourceLocation Loc = getSourceLocation();
163     while (!isEnd()) {
164       const char C = peek();
165       if (!isWhitespace(C)) {
166         WordText.push_back(C);
167         consumeChar();
168       } else
169         break;
170     }
171     const unsigned Length = WordText.size();
172     if (Length == 0) {
173       Pos = SavedPos;
174       return false;
175     }
176 
177     char *TextPtr = Allocator.Allocate<char>(Length + 1);
178 
179     memcpy(TextPtr, WordText.c_str(), Length + 1);
180     StringRef Text = StringRef(TextPtr, Length);
181 
182     formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
183     return true;
184   }
185 
186   bool lexDelimitedSeq(Token &Tok, char OpenDelim, char CloseDelim) {
187     if (isEnd())
188       return false;
189 
190     Position SavedPos = Pos;
191 
192     consumeWhitespace();
193     SmallString<32> WordText;
194     const char *WordBegin = Pos.BufferPtr;
195     SourceLocation Loc = getSourceLocation();
196     bool Error = false;
197     if (!isEnd()) {
198       const char C = peek();
199       if (C == OpenDelim) {
200         WordText.push_back(C);
201         consumeChar();
202       } else
203         Error = true;
204     }
205     char C = '\0';
206     while (!Error && !isEnd()) {
207       C = peek();
208       WordText.push_back(C);
209       consumeChar();
210       if (C == CloseDelim)
211         break;
212     }
213     if (!Error && C != CloseDelim)
214       Error = true;
215 
216     if (Error) {
217       Pos = SavedPos;
218       return false;
219     }
220 
221     const unsigned Length = WordText.size();
222     char *TextPtr = Allocator.Allocate<char>(Length + 1);
223 
224     memcpy(TextPtr, WordText.c_str(), Length + 1);
225     StringRef Text = StringRef(TextPtr, Length);
226 
227     formTokenWithChars(Tok, Loc, WordBegin,
228                        Pos.BufferPtr - WordBegin, Text);
229     return true;
230   }
231 
232   /// Put back tokens that we didn't consume.
233   void putBackLeftoverTokens() {
234     if (isEnd())
235       return;
236 
237     bool HavePartialTok = false;
238     Token PartialTok;
239     if (Pos.BufferPtr != Pos.BufferStart) {
240       formTokenWithChars(PartialTok, getSourceLocation(),
241                          Pos.BufferPtr, Pos.BufferEnd - Pos.BufferPtr,
242                          StringRef(Pos.BufferPtr,
243                                    Pos.BufferEnd - Pos.BufferPtr));
244       HavePartialTok = true;
245       Pos.CurToken++;
246     }
247 
248     P.putBack(llvm::makeArrayRef(Toks.begin() + Pos.CurToken, Toks.end()));
249     Pos.CurToken = Toks.size();
250 
251     if (HavePartialTok)
252       P.putBack(PartialTok);
253   }
254 };
255 
256 Parser::Parser(Lexer &L, Sema &S, llvm::BumpPtrAllocator &Allocator,
257                const SourceManager &SourceMgr, DiagnosticsEngine &Diags,
258                const CommandTraits &Traits):
259     L(L), S(S), Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags),
260     Traits(Traits) {
261   consumeToken();
262 }
263 
264 void Parser::parseParamCommandArgs(ParamCommandComment *PC,
265                                    TextTokenRetokenizer &Retokenizer) {
266   Token Arg;
267   // Check if argument looks like direction specification: [dir]
268   // e.g., [in], [out], [in,out]
269   if (Retokenizer.lexDelimitedSeq(Arg, '[', ']'))
270     S.actOnParamCommandDirectionArg(PC,
271                                     Arg.getLocation(),
272                                     Arg.getEndLocation(),
273                                     Arg.getText());
274 
275   if (Retokenizer.lexWord(Arg))
276     S.actOnParamCommandParamNameArg(PC,
277                                     Arg.getLocation(),
278                                     Arg.getEndLocation(),
279                                     Arg.getText());
280 }
281 
282 void Parser::parseTParamCommandArgs(TParamCommandComment *TPC,
283                                     TextTokenRetokenizer &Retokenizer) {
284   Token Arg;
285   if (Retokenizer.lexWord(Arg))
286     S.actOnTParamCommandParamNameArg(TPC,
287                                      Arg.getLocation(),
288                                      Arg.getEndLocation(),
289                                      Arg.getText());
290 }
291 
292 ArrayRef<Comment::Argument>
293 Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs) {
294   auto *Args = new (Allocator.Allocate<Comment::Argument>(NumArgs))
295       Comment::Argument[NumArgs];
296   unsigned ParsedArgs = 0;
297   Token Arg;
298   while (ParsedArgs < NumArgs && Retokenizer.lexWord(Arg)) {
299     Args[ParsedArgs] = Comment::Argument{
300         SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
301     ParsedArgs++;
302   }
303 
304   return llvm::makeArrayRef(Args, ParsedArgs);
305 }
306 
307 BlockCommandComment *Parser::parseBlockCommand() {
308   assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
309 
310   ParamCommandComment *PC = nullptr;
311   TParamCommandComment *TPC = nullptr;
312   BlockCommandComment *BC = nullptr;
313   const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
314   CommandMarkerKind CommandMarker =
315       Tok.is(tok::backslash_command) ? CMK_Backslash : CMK_At;
316   if (Info->IsParamCommand) {
317     PC = S.actOnParamCommandStart(Tok.getLocation(),
318                                   Tok.getEndLocation(),
319                                   Tok.getCommandID(),
320                                   CommandMarker);
321   } else if (Info->IsTParamCommand) {
322     TPC = S.actOnTParamCommandStart(Tok.getLocation(),
323                                     Tok.getEndLocation(),
324                                     Tok.getCommandID(),
325                                     CommandMarker);
326   } else {
327     BC = S.actOnBlockCommandStart(Tok.getLocation(),
328                                   Tok.getEndLocation(),
329                                   Tok.getCommandID(),
330                                   CommandMarker);
331   }
332   consumeToken();
333 
334   if (isTokBlockCommand()) {
335     // Block command ahead.  We can't nest block commands, so pretend that this
336     // command has an empty argument.
337     ParagraphComment *Paragraph = S.actOnParagraphComment(None);
338     if (PC) {
339       S.actOnParamCommandFinish(PC, Paragraph);
340       return PC;
341     } else if (TPC) {
342       S.actOnTParamCommandFinish(TPC, Paragraph);
343       return TPC;
344     } else {
345       S.actOnBlockCommandFinish(BC, Paragraph);
346       return BC;
347     }
348   }
349 
350   if (PC || TPC || Info->NumArgs > 0) {
351     // In order to parse command arguments we need to retokenize a few
352     // following text tokens.
353     TextTokenRetokenizer Retokenizer(Allocator, *this);
354 
355     if (PC)
356       parseParamCommandArgs(PC, Retokenizer);
357     else if (TPC)
358       parseTParamCommandArgs(TPC, Retokenizer);
359     else
360       S.actOnBlockCommandArgs(BC, parseCommandArgs(Retokenizer, Info->NumArgs));
361 
362     Retokenizer.putBackLeftoverTokens();
363   }
364 
365   // If there's a block command ahead, we will attach an empty paragraph to
366   // this command.
367   bool EmptyParagraph = false;
368   if (isTokBlockCommand())
369     EmptyParagraph = true;
370   else if (Tok.is(tok::newline)) {
371     Token PrevTok = Tok;
372     consumeToken();
373     EmptyParagraph = isTokBlockCommand();
374     putBack(PrevTok);
375   }
376 
377   ParagraphComment *Paragraph;
378   if (EmptyParagraph)
379     Paragraph = S.actOnParagraphComment(None);
380   else {
381     BlockContentComment *Block = parseParagraphOrBlockCommand();
382     // Since we have checked for a block command, we should have parsed a
383     // paragraph.
384     Paragraph = cast<ParagraphComment>(Block);
385   }
386 
387   if (PC) {
388     S.actOnParamCommandFinish(PC, Paragraph);
389     return PC;
390   } else if (TPC) {
391     S.actOnTParamCommandFinish(TPC, Paragraph);
392     return TPC;
393   } else {
394     S.actOnBlockCommandFinish(BC, Paragraph);
395     return BC;
396   }
397 }
398 
399 InlineCommandComment *Parser::parseInlineCommand() {
400   assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
401   const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
402 
403   const Token CommandTok = Tok;
404   consumeToken();
405 
406   TextTokenRetokenizer Retokenizer(Allocator, *this);
407   ArrayRef<Comment::Argument> Args =
408       parseCommandArgs(Retokenizer, Info->NumArgs);
409 
410   InlineCommandComment *IC = S.actOnInlineCommand(
411       CommandTok.getLocation(), CommandTok.getEndLocation(),
412       CommandTok.getCommandID(), Args);
413 
414   if (Args.size() < Info->NumArgs) {
415     Diag(CommandTok.getEndLocation().getLocWithOffset(1),
416          diag::warn_doc_inline_command_not_enough_arguments)
417         << CommandTok.is(tok::at_command) << Info->Name << Args.size()
418         << Info->NumArgs
419         << SourceRange(CommandTok.getLocation(), CommandTok.getEndLocation());
420   }
421 
422   Retokenizer.putBackLeftoverTokens();
423 
424   return IC;
425 }
426 
427 HTMLStartTagComment *Parser::parseHTMLStartTag() {
428   assert(Tok.is(tok::html_start_tag));
429   HTMLStartTagComment *HST =
430       S.actOnHTMLStartTagStart(Tok.getLocation(),
431                                Tok.getHTMLTagStartName());
432   consumeToken();
433 
434   SmallVector<HTMLStartTagComment::Attribute, 2> Attrs;
435   while (true) {
436     switch (Tok.getKind()) {
437     case tok::html_ident: {
438       Token Ident = Tok;
439       consumeToken();
440       if (Tok.isNot(tok::html_equals)) {
441         Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
442                                                        Ident.getHTMLIdent()));
443         continue;
444       }
445       Token Equals = Tok;
446       consumeToken();
447       if (Tok.isNot(tok::html_quoted_string)) {
448         Diag(Tok.getLocation(),
449              diag::warn_doc_html_start_tag_expected_quoted_string)
450           << SourceRange(Equals.getLocation());
451         Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
452                                                        Ident.getHTMLIdent()));
453         while (Tok.is(tok::html_equals) ||
454                Tok.is(tok::html_quoted_string))
455           consumeToken();
456         continue;
457       }
458       Attrs.push_back(HTMLStartTagComment::Attribute(
459                               Ident.getLocation(),
460                               Ident.getHTMLIdent(),
461                               Equals.getLocation(),
462                               SourceRange(Tok.getLocation(),
463                                           Tok.getEndLocation()),
464                               Tok.getHTMLQuotedString()));
465       consumeToken();
466       continue;
467     }
468 
469     case tok::html_greater:
470       S.actOnHTMLStartTagFinish(HST,
471                                 S.copyArray(llvm::makeArrayRef(Attrs)),
472                                 Tok.getLocation(),
473                                 /* IsSelfClosing = */ false);
474       consumeToken();
475       return HST;
476 
477     case tok::html_slash_greater:
478       S.actOnHTMLStartTagFinish(HST,
479                                 S.copyArray(llvm::makeArrayRef(Attrs)),
480                                 Tok.getLocation(),
481                                 /* IsSelfClosing = */ true);
482       consumeToken();
483       return HST;
484 
485     case tok::html_equals:
486     case tok::html_quoted_string:
487       Diag(Tok.getLocation(),
488            diag::warn_doc_html_start_tag_expected_ident_or_greater);
489       while (Tok.is(tok::html_equals) ||
490              Tok.is(tok::html_quoted_string))
491         consumeToken();
492       if (Tok.is(tok::html_ident) ||
493           Tok.is(tok::html_greater) ||
494           Tok.is(tok::html_slash_greater))
495         continue;
496 
497       S.actOnHTMLStartTagFinish(HST,
498                                 S.copyArray(llvm::makeArrayRef(Attrs)),
499                                 SourceLocation(),
500                                 /* IsSelfClosing = */ false);
501       return HST;
502 
503     default:
504       // Not a token from an HTML start tag.  Thus HTML tag prematurely ended.
505       S.actOnHTMLStartTagFinish(HST,
506                                 S.copyArray(llvm::makeArrayRef(Attrs)),
507                                 SourceLocation(),
508                                 /* IsSelfClosing = */ false);
509       bool StartLineInvalid;
510       const unsigned StartLine = SourceMgr.getPresumedLineNumber(
511                                                   HST->getLocation(),
512                                                   &StartLineInvalid);
513       bool EndLineInvalid;
514       const unsigned EndLine = SourceMgr.getPresumedLineNumber(
515                                                   Tok.getLocation(),
516                                                   &EndLineInvalid);
517       if (StartLineInvalid || EndLineInvalid || StartLine == EndLine)
518         Diag(Tok.getLocation(),
519              diag::warn_doc_html_start_tag_expected_ident_or_greater)
520           << HST->getSourceRange();
521       else {
522         Diag(Tok.getLocation(),
523              diag::warn_doc_html_start_tag_expected_ident_or_greater);
524         Diag(HST->getLocation(), diag::note_doc_html_tag_started_here)
525           << HST->getSourceRange();
526       }
527       return HST;
528     }
529   }
530 }
531 
532 HTMLEndTagComment *Parser::parseHTMLEndTag() {
533   assert(Tok.is(tok::html_end_tag));
534   Token TokEndTag = Tok;
535   consumeToken();
536   SourceLocation Loc;
537   if (Tok.is(tok::html_greater)) {
538     Loc = Tok.getLocation();
539     consumeToken();
540   }
541 
542   return S.actOnHTMLEndTag(TokEndTag.getLocation(),
543                            Loc,
544                            TokEndTag.getHTMLTagEndName());
545 }
546 
547 BlockContentComment *Parser::parseParagraphOrBlockCommand() {
548   SmallVector<InlineContentComment *, 8> Content;
549 
550   while (true) {
551     switch (Tok.getKind()) {
552     case tok::verbatim_block_begin:
553     case tok::verbatim_line_name:
554     case tok::eof:
555       break; // Block content or EOF ahead, finish this parapgaph.
556 
557     case tok::unknown_command:
558       Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
559                                               Tok.getEndLocation(),
560                                               Tok.getUnknownCommandName()));
561       consumeToken();
562       continue;
563 
564     case tok::backslash_command:
565     case tok::at_command: {
566       const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
567       if (Info->IsBlockCommand) {
568         if (Content.size() == 0)
569           return parseBlockCommand();
570         break; // Block command ahead, finish this parapgaph.
571       }
572       if (Info->IsVerbatimBlockEndCommand) {
573         Diag(Tok.getLocation(),
574              diag::warn_verbatim_block_end_without_start)
575           << Tok.is(tok::at_command)
576           << Info->Name
577           << SourceRange(Tok.getLocation(), Tok.getEndLocation());
578         consumeToken();
579         continue;
580       }
581       if (Info->IsUnknownCommand) {
582         Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
583                                                 Tok.getEndLocation(),
584                                                 Info->getID()));
585         consumeToken();
586         continue;
587       }
588       assert(Info->IsInlineCommand);
589       Content.push_back(parseInlineCommand());
590       continue;
591     }
592 
593     case tok::newline: {
594       consumeToken();
595       if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
596         consumeToken();
597         break; // Two newlines -- end of paragraph.
598       }
599       // Also allow [tok::newline, tok::text, tok::newline] if the middle
600       // tok::text is just whitespace.
601       if (Tok.is(tok::text) && isWhitespace(Tok.getText())) {
602         Token WhitespaceTok = Tok;
603         consumeToken();
604         if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
605           consumeToken();
606           break;
607         }
608         // We have [tok::newline, tok::text, non-newline].  Put back tok::text.
609         putBack(WhitespaceTok);
610       }
611       if (Content.size() > 0)
612         Content.back()->addTrailingNewline();
613       continue;
614     }
615 
616     // Don't deal with HTML tag soup now.
617     case tok::html_start_tag:
618       Content.push_back(parseHTMLStartTag());
619       continue;
620 
621     case tok::html_end_tag:
622       Content.push_back(parseHTMLEndTag());
623       continue;
624 
625     case tok::text:
626       Content.push_back(S.actOnText(Tok.getLocation(),
627                                     Tok.getEndLocation(),
628                                     Tok.getText()));
629       consumeToken();
630       continue;
631 
632     case tok::verbatim_block_line:
633     case tok::verbatim_block_end:
634     case tok::verbatim_line_text:
635     case tok::html_ident:
636     case tok::html_equals:
637     case tok::html_quoted_string:
638     case tok::html_greater:
639     case tok::html_slash_greater:
640       llvm_unreachable("should not see this token");
641     }
642     break;
643   }
644 
645   return S.actOnParagraphComment(S.copyArray(llvm::makeArrayRef(Content)));
646 }
647 
648 VerbatimBlockComment *Parser::parseVerbatimBlock() {
649   assert(Tok.is(tok::verbatim_block_begin));
650 
651   VerbatimBlockComment *VB =
652       S.actOnVerbatimBlockStart(Tok.getLocation(),
653                                 Tok.getVerbatimBlockID());
654   consumeToken();
655 
656   // Don't create an empty line if verbatim opening command is followed
657   // by a newline.
658   if (Tok.is(tok::newline))
659     consumeToken();
660 
661   SmallVector<VerbatimBlockLineComment *, 8> Lines;
662   while (Tok.is(tok::verbatim_block_line) ||
663          Tok.is(tok::newline)) {
664     VerbatimBlockLineComment *Line;
665     if (Tok.is(tok::verbatim_block_line)) {
666       Line = S.actOnVerbatimBlockLine(Tok.getLocation(),
667                                       Tok.getVerbatimBlockText());
668       consumeToken();
669       if (Tok.is(tok::newline)) {
670         consumeToken();
671       }
672     } else {
673       // Empty line, just a tok::newline.
674       Line = S.actOnVerbatimBlockLine(Tok.getLocation(), "");
675       consumeToken();
676     }
677     Lines.push_back(Line);
678   }
679 
680   if (Tok.is(tok::verbatim_block_end)) {
681     const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
682     S.actOnVerbatimBlockFinish(VB, Tok.getLocation(),
683                                Info->Name,
684                                S.copyArray(llvm::makeArrayRef(Lines)));
685     consumeToken();
686   } else {
687     // Unterminated \\verbatim block
688     S.actOnVerbatimBlockFinish(VB, SourceLocation(), "",
689                                S.copyArray(llvm::makeArrayRef(Lines)));
690   }
691 
692   return VB;
693 }
694 
695 VerbatimLineComment *Parser::parseVerbatimLine() {
696   assert(Tok.is(tok::verbatim_line_name));
697 
698   Token NameTok = Tok;
699   consumeToken();
700 
701   SourceLocation TextBegin;
702   StringRef Text;
703   // Next token might not be a tok::verbatim_line_text if verbatim line
704   // starting command comes just before a newline or comment end.
705   if (Tok.is(tok::verbatim_line_text)) {
706     TextBegin = Tok.getLocation();
707     Text = Tok.getVerbatimLineText();
708   } else {
709     TextBegin = NameTok.getEndLocation();
710     Text = "";
711   }
712 
713   VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(),
714                                                 NameTok.getVerbatimLineID(),
715                                                 TextBegin,
716                                                 Text);
717   consumeToken();
718   return VL;
719 }
720 
721 BlockContentComment *Parser::parseBlockContent() {
722   switch (Tok.getKind()) {
723   case tok::text:
724   case tok::unknown_command:
725   case tok::backslash_command:
726   case tok::at_command:
727   case tok::html_start_tag:
728   case tok::html_end_tag:
729     return parseParagraphOrBlockCommand();
730 
731   case tok::verbatim_block_begin:
732     return parseVerbatimBlock();
733 
734   case tok::verbatim_line_name:
735     return parseVerbatimLine();
736 
737   case tok::eof:
738   case tok::newline:
739   case tok::verbatim_block_line:
740   case tok::verbatim_block_end:
741   case tok::verbatim_line_text:
742   case tok::html_ident:
743   case tok::html_equals:
744   case tok::html_quoted_string:
745   case tok::html_greater:
746   case tok::html_slash_greater:
747     llvm_unreachable("should not see this token");
748   }
749   llvm_unreachable("bogus token kind");
750 }
751 
752 FullComment *Parser::parseFullComment() {
753   // Skip newlines at the beginning of the comment.
754   while (Tok.is(tok::newline))
755     consumeToken();
756 
757   SmallVector<BlockContentComment *, 8> Blocks;
758   while (Tok.isNot(tok::eof)) {
759     Blocks.push_back(parseBlockContent());
760 
761     // Skip extra newlines after paragraph end.
762     while (Tok.is(tok::newline))
763       consumeToken();
764   }
765   return S.actOnFullComment(S.copyArray(llvm::makeArrayRef(Blocks)));
766 }
767 
768 } // end namespace comments
769 } // end namespace clang
770