1f4a2713aSLionel Sambuc //===--- CommentLexer.h - Lexer for structured comments ---------*- C++ -*-===// 2f4a2713aSLionel Sambuc // 3f4a2713aSLionel Sambuc // The LLVM Compiler Infrastructure 4f4a2713aSLionel Sambuc // 5f4a2713aSLionel Sambuc // This file is distributed under the University of Illinois Open Source 6f4a2713aSLionel Sambuc // License. See LICENSE.TXT for details. 7f4a2713aSLionel Sambuc // 8f4a2713aSLionel Sambuc //===----------------------------------------------------------------------===// 9f4a2713aSLionel Sambuc // 10f4a2713aSLionel Sambuc // This file defines lexer for structured comments and supporting token class. 11f4a2713aSLionel Sambuc // 12f4a2713aSLionel Sambuc //===----------------------------------------------------------------------===// 13f4a2713aSLionel Sambuc 14*0a6a1f1dSLionel Sambuc #ifndef LLVM_CLANG_AST_COMMENTLEXER_H 15*0a6a1f1dSLionel Sambuc #define LLVM_CLANG_AST_COMMENTLEXER_H 16f4a2713aSLionel Sambuc 17f4a2713aSLionel Sambuc #include "clang/Basic/Diagnostic.h" 18*0a6a1f1dSLionel Sambuc #include "clang/Basic/SourceManager.h" 19f4a2713aSLionel Sambuc #include "llvm/ADT/SmallString.h" 20f4a2713aSLionel Sambuc #include "llvm/ADT/SmallVector.h" 21f4a2713aSLionel Sambuc #include "llvm/ADT/StringRef.h" 22f4a2713aSLionel Sambuc #include "llvm/Support/Allocator.h" 23f4a2713aSLionel Sambuc #include "llvm/Support/raw_ostream.h" 24f4a2713aSLionel Sambuc 25f4a2713aSLionel Sambuc namespace clang { 26f4a2713aSLionel Sambuc namespace comments { 27f4a2713aSLionel Sambuc 28f4a2713aSLionel Sambuc class Lexer; 29f4a2713aSLionel Sambuc class TextTokenRetokenizer; 30f4a2713aSLionel Sambuc struct CommandInfo; 31f4a2713aSLionel Sambuc class CommandTraits; 32f4a2713aSLionel Sambuc 33f4a2713aSLionel Sambuc namespace tok { 34f4a2713aSLionel Sambuc enum TokenKind { 35f4a2713aSLionel Sambuc eof, 36f4a2713aSLionel Sambuc newline, 37f4a2713aSLionel Sambuc text, 38f4a2713aSLionel Sambuc unknown_command, // Command that does not have an ID. 39f4a2713aSLionel Sambuc backslash_command, // Command with an ID, that used backslash marker. 40f4a2713aSLionel Sambuc at_command, // Command with an ID, that used 'at' marker. 41f4a2713aSLionel Sambuc verbatim_block_begin, 42f4a2713aSLionel Sambuc verbatim_block_line, 43f4a2713aSLionel Sambuc verbatim_block_end, 44f4a2713aSLionel Sambuc verbatim_line_name, 45f4a2713aSLionel Sambuc verbatim_line_text, 46f4a2713aSLionel Sambuc html_start_tag, // <tag 47f4a2713aSLionel Sambuc html_ident, // attr 48f4a2713aSLionel Sambuc html_equals, // = 49f4a2713aSLionel Sambuc html_quoted_string, // "blah\"blah" or 'blah\'blah' 50f4a2713aSLionel Sambuc html_greater, // > 51f4a2713aSLionel Sambuc html_slash_greater, // /> 52f4a2713aSLionel Sambuc html_end_tag // </tag 53f4a2713aSLionel Sambuc }; 54f4a2713aSLionel Sambuc } // end namespace tok 55f4a2713aSLionel Sambuc 56f4a2713aSLionel Sambuc /// \brief Comment token. 57f4a2713aSLionel Sambuc class Token { 58f4a2713aSLionel Sambuc friend class Lexer; 59f4a2713aSLionel Sambuc friend class TextTokenRetokenizer; 60f4a2713aSLionel Sambuc 61f4a2713aSLionel Sambuc /// The location of the token. 62f4a2713aSLionel Sambuc SourceLocation Loc; 63f4a2713aSLionel Sambuc 64f4a2713aSLionel Sambuc /// The actual kind of the token. 65f4a2713aSLionel Sambuc tok::TokenKind Kind; 66f4a2713aSLionel Sambuc 67f4a2713aSLionel Sambuc /// Length of the token spelling in comment. Can be 0 for synthenized 68f4a2713aSLionel Sambuc /// tokens. 69f4a2713aSLionel Sambuc unsigned Length; 70f4a2713aSLionel Sambuc 71f4a2713aSLionel Sambuc /// Contains text value associated with a token. 72f4a2713aSLionel Sambuc const char *TextPtr; 73f4a2713aSLionel Sambuc 74f4a2713aSLionel Sambuc /// Integer value associated with a token. 75f4a2713aSLionel Sambuc /// 76f4a2713aSLionel Sambuc /// If the token is a konwn command, contains command ID and TextPtr is 77f4a2713aSLionel Sambuc /// unused (command spelling can be found with CommandTraits). Otherwise, 78f4a2713aSLionel Sambuc /// contains the length of the string that starts at TextPtr. 79f4a2713aSLionel Sambuc unsigned IntVal; 80f4a2713aSLionel Sambuc 81f4a2713aSLionel Sambuc public: getLocation()82f4a2713aSLionel Sambuc SourceLocation getLocation() const LLVM_READONLY { return Loc; } setLocation(SourceLocation SL)83f4a2713aSLionel Sambuc void setLocation(SourceLocation SL) { Loc = SL; } 84f4a2713aSLionel Sambuc getEndLocation()85f4a2713aSLionel Sambuc SourceLocation getEndLocation() const LLVM_READONLY { 86f4a2713aSLionel Sambuc if (Length == 0 || Length == 1) 87f4a2713aSLionel Sambuc return Loc; 88f4a2713aSLionel Sambuc return Loc.getLocWithOffset(Length - 1); 89f4a2713aSLionel Sambuc } 90f4a2713aSLionel Sambuc getKind()91f4a2713aSLionel Sambuc tok::TokenKind getKind() const LLVM_READONLY { return Kind; } setKind(tok::TokenKind K)92f4a2713aSLionel Sambuc void setKind(tok::TokenKind K) { Kind = K; } 93f4a2713aSLionel Sambuc is(tok::TokenKind K)94f4a2713aSLionel Sambuc bool is(tok::TokenKind K) const LLVM_READONLY { return Kind == K; } isNot(tok::TokenKind K)95f4a2713aSLionel Sambuc bool isNot(tok::TokenKind K) const LLVM_READONLY { return Kind != K; } 96f4a2713aSLionel Sambuc getLength()97f4a2713aSLionel Sambuc unsigned getLength() const LLVM_READONLY { return Length; } setLength(unsigned L)98f4a2713aSLionel Sambuc void setLength(unsigned L) { Length = L; } 99f4a2713aSLionel Sambuc getText()100f4a2713aSLionel Sambuc StringRef getText() const LLVM_READONLY { 101f4a2713aSLionel Sambuc assert(is(tok::text)); 102f4a2713aSLionel Sambuc return StringRef(TextPtr, IntVal); 103f4a2713aSLionel Sambuc } 104f4a2713aSLionel Sambuc setText(StringRef Text)105f4a2713aSLionel Sambuc void setText(StringRef Text) { 106f4a2713aSLionel Sambuc assert(is(tok::text)); 107f4a2713aSLionel Sambuc TextPtr = Text.data(); 108f4a2713aSLionel Sambuc IntVal = Text.size(); 109f4a2713aSLionel Sambuc } 110f4a2713aSLionel Sambuc getUnknownCommandName()111f4a2713aSLionel Sambuc StringRef getUnknownCommandName() const LLVM_READONLY { 112f4a2713aSLionel Sambuc assert(is(tok::unknown_command)); 113f4a2713aSLionel Sambuc return StringRef(TextPtr, IntVal); 114f4a2713aSLionel Sambuc } 115f4a2713aSLionel Sambuc setUnknownCommandName(StringRef Name)116f4a2713aSLionel Sambuc void setUnknownCommandName(StringRef Name) { 117f4a2713aSLionel Sambuc assert(is(tok::unknown_command)); 118f4a2713aSLionel Sambuc TextPtr = Name.data(); 119f4a2713aSLionel Sambuc IntVal = Name.size(); 120f4a2713aSLionel Sambuc } 121f4a2713aSLionel Sambuc getCommandID()122f4a2713aSLionel Sambuc unsigned getCommandID() const LLVM_READONLY { 123f4a2713aSLionel Sambuc assert(is(tok::backslash_command) || is(tok::at_command)); 124f4a2713aSLionel Sambuc return IntVal; 125f4a2713aSLionel Sambuc } 126f4a2713aSLionel Sambuc setCommandID(unsigned ID)127f4a2713aSLionel Sambuc void setCommandID(unsigned ID) { 128f4a2713aSLionel Sambuc assert(is(tok::backslash_command) || is(tok::at_command)); 129f4a2713aSLionel Sambuc IntVal = ID; 130f4a2713aSLionel Sambuc } 131f4a2713aSLionel Sambuc getVerbatimBlockID()132f4a2713aSLionel Sambuc unsigned getVerbatimBlockID() const LLVM_READONLY { 133f4a2713aSLionel Sambuc assert(is(tok::verbatim_block_begin) || is(tok::verbatim_block_end)); 134f4a2713aSLionel Sambuc return IntVal; 135f4a2713aSLionel Sambuc } 136f4a2713aSLionel Sambuc setVerbatimBlockID(unsigned ID)137f4a2713aSLionel Sambuc void setVerbatimBlockID(unsigned ID) { 138f4a2713aSLionel Sambuc assert(is(tok::verbatim_block_begin) || is(tok::verbatim_block_end)); 139f4a2713aSLionel Sambuc IntVal = ID; 140f4a2713aSLionel Sambuc } 141f4a2713aSLionel Sambuc getVerbatimBlockText()142f4a2713aSLionel Sambuc StringRef getVerbatimBlockText() const LLVM_READONLY { 143f4a2713aSLionel Sambuc assert(is(tok::verbatim_block_line)); 144f4a2713aSLionel Sambuc return StringRef(TextPtr, IntVal); 145f4a2713aSLionel Sambuc } 146f4a2713aSLionel Sambuc setVerbatimBlockText(StringRef Text)147f4a2713aSLionel Sambuc void setVerbatimBlockText(StringRef Text) { 148f4a2713aSLionel Sambuc assert(is(tok::verbatim_block_line)); 149f4a2713aSLionel Sambuc TextPtr = Text.data(); 150f4a2713aSLionel Sambuc IntVal = Text.size(); 151f4a2713aSLionel Sambuc } 152f4a2713aSLionel Sambuc getVerbatimLineID()153f4a2713aSLionel Sambuc unsigned getVerbatimLineID() const LLVM_READONLY { 154f4a2713aSLionel Sambuc assert(is(tok::verbatim_line_name)); 155f4a2713aSLionel Sambuc return IntVal; 156f4a2713aSLionel Sambuc } 157f4a2713aSLionel Sambuc setVerbatimLineID(unsigned ID)158f4a2713aSLionel Sambuc void setVerbatimLineID(unsigned ID) { 159f4a2713aSLionel Sambuc assert(is(tok::verbatim_line_name)); 160f4a2713aSLionel Sambuc IntVal = ID; 161f4a2713aSLionel Sambuc } 162f4a2713aSLionel Sambuc getVerbatimLineText()163f4a2713aSLionel Sambuc StringRef getVerbatimLineText() const LLVM_READONLY { 164f4a2713aSLionel Sambuc assert(is(tok::verbatim_line_text)); 165f4a2713aSLionel Sambuc return StringRef(TextPtr, IntVal); 166f4a2713aSLionel Sambuc } 167f4a2713aSLionel Sambuc setVerbatimLineText(StringRef Text)168f4a2713aSLionel Sambuc void setVerbatimLineText(StringRef Text) { 169f4a2713aSLionel Sambuc assert(is(tok::verbatim_line_text)); 170f4a2713aSLionel Sambuc TextPtr = Text.data(); 171f4a2713aSLionel Sambuc IntVal = Text.size(); 172f4a2713aSLionel Sambuc } 173f4a2713aSLionel Sambuc getHTMLTagStartName()174f4a2713aSLionel Sambuc StringRef getHTMLTagStartName() const LLVM_READONLY { 175f4a2713aSLionel Sambuc assert(is(tok::html_start_tag)); 176f4a2713aSLionel Sambuc return StringRef(TextPtr, IntVal); 177f4a2713aSLionel Sambuc } 178f4a2713aSLionel Sambuc setHTMLTagStartName(StringRef Name)179f4a2713aSLionel Sambuc void setHTMLTagStartName(StringRef Name) { 180f4a2713aSLionel Sambuc assert(is(tok::html_start_tag)); 181f4a2713aSLionel Sambuc TextPtr = Name.data(); 182f4a2713aSLionel Sambuc IntVal = Name.size(); 183f4a2713aSLionel Sambuc } 184f4a2713aSLionel Sambuc getHTMLIdent()185f4a2713aSLionel Sambuc StringRef getHTMLIdent() const LLVM_READONLY { 186f4a2713aSLionel Sambuc assert(is(tok::html_ident)); 187f4a2713aSLionel Sambuc return StringRef(TextPtr, IntVal); 188f4a2713aSLionel Sambuc } 189f4a2713aSLionel Sambuc setHTMLIdent(StringRef Name)190f4a2713aSLionel Sambuc void setHTMLIdent(StringRef Name) { 191f4a2713aSLionel Sambuc assert(is(tok::html_ident)); 192f4a2713aSLionel Sambuc TextPtr = Name.data(); 193f4a2713aSLionel Sambuc IntVal = Name.size(); 194f4a2713aSLionel Sambuc } 195f4a2713aSLionel Sambuc getHTMLQuotedString()196f4a2713aSLionel Sambuc StringRef getHTMLQuotedString() const LLVM_READONLY { 197f4a2713aSLionel Sambuc assert(is(tok::html_quoted_string)); 198f4a2713aSLionel Sambuc return StringRef(TextPtr, IntVal); 199f4a2713aSLionel Sambuc } 200f4a2713aSLionel Sambuc setHTMLQuotedString(StringRef Str)201f4a2713aSLionel Sambuc void setHTMLQuotedString(StringRef Str) { 202f4a2713aSLionel Sambuc assert(is(tok::html_quoted_string)); 203f4a2713aSLionel Sambuc TextPtr = Str.data(); 204f4a2713aSLionel Sambuc IntVal = Str.size(); 205f4a2713aSLionel Sambuc } 206f4a2713aSLionel Sambuc getHTMLTagEndName()207f4a2713aSLionel Sambuc StringRef getHTMLTagEndName() const LLVM_READONLY { 208f4a2713aSLionel Sambuc assert(is(tok::html_end_tag)); 209f4a2713aSLionel Sambuc return StringRef(TextPtr, IntVal); 210f4a2713aSLionel Sambuc } 211f4a2713aSLionel Sambuc setHTMLTagEndName(StringRef Name)212f4a2713aSLionel Sambuc void setHTMLTagEndName(StringRef Name) { 213f4a2713aSLionel Sambuc assert(is(tok::html_end_tag)); 214f4a2713aSLionel Sambuc TextPtr = Name.data(); 215f4a2713aSLionel Sambuc IntVal = Name.size(); 216f4a2713aSLionel Sambuc } 217f4a2713aSLionel Sambuc 218f4a2713aSLionel Sambuc void dump(const Lexer &L, const SourceManager &SM) const; 219f4a2713aSLionel Sambuc }; 220f4a2713aSLionel Sambuc 221f4a2713aSLionel Sambuc /// \brief Comment lexer. 222f4a2713aSLionel Sambuc class Lexer { 223f4a2713aSLionel Sambuc private: 224f4a2713aSLionel Sambuc Lexer(const Lexer &) LLVM_DELETED_FUNCTION; 225f4a2713aSLionel Sambuc void operator=(const Lexer &) LLVM_DELETED_FUNCTION; 226f4a2713aSLionel Sambuc 227f4a2713aSLionel Sambuc /// Allocator for strings that are semantic values of tokens and have to be 228f4a2713aSLionel Sambuc /// computed (for example, resolved decimal character references). 229f4a2713aSLionel Sambuc llvm::BumpPtrAllocator &Allocator; 230f4a2713aSLionel Sambuc 231f4a2713aSLionel Sambuc DiagnosticsEngine &Diags; 232f4a2713aSLionel Sambuc 233f4a2713aSLionel Sambuc const CommandTraits &Traits; 234f4a2713aSLionel Sambuc 235f4a2713aSLionel Sambuc const char *const BufferStart; 236f4a2713aSLionel Sambuc const char *const BufferEnd; 237f4a2713aSLionel Sambuc SourceLocation FileLoc; 238f4a2713aSLionel Sambuc 239f4a2713aSLionel Sambuc const char *BufferPtr; 240f4a2713aSLionel Sambuc 241f4a2713aSLionel Sambuc /// One past end pointer for the current comment. For BCPL comments points 242f4a2713aSLionel Sambuc /// to newline or BufferEnd, for C comments points to star in '*/'. 243f4a2713aSLionel Sambuc const char *CommentEnd; 244f4a2713aSLionel Sambuc 245f4a2713aSLionel Sambuc enum LexerCommentState { 246f4a2713aSLionel Sambuc LCS_BeforeComment, 247f4a2713aSLionel Sambuc LCS_InsideBCPLComment, 248f4a2713aSLionel Sambuc LCS_InsideCComment, 249f4a2713aSLionel Sambuc LCS_BetweenComments 250f4a2713aSLionel Sambuc }; 251f4a2713aSLionel Sambuc 252f4a2713aSLionel Sambuc /// Low-level lexer state, track if we are inside or outside of comment. 253f4a2713aSLionel Sambuc LexerCommentState CommentState; 254f4a2713aSLionel Sambuc 255f4a2713aSLionel Sambuc enum LexerState { 256f4a2713aSLionel Sambuc /// Lexing normal comment text 257f4a2713aSLionel Sambuc LS_Normal, 258f4a2713aSLionel Sambuc 259f4a2713aSLionel Sambuc /// Finished lexing verbatim block beginning command, will lex first body 260f4a2713aSLionel Sambuc /// line. 261f4a2713aSLionel Sambuc LS_VerbatimBlockFirstLine, 262f4a2713aSLionel Sambuc 263f4a2713aSLionel Sambuc /// Lexing verbatim block body line-by-line, skipping line-starting 264f4a2713aSLionel Sambuc /// decorations. 265f4a2713aSLionel Sambuc LS_VerbatimBlockBody, 266f4a2713aSLionel Sambuc 267f4a2713aSLionel Sambuc /// Finished lexing verbatim line beginning command, will lex text (one 268f4a2713aSLionel Sambuc /// line). 269f4a2713aSLionel Sambuc LS_VerbatimLineText, 270f4a2713aSLionel Sambuc 271f4a2713aSLionel Sambuc /// Finished lexing \verbatim <TAG \endverbatim part, lexing tag attributes. 272f4a2713aSLionel Sambuc LS_HTMLStartTag, 273f4a2713aSLionel Sambuc 274f4a2713aSLionel Sambuc /// Finished lexing \verbatim </TAG \endverbatim part, lexing '>'. 275f4a2713aSLionel Sambuc LS_HTMLEndTag 276f4a2713aSLionel Sambuc }; 277f4a2713aSLionel Sambuc 278f4a2713aSLionel Sambuc /// Current lexing mode. 279f4a2713aSLionel Sambuc LexerState State; 280f4a2713aSLionel Sambuc 281f4a2713aSLionel Sambuc /// If State is LS_VerbatimBlock, contains the name of verbatim end 282f4a2713aSLionel Sambuc /// command, including command marker. 283f4a2713aSLionel Sambuc SmallString<16> VerbatimBlockEndCommandName; 284f4a2713aSLionel Sambuc 285f4a2713aSLionel Sambuc /// Given a character reference name (e.g., "lt"), return the character that 286f4a2713aSLionel Sambuc /// it stands for (e.g., "<"). 287f4a2713aSLionel Sambuc StringRef resolveHTMLNamedCharacterReference(StringRef Name) const; 288f4a2713aSLionel Sambuc 289f4a2713aSLionel Sambuc /// Given a Unicode codepoint as base-10 integer, return the character. 290f4a2713aSLionel Sambuc StringRef resolveHTMLDecimalCharacterReference(StringRef Name) const; 291f4a2713aSLionel Sambuc 292f4a2713aSLionel Sambuc /// Given a Unicode codepoint as base-16 integer, return the character. 293f4a2713aSLionel Sambuc StringRef resolveHTMLHexCharacterReference(StringRef Name) const; 294f4a2713aSLionel Sambuc 295f4a2713aSLionel Sambuc void formTokenWithChars(Token &Result, const char *TokEnd, 296*0a6a1f1dSLionel Sambuc tok::TokenKind Kind); 297f4a2713aSLionel Sambuc formTextToken(Token & Result,const char * TokEnd)298f4a2713aSLionel Sambuc void formTextToken(Token &Result, const char *TokEnd) { 299f4a2713aSLionel Sambuc StringRef Text(BufferPtr, TokEnd - BufferPtr); 300f4a2713aSLionel Sambuc formTokenWithChars(Result, TokEnd, tok::text); 301f4a2713aSLionel Sambuc Result.setText(Text); 302f4a2713aSLionel Sambuc } 303f4a2713aSLionel Sambuc getSourceLocation(const char * Loc)304f4a2713aSLionel Sambuc SourceLocation getSourceLocation(const char *Loc) const { 305f4a2713aSLionel Sambuc assert(Loc >= BufferStart && Loc <= BufferEnd && 306f4a2713aSLionel Sambuc "Location out of range for this buffer!"); 307f4a2713aSLionel Sambuc 308f4a2713aSLionel Sambuc const unsigned CharNo = Loc - BufferStart; 309f4a2713aSLionel Sambuc return FileLoc.getLocWithOffset(CharNo); 310f4a2713aSLionel Sambuc } 311f4a2713aSLionel Sambuc Diag(SourceLocation Loc,unsigned DiagID)312f4a2713aSLionel Sambuc DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) { 313f4a2713aSLionel Sambuc return Diags.Report(Loc, DiagID); 314f4a2713aSLionel Sambuc } 315f4a2713aSLionel Sambuc 316f4a2713aSLionel Sambuc /// Eat string matching regexp \code \s*\* \endcode. 317f4a2713aSLionel Sambuc void skipLineStartingDecorations(); 318f4a2713aSLionel Sambuc 319f4a2713aSLionel Sambuc /// Lex stuff inside comments. CommentEnd should be set correctly. 320f4a2713aSLionel Sambuc void lexCommentText(Token &T); 321f4a2713aSLionel Sambuc 322f4a2713aSLionel Sambuc void setupAndLexVerbatimBlock(Token &T, 323f4a2713aSLionel Sambuc const char *TextBegin, 324f4a2713aSLionel Sambuc char Marker, const CommandInfo *Info); 325f4a2713aSLionel Sambuc 326f4a2713aSLionel Sambuc void lexVerbatimBlockFirstLine(Token &T); 327f4a2713aSLionel Sambuc 328f4a2713aSLionel Sambuc void lexVerbatimBlockBody(Token &T); 329f4a2713aSLionel Sambuc 330f4a2713aSLionel Sambuc void setupAndLexVerbatimLine(Token &T, const char *TextBegin, 331f4a2713aSLionel Sambuc const CommandInfo *Info); 332f4a2713aSLionel Sambuc 333f4a2713aSLionel Sambuc void lexVerbatimLineText(Token &T); 334f4a2713aSLionel Sambuc 335f4a2713aSLionel Sambuc void lexHTMLCharacterReference(Token &T); 336f4a2713aSLionel Sambuc 337f4a2713aSLionel Sambuc void setupAndLexHTMLStartTag(Token &T); 338f4a2713aSLionel Sambuc 339f4a2713aSLionel Sambuc void lexHTMLStartTag(Token &T); 340f4a2713aSLionel Sambuc 341f4a2713aSLionel Sambuc void setupAndLexHTMLEndTag(Token &T); 342f4a2713aSLionel Sambuc 343f4a2713aSLionel Sambuc void lexHTMLEndTag(Token &T); 344f4a2713aSLionel Sambuc 345f4a2713aSLionel Sambuc public: 346f4a2713aSLionel Sambuc Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags, 347f4a2713aSLionel Sambuc const CommandTraits &Traits, 348f4a2713aSLionel Sambuc SourceLocation FileLoc, 349f4a2713aSLionel Sambuc const char *BufferStart, const char *BufferEnd); 350f4a2713aSLionel Sambuc 351f4a2713aSLionel Sambuc void lex(Token &T); 352f4a2713aSLionel Sambuc 353f4a2713aSLionel Sambuc StringRef getSpelling(const Token &Tok, 354f4a2713aSLionel Sambuc const SourceManager &SourceMgr, 355*0a6a1f1dSLionel Sambuc bool *Invalid = nullptr) const; 356f4a2713aSLionel Sambuc }; 357f4a2713aSLionel Sambuc 358f4a2713aSLionel Sambuc } // end namespace comments 359f4a2713aSLionel Sambuc } // end namespace clang 360f4a2713aSLionel Sambuc 361f4a2713aSLionel Sambuc #endif 362f4a2713aSLionel Sambuc 363