1 2 /* Compiler implementation of the D programming language 3 * Copyright (C) 1999-2019 by The D Language Foundation, All Rights Reserved 4 * written by Walter Bright 5 * http://www.digitalmars.com 6 * Distributed under the Boost Software License, Version 1.0. 7 * http://www.boost.org/LICENSE_1_0.txt 8 * https://github.com/D-Programming-Language/dmd/blob/master/src/lexer.h 9 */ 10 11 #pragma once 12 13 #include "root/root.h" 14 #include "globals.h" 15 #include "tokens.h" 16 17 struct StringTable; 18 class Identifier; 19 20 class Lexer 21 { 22 public: 23 static OutBuffer stringbuffer; 24 25 Loc scanloc; // for error messages 26 27 const utf8_t *base; // pointer to start of buffer 28 const utf8_t *end; // past end of buffer 29 const utf8_t *p; // current character 30 const utf8_t *line; // start of current line 31 Token token; 32 bool doDocComment; // collect doc comment information 33 bool anyToken; // !=0 means seen at least one token 34 bool commentToken; // !=0 means comments are TOKcomment's 35 bool errors; // errors occurred during lexing or parsing 36 37 Lexer(const char *filename, 38 const utf8_t *base, size_t begoffset, size_t endoffset, 39 bool doDocComment, bool commentToken); 40 41 TOK nextToken(); 42 TOK peekNext(); 43 TOK peekNext2(); 44 void scan(Token *t); 45 Token *peek(Token *t); 46 Token *peekPastParen(Token *t); 47 unsigned escapeSequence(); 48 TOK wysiwygStringConstant(Token *t, int tc); 49 TOK hexStringConstant(Token *t); 50 TOK delimitedStringConstant(Token *t); 51 TOK tokenStringConstant(Token *t); 52 TOK escapeStringConstant(Token *t); 53 TOK charConstant(Token *t); 54 void stringPostfix(Token *t); 55 TOK number(Token *t); 56 TOK inreal(Token *t); 57 loc()58 Loc loc() 59 { 60 scanloc.charnum = (unsigned)(1 + p-line); 61 return scanloc; 62 } 63 64 void error(const char *format, ...); 65 void error(Loc loc, const char *format, ...); 66 void deprecation(const char *format, ...); 67 void poundLine(); 68 unsigned decodeUTF(); 69 void getDocComment(Token *t, unsigned lineComment); 70 71 static const utf8_t *combineComments(const utf8_t *c1, const utf8_t *c2); 72 73 private: 74 void endOfLine(); 75 }; 76