1 
2 /* Compiler implementation of the D programming language
3  * Copyright (C) 1999-2019 by The D Language Foundation, All Rights Reserved
4  * written by Walter Bright
5  * http://www.digitalmars.com
6  * Distributed under the Boost Software License, Version 1.0.
7  * http://www.boost.org/LICENSE_1_0.txt
8  * https://github.com/D-Programming-Language/dmd/blob/master/src/lexer.h
9  */
10 
11 #pragma once
12 
13 #include "root/root.h"
14 #include "globals.h"
15 #include "tokens.h"
16 
17 struct StringTable;
18 class Identifier;
19 
20 class Lexer
21 {
22 public:
23     static OutBuffer stringbuffer;
24 
25     Loc scanloc;                // for error messages
26 
27     const utf8_t *base;        // pointer to start of buffer
28     const utf8_t *end;         // past end of buffer
29     const utf8_t *p;           // current character
30     const utf8_t *line;        // start of current line
31     Token token;
32     bool doDocComment;          // collect doc comment information
33     bool anyToken;              // !=0 means seen at least one token
34     bool commentToken;          // !=0 means comments are TOKcomment's
35     bool errors;                // errors occurred during lexing or parsing
36 
37     Lexer(const char *filename,
38         const utf8_t *base, size_t begoffset, size_t endoffset,
39         bool doDocComment, bool commentToken);
40 
41     TOK nextToken();
42     TOK peekNext();
43     TOK peekNext2();
44     void scan(Token *t);
45     Token *peek(Token *t);
46     Token *peekPastParen(Token *t);
47     unsigned escapeSequence();
48     TOK wysiwygStringConstant(Token *t, int tc);
49     TOK hexStringConstant(Token *t);
50     TOK delimitedStringConstant(Token *t);
51     TOK tokenStringConstant(Token *t);
52     TOK escapeStringConstant(Token *t);
53     TOK charConstant(Token *t);
54     void stringPostfix(Token *t);
55     TOK number(Token *t);
56     TOK inreal(Token *t);
57 
loc()58     Loc loc()
59     {
60         scanloc.charnum = (unsigned)(1 + p-line);
61         return scanloc;
62     }
63 
64     void error(const char *format, ...);
65     void error(Loc loc, const char *format, ...);
66     void deprecation(const char *format, ...);
67     void poundLine();
68     unsigned decodeUTF();
69     void getDocComment(Token *t, unsigned lineComment);
70 
71     static const utf8_t *combineComments(const utf8_t *c1, const utf8_t *c2);
72 
73 private:
74     void endOfLine();
75 };
76