1 // tokenizer.h
2 //
3 // Copyright (C) 2001, Chris Laurel <claurel@shatters.net>
4 //
5 // This program is free software; you can redistribute it and/or
6 // modify it under the terms of the GNU General Public License
7 // as published by the Free Software Foundation; either version 2
8 // of the License, or (at your option) any later version.
9 
10 #ifndef _TOKENIZER_H_
11 #define _TOKENIZER_H_
12 
13 #include <string>
14 #include <iostream>
15 
16 using namespace std;
17 
18 
19 class Tokenizer
20 {
21 public:
22     enum TokenType
23     {
24         TokenName           = 0,
25         TokenString         = 1,
26         TokenNumber         = 2,
27         TokenBegin          = 3,
28         TokenEnd            = 4,
29         TokenNull           = 5,
30         TokenBeginGroup     = 6,
31         TokenEndGroup       = 7,
32         TokenBeginArray     = 8,
33         TokenEndArray       = 9,
34         TokenEquals         = 10,
35         TokenError          = 11,
36         TokenBar            = 12,
37     };
38 
39     Tokenizer(istream*);
40 
41     TokenType nextToken();
42     TokenType getTokenType();
43     void pushBack();
44     double getNumberValue();
45     string getNameValue();
46     string getStringValue();
47 
48     int getLineNumber() const;
49 
50 private:
51     enum State
52     {
53         StartState          = 0,
54         NameState           = 1,
55         NumberState         = 2,
56         FractionState       = 3,
57         ExponentState       = 4,
58         ExponentFirstState  = 5,
59         DotState            = 6,
60         CommentState        = 7,
61         StringState         = 8,
62         ErrorState          = 9,
63         StringEscapeState   = 10,
64         UnicodeEscapeState  = 11,
65     };
66 
67     istream* in;
68 
69     int nextChar;
70     TokenType tokenType;
71     bool haveValidNumber;
72     bool haveValidName;
73     bool haveValidString;
74 
75     unsigned int unicodeValue;
76     unsigned int unicodeEscapeDigits;
77 
78     bool pushedBack;
79 
80     int readChar();
81     void syntaxError(const char*);
82 
83     double numberValue;
84 
85     string textToken;
86 
87     int lineNum;
88 };
89 
90 #endif // _TOKENIZER_H_
91