1 // tokenizer.h 2 // 3 // Copyright (C) 2001, Chris Laurel <claurel@shatters.net> 4 // 5 // This program is free software; you can redistribute it and/or 6 // modify it under the terms of the GNU General Public License 7 // as published by the Free Software Foundation; either version 2 8 // of the License, or (at your option) any later version. 9 10 #ifndef _TOKENIZER_H_ 11 #define _TOKENIZER_H_ 12 13 #include <string> 14 #include <iostream> 15 16 using namespace std; 17 18 19 class Tokenizer 20 { 21 public: 22 enum TokenType 23 { 24 TokenName = 0, 25 TokenString = 1, 26 TokenNumber = 2, 27 TokenBegin = 3, 28 TokenEnd = 4, 29 TokenNull = 5, 30 TokenBeginGroup = 6, 31 TokenEndGroup = 7, 32 TokenBeginArray = 8, 33 TokenEndArray = 9, 34 TokenEquals = 10, 35 TokenError = 11, 36 TokenBar = 12, 37 }; 38 39 Tokenizer(istream*); 40 41 TokenType nextToken(); 42 TokenType getTokenType(); 43 void pushBack(); 44 double getNumberValue(); 45 string getNameValue(); 46 string getStringValue(); 47 48 int getLineNumber() const; 49 50 private: 51 enum State 52 { 53 StartState = 0, 54 NameState = 1, 55 NumberState = 2, 56 FractionState = 3, 57 ExponentState = 4, 58 ExponentFirstState = 5, 59 DotState = 6, 60 CommentState = 7, 61 StringState = 8, 62 ErrorState = 9, 63 StringEscapeState = 10, 64 UnicodeEscapeState = 11, 65 }; 66 67 istream* in; 68 69 int nextChar; 70 TokenType tokenType; 71 bool haveValidNumber; 72 bool haveValidName; 73 bool haveValidString; 74 75 unsigned int unicodeValue; 76 unsigned int unicodeEscapeDigits; 77 78 bool pushedBack; 79 80 int readChar(); 81 void syntaxError(const char*); 82 83 double numberValue; 84 85 string textToken; 86 87 int lineNum; 88 }; 89 90 #endif // _TOKENIZER_H_ 91