1 #pragma once 2 #include "Util/FileClasses.h" 3 4 enum class TokenType 5 { 6 Invalid, 7 Identifier, 8 Integer, 9 String, 10 Float, 11 LParen, 12 RParen, 13 Plus, 14 Minus, 15 Mult, 16 Div, 17 Mod, 18 Caret, 19 Tilde, 20 LeftShift, 21 RightShift, 22 Less, 23 Greater, 24 LessEqual, 25 GreaterEqual, 26 Equal, 27 NotEqual, 28 BitAnd, 29 BitOr, 30 LogAnd, 31 LogOr, 32 Exclamation, 33 Question, 34 Colon, 35 LBrack, 36 RBrack, 37 Comma, 38 Assign, 39 Equ, 40 EquValue, 41 Hash, 42 LBrace, 43 RBrace, 44 Dollar, 45 NumberString, 46 Degree, 47 Separator 48 }; 49 50 struct Token 51 { 52 friend class Tokenizer; 53 TokenToken54 Token() : originalText(nullptr), stringValue(nullptr), checked(false) 55 { 56 } 57 TokenToken58 Token(Token &&src) 59 { 60 // Move strings. 61 originalText = src.originalText; 62 src.originalText = nullptr; 63 stringValue = src.stringValue; 64 src.stringValue = nullptr; 65 66 // Just copy the rest. 67 type = src.type; 68 line = src.line; 69 column = src.column; 70 floatValue = src.floatValue; 71 checked = src.checked; 72 } 73 TokenToken74 Token(const Token &src) { 75 // Copy strings. 76 originalText = nullptr; 77 if (src.originalText) 78 setOriginalText(src.originalText); 79 stringValue = nullptr; 80 if (src.stringValue) 81 setStringValue(src.stringValue); 82 83 // And copy the rest. 84 type = src.type; 85 line = src.line; 86 column = src.column; 87 floatValue = src.floatValue; 88 checked = src.checked; 89 } 90 ~TokenToken91 ~Token() 92 { 93 clearOriginalText(); 94 clearStringValue(); 95 } 96 97 Token& operator=(const Token& src) 98 { 99 // Copy strings. 100 originalText = nullptr; 101 if (src.originalText) 102 setOriginalText(src.originalText); 103 stringValue = nullptr; 104 if (src.stringValue) 105 setStringValue(src.stringValue); 106 107 // And copy the rest. 108 type = src.type; 109 line = src.line; 110 column = src.column; 111 floatValue = src.floatValue; 112 checked = src.checked; 113 114 return *this; 115 } 116 setOriginalTextToken117 void setOriginalText(const std::wstring& t) 118 { 119 setOriginalText(t, 0, t.length()); 120 } 121 setOriginalTextToken122 void setOriginalText(const std::wstring& t, const size_t pos, const size_t len) 123 { 124 clearOriginalText(); 125 originalText = new wchar_t[len + 1]; 126 wmemcpy(originalText, t.data() + pos, len); 127 originalText[len] = 0; 128 } 129 getOriginalTextToken130 std::wstring getOriginalText() const 131 { 132 return originalText; 133 } 134 setStringValueToken135 void setStringValue(const std::wstring& t) 136 { 137 setStringValue(t, 0, t.length()); 138 } 139 setStringValueToken140 void setStringValue(const std::wstring& t, const size_t pos, const size_t len) 141 { 142 clearStringValue(); 143 stringValue = new wchar_t[len + 1]; 144 wmemcpy(stringValue, t.data() + pos, len); 145 stringValue[len] = 0; 146 } 147 setStringAndOriginalValueToken148 void setStringAndOriginalValue(const std::wstring& t) 149 { 150 setStringAndOriginalValue(t, 0, t.length()); 151 } 152 setStringAndOriginalValueToken153 void setStringAndOriginalValue(const std::wstring& t, const size_t pos, const size_t len) 154 { 155 setStringValue(t, pos, len); 156 clearOriginalText(); 157 originalText = stringValue; 158 } 159 getStringValueToken160 std::wstring getStringValue() const 161 { 162 if (stringValue) 163 return stringValue; 164 return L""; 165 } 166 stringValueStartsWithToken167 bool stringValueStartsWith(wchar_t c) const 168 { 169 if (stringValue) 170 return stringValue[0] == c; 171 return false; 172 } 173 174 TokenType type; 175 size_t line; 176 size_t column; 177 178 union 179 { 180 int64_t intValue; 181 double floatValue; 182 }; 183 184 protected: clearOriginalTextToken185 void clearOriginalText() 186 { 187 if (originalText != stringValue) 188 delete [] originalText; 189 originalText = nullptr; 190 } 191 clearStringValueToken192 void clearStringValue() 193 { 194 if (stringValue != originalText) 195 delete [] stringValue; 196 stringValue = nullptr; 197 } 198 199 wchar_t* originalText; 200 wchar_t* stringValue; 201 202 bool checked; 203 }; 204 205 typedef std::list<Token> TokenList; 206 207 struct TokenizerPosition 208 { 209 friend class Tokenizer; 210 previousTokenizerPosition211 TokenizerPosition previous() 212 { 213 TokenizerPosition pos = *this; 214 pos.it--; 215 return pos; 216 } 217 private: 218 TokenList::iterator it; 219 }; 220 221 class Tokenizer 222 { 223 public: 224 Tokenizer(); 225 const Token& nextToken(); 226 const Token& peekToken(int ahead = 0); eatToken()227 void eatToken() { eatTokens(1); } 228 void eatTokens(int num); atEnd()229 bool atEnd() { return position.it == tokens.end(); } getPosition()230 TokenizerPosition getPosition() { return position; } setPosition(TokenizerPosition pos)231 void setPosition(TokenizerPosition pos) { position = pos; } 232 void skipLookahead(); 233 std::vector<Token> getTokens(TokenizerPosition start, TokenizerPosition end) const; 234 void registerReplacement(const std::wstring& identifier, std::vector<Token>& tokens); 235 void registerReplacement(const std::wstring& identifier, const std::wstring& newValue); 236 static size_t addEquValue(const std::vector<Token>& tokens); clearEquValues()237 static void clearEquValues() { equValues.clear(); } 238 void resetLookaheadCheckMarks(); 239 protected: clearTokens()240 void clearTokens() { tokens.clear(); }; resetPosition()241 void resetPosition() { position.it = tokens.begin(); } 242 void addToken(Token token); 243 private: 244 bool processElement(TokenList::iterator& it); 245 246 TokenList tokens; 247 TokenizerPosition position; 248 249 struct Replacement 250 { 251 std::wstring identifier; 252 std::vector<Token> value; 253 }; 254 255 Token invalidToken; 256 std::vector<Replacement> replacements; 257 static std::vector<std::vector<Token>> equValues; 258 }; 259 260 class FileTokenizer: public Tokenizer 261 { 262 public: 263 bool init(TextFile* input); 264 protected: 265 Token loadToken(); isInputAtEnd()266 bool isInputAtEnd() { return linePos >= currentLine.size() && input->atEnd(); }; 267 268 void skipWhitespace(); 269 void createToken(TokenType type, size_t length); 270 void createToken(TokenType type, size_t length, int64_t value); 271 void createToken(TokenType type, size_t length, double value); 272 void createToken(TokenType type, size_t length, const std::wstring& value); 273 void createToken(TokenType type, size_t length, const std::wstring& value, size_t valuePos, size_t valueLength); 274 void createTokenCurrentString(TokenType type, size_t length); 275 276 bool convertInteger(size_t start, size_t end, int64_t& result); 277 bool convertFloat(size_t start, size_t end, double& result); 278 bool parseOperator(); 279 280 TextFile* input; 281 std::wstring currentLine; 282 size_t lineNumber; 283 size_t linePos; 284 285 Token token; 286 bool equActive; 287 }; 288 289 class TokenStreamTokenizer: public Tokenizer 290 { 291 public: init(const std::vector<Token> & tokens)292 void init(const std::vector<Token>& tokens) 293 { 294 clearTokens(); 295 296 for (const Token &tok: tokens) 297 addToken(tok); 298 299 resetPosition(); 300 } 301 }; 302