1 /* 2 AngelCode Scripting Library 3 Copyright (c) 2003-2013 Andreas Jonsson 4 5 This software is provided 'as-is', without any express or implied 6 warranty. In no event will the authors be held liable for any 7 damages arising from the use of this software. 8 9 Permission is granted to anyone to use this software for any 10 purpose, including commercial applications, and to alter it and 11 redistribute it freely, subject to the following restrictions: 12 13 1. The origin of this software must not be misrepresented; you 14 must not claim that you wrote the original software. If you use 15 this software in a product, an acknowledgment in the product 16 documentation would be appreciated but is not required. 17 18 2. Altered source versions must be plainly marked as such, and 19 must not be misrepresented as being the original software. 20 21 3. This notice may not be removed or altered from any source 22 distribution. 23 24 The original version of this library can be located at: 25 http://www.angelcode.com/angelscript/ 26 27 Andreas Jonsson 28 andreas@angelcode.com 29 */ 30 31 32 // 33 // as_tokenizer.cpp 34 // 35 // This class identifies tokens from the script code 36 // 37 38 39 40 #ifndef AS_TOKENIZER_H 41 #define AS_TOKENIZER_H 42 43 #include "as_config.h" 44 #include "as_tokendef.h" 45 #include "as_map.h" 46 #include "as_string.h" 47 48 BEGIN_AS_NAMESPACE 49 50 class asCTokenizer 51 { 52 public: 53 eTokenType GetToken(const char *source, size_t sourceLength, size_t *tokenLength, asETokenClass *tc = 0) const; 54 55 static const char *GetDefinition(int tokenType); 56 57 protected: 58 friend class asCScriptEngine; 59 60 asCTokenizer(); 61 ~asCTokenizer(); 62 63 asETokenClass ParseToken(const char *source, size_t sourceLength, size_t &tokenLength, eTokenType &tokenType) const; 64 bool IsWhiteSpace(const char *source, size_t sourceLength, size_t &tokenLength, eTokenType &tokenType) const; 65 bool IsComment(const char *source, size_t sourceLength, size_t &tokenLength, eTokenType &tokenType) const; 66 bool IsConstant(const char *source, size_t sourceLength, size_t &tokenLength, eTokenType &tokenType) const; 67 bool IsKeyWord(const char *source, size_t sourceLength, size_t &tokenLength, eTokenType &tokenType) const; 68 bool IsIdentifier(const char *source, size_t sourceLength, size_t &tokenLength, eTokenType &tokenType) const; 69 bool IsDigitInRadix(char ch, int radix) const; 70 71 const asCScriptEngine *engine; 72 73 const sTokenWord **keywordTable[256]; 74 }; 75 76 END_AS_NAMESPACE 77 78 #endif 79 80