1 #ifndef _TOKENIZER_H_INCLUDED_ 2 #define _TOKENIZER_H_INCLUDED_ 3 /////////////////////////////////////////////////////////////////////////////// 4 // 5 // Tokenizer.h 6 // ----------- 7 // Tokenizer class definition 8 // 9 // Design and Implementation by Bjoern Lemke 10 // 11 // (C)opyright 2000-2016 Bjoern Lemke 12 // 13 // INTERFACE MODULE 14 // 15 // Class: Tokenizer 16 // 17 // Description: Splitting a string into tokens 18 // 19 // Status: CLEAN 20 // 21 /////////////////////////////////////////////////////////////////////////////// 22 23 // INCLUDES 24 25 #include "Chain.h" 26 27 // CLASS DEFINITION 28 29 class Tokenizer { 30 31 public: 32 33 Tokenizer(const Chain& tokenStream, const Chain& sep, char esc = '\\'); 34 Tokenizer(char* buf, int len, const Chain& sep, char esc = '\\'); 35 ~Tokenizer(); 36 37 bool nextToken(Chain& token); 38 bool getTail(Chain& token); 39 40 void skip(int n); 41 bool rawRead(Chain& token, int n); 42 43 int numAhead() const; 44 45 void reset(char* buf, int len); 46 47 Tokenizer& operator = (const Tokenizer& t); 48 49 private: 50 51 bool checkSeparator(); 52 53 bool _isLocal; 54 char *_streamBuf; 55 int _bufLen; 56 57 Chain _tokenStream; 58 Chain _sep; 59 char _esc; 60 61 int _lower; 62 int _upper; 63 }; 64 #endif 65 66 67 68