1 /* 2 * tokenizer.cc 3 * DIN Is Noise is copyright (c) 2006-2021 Jagannathan Sampath 4 * DIN Is Noise is released under GNU Public License 2.0 5 * For more information, please visit https://dinisnoise.org/ 6 */ 7 8 9 // string tokenizer 10 // based on initial code by Song Ho Ahn (song.ahn@gmail.com) 11 12 #include "tokenizer.h" 13 #include <cstdlib> 14 15 #include <iostream> 16 using namespace std; 17 tokenizer()18tokenizer::tokenizer() { 19 str (""); 20 del (DEFAULT_DELIMITER); 21 } 22 tokenizer(const std::string & s,const std::string & d)23tokenizer::tokenizer(const std::string& s, const std::string& d) { 24 str (s); 25 del (d); 26 } 27 tokenizer(const std::vector<std::string> & t)28tokenizer::tokenizer (const std::vector<std::string>& t) { 29 vec (t); 30 } 31 set(const std::string & s,const std::string & d)32void tokenizer::set (const std::string& s, const std::string& d) { 33 str (s); 34 del (d); 35 } 36 str(const std::string & s)37void tokenizer::str (const std::string& s) { 38 buffer = s; 39 blen = buffer.length (); 40 init_cur (); 41 tokens_available = 0; 42 } 43 del(const std::string & d)44void tokenizer::del (const std::string& d) { 45 delimiter = d; 46 dlen = delimiter.length (); 47 } 48 vec(const std::vector<std::string> & t)49void tokenizer::vec (const std::vector<std::string>& t) { 50 tokens = t; 51 tokens_available = 1; 52 tokid = 0; 53 } 54 init_cur()55void tokenizer::init_cur () { 56 if (blen) cur = 0; else cur = -1; 57 } 58 operator >>(std::string & s)59tokenizer& tokenizer::operator>> (std::string& s) { 60 if (tokens_available) { 61 if (tokid < tokens.size()) s = tokens[tokid++]; else { 62 s = ""; 63 cur = blen; 64 } 65 } else { 66 token = ""; 67 if (blen < 1) { 68 s = token; 69 } else { 70 skip_ws (); 71 while ((cur < blen) && !isdelim (buffer[cur])) { 72 token += buffer[cur]; 73 ++cur; 74 } 75 s = token; 76 } 77 } 78 return *this; 79 } 80 operator >>(double & d)81tokenizer& tokenizer::operator>> (double& d) { 82 string s; *this >> s; 83 d = atof (s.c_str()); 84 return *this; 85 } 86 operator >>(float & f)87tokenizer& tokenizer::operator>> (float& f) { 88 string s; *this >> s; 89 f = (float) atof (s.c_str()); 90 return *this; 91 } 92 operator >>(int & i)93tokenizer& tokenizer::operator>> (int& i) { 94 string s; *this >> s; 95 i = atoi (s.c_str()); 96 return *this; 97 } 98 operator >>(char & c)99tokenizer& tokenizer::operator>> (char& c) { 100 string s; *this >> s; 101 if (s.length()) c = s[0]; 102 return *this; 103 } 104 operator >>(unsigned char & c)105tokenizer& tokenizer::operator>> (unsigned char& c) { 106 string s; *this >> s; 107 if (s.length()) c = s[0]; 108 return *this; 109 } 110 skip_ws()111void tokenizer::skip_ws () { 112 while (cur < blen) if (!isdelim (buffer[cur])) break; else ++cur; 113 } 114 isdelim(char c)115bool tokenizer::isdelim (char c) { 116 for (int i = 0; i < dlen; ++i) { 117 if (delimiter[i] == c) return true; 118 } 119 return false; 120 } 121 cur2end()122string tokenizer::cur2end () { 123 if (tokens_available) { 124 string result; 125 static const char spc = ' '; 126 for (int i = tokid, j = tokens.size(); i < j; ++i) result = result + tokens[i] + spc; 127 return result; 128 } else { 129 if (cur > -1) { 130 return buffer.substr (cur, blen - 1); 131 } else return ""; 132 } 133 } 134