1 /*
2 * tokenizer.cc
3 * DIN Is Noise is copyright (c) 2006-2021 Jagannathan Sampath
4 * DIN Is Noise is released under GNU Public License 2.0
5 * For more information, please visit https://dinisnoise.org/
6 */
7 
8 
9 // string tokenizer
10 // based on initial code by Song Ho Ahn (song.ahn@gmail.com)
11 
12 #include "tokenizer.h"
13 #include <cstdlib>
14 
15 #include <iostream>
16 using namespace std;
17 
tokenizer()18 tokenizer::tokenizer() {
19   str ("");
20   del (DEFAULT_DELIMITER);
21 }
22 
tokenizer(const std::string & s,const std::string & d)23 tokenizer::tokenizer(const std::string& s, const std::string& d) {
24   str (s);
25   del (d);
26 }
27 
tokenizer(const std::vector<std::string> & t)28 tokenizer::tokenizer (const std::vector<std::string>& t) {
29   vec (t);
30 }
31 
set(const std::string & s,const std::string & d)32 void tokenizer::set (const std::string& s, const std::string& d) {
33   str (s);
34   del (d);
35 }
36 
str(const std::string & s)37 void tokenizer::str (const std::string& s) {
38   buffer = s;
39   blen = buffer.length ();
40   init_cur ();
41   tokens_available = 0;
42 }
43 
del(const std::string & d)44 void tokenizer::del (const std::string& d) {
45   delimiter = d;
46   dlen = delimiter.length ();
47 }
48 
vec(const std::vector<std::string> & t)49 void tokenizer::vec (const std::vector<std::string>& t) {
50   tokens = t;
51   tokens_available = 1;
52   tokid = 0;
53 }
54 
init_cur()55 void tokenizer::init_cur () {
56   if (blen) cur = 0; else cur = -1;
57 }
58 
operator >>(std::string & s)59 tokenizer& tokenizer::operator>> (std::string& s) {
60   if (tokens_available) {
61     if (tokid < tokens.size()) s = tokens[tokid++]; else {
62       s = "";
63       cur = blen;
64     }
65   } else {
66     token = "";
67     if (blen < 1) {
68       s = token;
69     } else {
70       skip_ws ();
71       while ((cur < blen) && !isdelim (buffer[cur])) {
72         token += buffer[cur];
73         ++cur;
74       }
75       s = token;
76     }
77   }
78   return *this;
79 }
80 
operator >>(double & d)81 tokenizer& tokenizer::operator>> (double& d) {
82   string s; *this >> s;
83   d = atof (s.c_str());
84   return *this;
85 }
86 
operator >>(float & f)87 tokenizer& tokenizer::operator>> (float& f) {
88   string s; *this >> s;
89   f = (float) atof (s.c_str());
90   return *this;
91 }
92 
operator >>(int & i)93 tokenizer& tokenizer::operator>> (int& i) {
94   string s; *this >> s;
95   i = atoi (s.c_str());
96   return *this;
97 }
98 
operator >>(char & c)99 tokenizer& tokenizer::operator>> (char& c) {
100   string s; *this >> s;
101   if (s.length()) c = s[0];
102   return *this;
103 }
104 
operator >>(unsigned char & c)105 tokenizer& tokenizer::operator>> (unsigned char& c) {
106   string s; *this >> s;
107   if (s.length()) c = s[0];
108   return *this;
109 }
110 
skip_ws()111 void tokenizer::skip_ws () {
112   while (cur < blen) if (!isdelim (buffer[cur])) break; else ++cur;
113 }
114 
isdelim(char c)115 bool tokenizer::isdelim (char c) {
116   for (int i = 0; i < dlen; ++i) {
117     if (delimiter[i] == c) return true;
118   }
119   return false;
120 }
121 
cur2end()122 string tokenizer::cur2end () {
123   if (tokens_available) {
124     string result;
125     static const char spc = ' ';
126     for (int i = tokid, j = tokens.size(); i < j; ++i) result = result + tokens[i] + spc;
127     return result;
128   } else {
129     if (cur > -1) {
130       return buffer.substr (cur, blen - 1);
131     } else return "";
132   }
133 }
134