1 use std::str::FromStr;
2
3 use regex::Regex;
4
5 use crate::field::Field;
6 use crate::function::Function;
7
8 #[derive(Clone, PartialEq, Debug)]
9 pub enum Lexem {
10 RawString(String),
11 Comma,
12 From,
13 Where,
14 Operator(String),
15 String(String),
16 Open,
17 Close,
18 ArithmeticOperator(String),
19 And,
20 Or,
21 Not,
22 Order,
23 By,
24 DescendingOrder,
25 Limit,
26 Into,
27 }
28
29 #[derive(Debug, PartialEq)]
30 enum LexingMode {
31 Undefined,
32 RawString,
33 Comma,
34 Operator,
35 ArithmeticOperator,
36 String,
37 Open,
38 Close,
39 }
40
41 pub struct Lexer<'a> {
42 input: &'a str,
43 index: usize,
44 before_from: bool,
45 after_open: bool,
46 after_where: bool,
47 after_operator: bool,
48 }
49
50 impl<'a> Lexer<'a> {
new(input: &str) -> Lexer51 pub fn new(input: &str) -> Lexer {
52 return Lexer { input, index: 0, before_from: true, after_open: false, after_where: false, after_operator: false, }
53 }
54
next_lexem(&mut self) -> Option<Lexem>55 pub fn next_lexem(&mut self) -> Option<Lexem> {
56 let mut s = String::new();
57 let mut mode = LexingMode::Undefined;
58
59 for c in self.input.chars().skip(self.index) {
60 match mode {
61 LexingMode::Comma | LexingMode::Open | LexingMode::Close => {
62 break
63 },
64 LexingMode::String => {
65 self.index += 1;
66 if c == '\'' {
67 break
68 }
69 s.push(c);
70 },
71 LexingMode::Operator => {
72 if !self.is_op_char(c) {
73 break
74 }
75
76 self.index += 1;
77 s.push(c);
78 },
79 LexingMode::ArithmeticOperator => {
80 break;
81 },
82 LexingMode::RawString => {
83 let is_date = c == '-' && looks_like_date(&s);
84 if !is_date {
85 if self.is_arithmetic_op_char(c) {
86 let maybe_expr = looks_like_expression(&s);
87 if maybe_expr {
88 break;
89 }
90 } else if c == ' ' || c == ',' || c == '(' || c == ')' || self.is_op_char(c) {
91 break
92 }
93 }
94
95 self.index += 1;
96 s.push(c);
97 },
98 LexingMode::Undefined => {
99 self.index += 1;
100 match c {
101 ' ' => {},
102 '\'' => mode = LexingMode::String,
103 ',' => mode = LexingMode::Comma,
104 '(' => mode = LexingMode::Open,
105 ')' => mode = LexingMode::Close,
106 _ => {
107 mode = if self.is_op_char(c) {
108 LexingMode::Operator
109 } else if self.is_arithmetic_op_char(c) {
110 LexingMode::ArithmeticOperator
111 } else {
112 LexingMode::RawString
113 };
114 s.push(c);
115 }
116 }
117
118 if mode == LexingMode::Open {
119 self.after_open = true;
120 } else {
121 self.after_open = false;
122 }
123 },
124 }
125 }
126
127 let lexem = match mode {
128 LexingMode::String => Some(Lexem::String(s)),
129 LexingMode::Operator => Some(Lexem::Operator(s)),
130 LexingMode::ArithmeticOperator => Some(Lexem::ArithmeticOperator(s)),
131 LexingMode::Comma => Some(Lexem::Comma),
132 LexingMode::Open => Some(Lexem::Open),
133 LexingMode::Close => Some(Lexem::Close),
134 LexingMode::RawString => {
135 match s.to_lowercase().as_str() {
136 "from" => { self.before_from = false; Some(Lexem::From) },
137 "where" => { self.after_where = true; Some(Lexem::Where) },
138 "or" => Some(Lexem::Or),
139 "and" => Some(Lexem::And),
140 "not" if self.after_where => Some(Lexem::Not),
141 "order" => Some(Lexem::Order),
142 "by" => Some(Lexem::By),
143 "asc" => self.next_lexem(),
144 "desc" => Some(Lexem::DescendingOrder),
145 "limit" => Some(Lexem::Limit),
146 "into" => Some(Lexem::Into),
147 "eq" | "ne" | "gt" | "lt" | "ge" | "le" | "gte" | "lte" |
148 "regexp" | "rx" | "like" => Some(Lexem::Operator(s)),
149 "mul" | "div" | "plus" | "minus" => Some(Lexem::ArithmeticOperator(s)),
150 _ => Some(Lexem::RawString(s)),
151 }
152 },
153 _ => None
154 };
155
156 self.after_operator = match lexem {
157 Some(Lexem::Operator(_)) => true,
158 _ => false
159 };
160
161 lexem
162 }
163
is_arithmetic_op_char(&self, c: char) -> bool164 fn is_arithmetic_op_char(&self, c: char) -> bool {
165 match c {
166 '+' | '-' => self.before_from || self.after_where,
167 '*' | '/' => (self.before_from || self.after_where) && !self.after_open && !self.after_operator,
168 _ => false
169 }
170 }
171
is_op_char(&self, c: char) -> bool172 fn is_op_char(&self, c: char) -> bool {
173 if !self.before_from && !self.after_where {
174 return false;
175 }
176
177 match c {
178 '=' | '!' | '<' | '>' | '~' => true,
179 _ => false
180 }
181 }
182 }
183
184
185 lazy_static! {
186 static ref DATE_ALIKE_REGEX: Regex = Regex::new("(\\d{4})-?(\\d{2})?").unwrap();
187 }
188
looks_like_expression(s: &str) -> bool189 fn looks_like_expression(s: &str) -> bool {
190 !s.split(|c: char| !c.is_ascii_alphanumeric())
191 .any(|s| Field::from_str(s).is_err() && Function::from_str(s).is_err() && s.parse::<i64>().is_err())
192 }
193
looks_like_date(s: &str) -> bool194 fn looks_like_date(s: &str) -> bool {
195 match DATE_ALIKE_REGEX.captures(s) {
196 Some(cap) => {
197 let year = cap[1].parse::<i32>();
198 let year_ok = match year {
199 Ok(year) => year >= 1970 && year < 3000, // optimistic assumption
200 _ => false
201 };
202
203 if !year_ok {
204 return false;
205 }
206
207 match cap.get(2) {
208 Some(month) => {
209 let month = month.as_str().parse::<i32>();
210 let month_ok = match month {
211 Ok(month) => month >= 1 && month <= 12,
212 _ => false
213 };
214
215 month_ok
216 },
217 _ => true
218 }
219 },
220 _ => false
221 }
222 }
223
224 #[cfg(test)]
225 mod tests {
226 use super::*;
227
228 #[test]
looks_like_date_test()229 fn looks_like_date_test() {
230 assert!(looks_like_date("2018"));
231 assert!(looks_like_date("2018-01"));
232 }
233
234 #[test]
lexems()235 fn lexems() {
236 let mut lexer = Lexer::new("select name, path ,size , fsize from /test depth 2, /test2 archives,/test3 depth 3 archives , /test4 ,'/test5' where name != 123 AND ( size gt 456 or fsize lte 758) or name = 'xxx' order by 1 ,3 desc , path asc limit 50");
237
238 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("select"))));
239 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
240 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
241 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("path"))));
242 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
243 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("size"))));
244 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
245 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("fsize"))));
246 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
247 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("/test"))));
248 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("depth"))));
249 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("2"))));
250 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
251 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("/test2"))));
252 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("archives"))));
253 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
254 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("/test3"))));
255 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("depth"))));
256 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("3"))));
257 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("archives"))));
258 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
259 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("/test4"))));
260 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
261 assert_eq!(lexer.next_lexem(), Some(Lexem::String(String::from("/test5"))));
262 assert_eq!(lexer.next_lexem(), Some(Lexem::Where));
263 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
264 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("!="))));
265 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("123"))));
266 assert_eq!(lexer.next_lexem(), Some(Lexem::And));
267 assert_eq!(lexer.next_lexem(), Some(Lexem::Open));
268 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("size"))));
269 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("gt"))));
270 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("456"))));
271 assert_eq!(lexer.next_lexem(), Some(Lexem::Or));
272 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("fsize"))));
273 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("lte"))));
274 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("758"))));
275 assert_eq!(lexer.next_lexem(), Some(Lexem::Close));
276 assert_eq!(lexer.next_lexem(), Some(Lexem::Or));
277 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
278 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("="))));
279 assert_eq!(lexer.next_lexem(), Some(Lexem::String(String::from("xxx"))));
280 assert_eq!(lexer.next_lexem(), Some(Lexem::Order));
281 assert_eq!(lexer.next_lexem(), Some(Lexem::By));
282 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("1"))));
283 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
284 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("3"))));
285 assert_eq!(lexer.next_lexem(), Some(Lexem::DescendingOrder));
286 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
287 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("path"))));
288 assert_eq!(lexer.next_lexem(), Some(Lexem::Limit));
289 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("50"))));
290 }
291
292 #[test]
spaces()293 fn spaces() {
294 let lexer = Lexer::new("path,size from . where size=0");
295 assert_spaces(lexer);
296
297 let lexer = Lexer::new("path,size from . where size =0");
298 assert_spaces(lexer);
299
300 let lexer = Lexer::new("path,size from . where size= 0");
301 assert_spaces(lexer);
302
303 let lexer = Lexer::new("path,size from . where size = 0");
304 assert_spaces(lexer);
305
306 let lexer = Lexer::new("path,size from . where size = 0");
307 assert_spaces(lexer);
308 }
309
assert_spaces(mut lexer: Lexer)310 fn assert_spaces(mut lexer: Lexer) {
311 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("path"))));
312 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
313 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("size"))));
314 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
315 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("."))));
316 assert_eq!(lexer.next_lexem(), Some(Lexem::Where));
317 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("size"))));
318 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("="))));
319 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("0"))));
320 }
321
322 #[test]
func_calls()323 fn func_calls() {
324 let mut lexer = Lexer::new("name, length(name),UPPER( name ) from .");
325
326 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
327 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
328 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("length"))));
329 assert_eq!(lexer.next_lexem(), Some(Lexem::Open));
330 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
331 assert_eq!(lexer.next_lexem(), Some(Lexem::Close));
332 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
333 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("UPPER"))));
334 assert_eq!(lexer.next_lexem(), Some(Lexem::Open));
335 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
336 assert_eq!(lexer.next_lexem(), Some(Lexem::Close));
337 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
338 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("."))));
339 }
340
341 #[test]
func_calls2()342 fn func_calls2() {
343 let mut lexer = Lexer::new("select name, upper(name) from . depth 1");
344
345 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("select"))));
346 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
347 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
348 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("upper"))));
349 assert_eq!(lexer.next_lexem(), Some(Lexem::Open));
350 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
351 assert_eq!(lexer.next_lexem(), Some(Lexem::Close));
352 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
353 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("."))));
354 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("depth"))));
355 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("1"))));
356 }
357
358 #[test]
func_calls3()359 fn func_calls3() {
360 let mut lexer = Lexer::new("select name, rand() from . depth 1 order by rand() limit 10");
361
362 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("select"))));
363 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
364 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
365 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("rand"))));
366 assert_eq!(lexer.next_lexem(), Some(Lexem::Open));
367 assert_eq!(lexer.next_lexem(), Some(Lexem::Close));
368 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
369 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("."))));
370 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("depth"))));
371 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("1"))));
372 assert_eq!(lexer.next_lexem(), Some(Lexem::Order));
373 assert_eq!(lexer.next_lexem(), Some(Lexem::By));
374 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("rand"))));
375 assert_eq!(lexer.next_lexem(), Some(Lexem::Open));
376 assert_eq!(lexer.next_lexem(), Some(Lexem::Close));
377 assert_eq!(lexer.next_lexem(), Some(Lexem::Limit));
378 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("10"))));
379 }
380
381 #[test]
agg_func_calls()382 fn agg_func_calls() {
383 let mut lexer = Lexer::new("COUNT(*), MIN(size), AVG(size), MAX(size) from .");
384
385 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("COUNT"))));
386 assert_eq!(lexer.next_lexem(), Some(Lexem::Open));
387 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("*"))));
388 assert_eq!(lexer.next_lexem(), Some(Lexem::Close));
389 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
390 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("MIN"))));
391 assert_eq!(lexer.next_lexem(), Some(Lexem::Open));
392 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("size"))));
393 assert_eq!(lexer.next_lexem(), Some(Lexem::Close));
394 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
395 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("AVG"))));
396 assert_eq!(lexer.next_lexem(), Some(Lexem::Open));
397 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("size"))));
398 assert_eq!(lexer.next_lexem(), Some(Lexem::Close));
399 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
400 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("MAX"))));
401 assert_eq!(lexer.next_lexem(), Some(Lexem::Open));
402 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("size"))));
403 assert_eq!(lexer.next_lexem(), Some(Lexem::Close));
404 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
405 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("."))));
406 }
407
408 #[test]
arithmetic_operators()409 fn arithmetic_operators() {
410 let mut lexer = Lexer::new("width + height, width-height, width mul height, path from .");
411
412 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("width"))));
413 assert_eq!(lexer.next_lexem(), Some(Lexem::ArithmeticOperator(String::from("+"))));
414 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("height"))));
415 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
416 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("width"))));
417 assert_eq!(lexer.next_lexem(), Some(Lexem::ArithmeticOperator(String::from("-"))));
418 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("height"))));
419 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
420 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("width"))));
421 assert_eq!(lexer.next_lexem(), Some(Lexem::ArithmeticOperator(String::from("mul"))));
422 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("height"))));
423 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
424 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("path"))));
425 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
426 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("."))));
427 }
428
429 #[test]
context_sensitive_date_string()430 fn context_sensitive_date_string() {
431 let mut lexer = Lexer::new("size,modified,path from . where modified gt 2018-08-01 and name='*.txt' order by modified");
432
433 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("size"))));
434 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
435 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("modified"))));
436 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
437 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("path"))));
438 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
439 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("."))));
440 assert_eq!(lexer.next_lexem(), Some(Lexem::Where));
441 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("modified"))));
442 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("gt"))));
443 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("2018-08-01"))));
444 assert_eq!(lexer.next_lexem(), Some(Lexem::And));
445 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
446 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("="))));
447 assert_eq!(lexer.next_lexem(), Some(Lexem::String(String::from("*.txt"))));
448 assert_eq!(lexer.next_lexem(), Some(Lexem::Order));
449 assert_eq!(lexer.next_lexem(), Some(Lexem::By));
450 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("modified"))));
451 }
452
453 #[test]
root_with_dashes()454 fn root_with_dashes() {
455 let mut lexer = Lexer::new("path from ./foo-bar");
456
457 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("path"))));
458 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
459 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("./foo-bar"))));
460 }
461
462 #[test]
another_workaround_for_raw_paths()463 fn another_workaround_for_raw_paths() {
464 let mut lexer = Lexer::new("name, size where path eq \\*some/stuff-inside/\\*.rs");
465
466 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
467 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
468 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("size"))));
469 assert_eq!(lexer.next_lexem(), Some(Lexem::Where));
470 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("path"))));
471 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("eq"))));
472 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("\\*some/stuff-inside/\\*.rs"))));
473 }
474
475 #[test]
mime_types()476 fn mime_types() {
477 let mut lexer = Lexer::new("mime from . where mime = application/pkcs8+pem");
478
479 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("mime"))));
480 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
481 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("."))));
482 assert_eq!(lexer.next_lexem(), Some(Lexem::Where));
483 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("mime"))));
484 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("="))));
485 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("application/pkcs8+pem"))));
486 }
487
488 #[test]
raw_abs_path_after_eq()489 fn raw_abs_path_after_eq() {
490 let mut lexer = Lexer::new("abspath,absdir,name where absdir = /home/user/docs");
491
492 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("abspath"))));
493 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
494 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("absdir"))));
495 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
496 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
497 assert_eq!(lexer.next_lexem(), Some(Lexem::Where));
498 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("absdir"))));
499 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("="))));
500 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("/home/user/docs"))));
501
502 let mut lexer = Lexer::new("abspath,absdir,name where absdir == /home/user/docs");
503
504 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("abspath"))));
505 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
506 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("absdir"))));
507 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
508 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
509 assert_eq!(lexer.next_lexem(), Some(Lexem::Where));
510 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("absdir"))));
511 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("=="))));
512 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("/home/user/docs"))));
513
514 let mut lexer = Lexer::new("abspath,absdir,name where absdir === /home/user/docs");
515
516 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("abspath"))));
517 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
518 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("absdir"))));
519 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
520 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
521 assert_eq!(lexer.next_lexem(), Some(Lexem::Where));
522 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("absdir"))));
523 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("==="))));
524 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("/home/user/docs"))));
525
526 let mut lexer = Lexer::new("abspath,absdir,name where absdir eq /home/user/docs");
527
528 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("abspath"))));
529 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
530 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("absdir"))));
531 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
532 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("name"))));
533 assert_eq!(lexer.next_lexem(), Some(Lexem::Where));
534 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("absdir"))));
535 assert_eq!(lexer.next_lexem(), Some(Lexem::Operator(String::from("eq"))));
536 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("/home/user/docs"))));
537 }
538
539 #[test]
star_field()540 fn star_field() {
541 let mut lexer = Lexer::new("select modified,* from /test");
542
543 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("select"))));
544 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("modified"))));
545 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
546 assert_eq!(lexer.next_lexem(), Some(Lexem::ArithmeticOperator(String::from("*"))));
547 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
548 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("/test"))));
549
550 let mut lexer = Lexer::new("select modified, * from /test limit 10");
551
552 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("select"))));
553 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("modified"))));
554 assert_eq!(lexer.next_lexem(), Some(Lexem::Comma));
555 assert_eq!(lexer.next_lexem(), Some(Lexem::ArithmeticOperator(String::from("*"))));
556 assert_eq!(lexer.next_lexem(), Some(Lexem::From));
557 assert_eq!(lexer.next_lexem(), Some(Lexem::RawString(String::from("/test"))));
558 }
559 }
560