1 // +------------------------------------------------------------------+
2 // | ____ _ _ __ __ _ __ |
3 // | / ___| |__ ___ ___| | __ | \/ | |/ / |
4 // | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / |
5 // | | |___| | | | __/ (__| < | | | | . \ |
6 // | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ |
7 // | |
8 // | Copyright Mathias Kettner 2014 mk@mathias-kettner.de |
9 // +------------------------------------------------------------------+
10 //
11 // This file is part of Check_MK.
12 // The official homepage is at http://mathias-kettner.de/check_mk.
13 //
14 // check_mk is free software; you can redistribute it and/or modify it
15 // under the terms of the GNU General Public License as published by
16 // the Free Software Foundation in version 2. check_mk is distributed
17 // in the hope that it will be useful, but WITHOUT ANY WARRANTY; with-
18 // out even the implied warranty of MERCHANTABILITY or FITNESS FOR A
19 // PARTICULAR PURPOSE. See the GNU General Public License for more de-
20 // ails. You should have received a copy of the GNU General Public
21 // License along with GNU Make; see the file COPYING. If not, write
22 // to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
23 // Boston, MA 02110-1301 USA.
24
25 #include "Query.h"
26 #include <ctype.h>
27 #include <math.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <sys/time.h>
32 #include <syslog.h>
33 #include <utility>
34 #include <vector>
35 #include "Aggregator.h"
36 #include "Column.h"
37 #include "Filter.h"
38 #include "NegatingFilter.h"
39 #include "NullColumn.h"
40 #include "OringFilter.h"
41 #include "OutputBuffer.h"
42 #include "StatsColumn.h"
43 #include "Table.h"
44 #include "auth.h"
45 #include "data_encoding.h"
46 #include "logger.h"
47 #include "opids.h"
48 #include "strutil.h"
49 #include "waittriggers.h"
50
51 extern int g_debug_level;
52 extern unsigned long g_max_response_size;
53 extern int g_data_encoding;
54
55 using std::list;
56 using std::string;
57 using std::vector;
58
Query(const list<string> & lines,OutputBuffer * output,Table * table)59 Query::Query(const list<string> &lines, OutputBuffer *output, Table *table)
60 : _output(output)
61 , _table(table)
62 , _auth_user(nullptr)
63 , _wait_timeout(0)
64 , _wait_trigger(nullptr)
65 , _wait_object(nullptr)
66 , _field_separator(";")
67 , _dataset_separator("\n")
68 , _list_separator(",")
69 , _host_service_separator("|")
70 , _show_column_headers(true)
71 , _need_ds_separator(false)
72 , _output_format(OUTPUT_FORMAT_CSV)
73 , _limit(-1)
74 , _time_limit(-1)
75 , _time_limit_timeout(0)
76 , _current_line(0)
77 , _timezone_offset(0) {
78 for (auto &line : lines) {
79 vector<char> line_copy(line.begin(), line.end());
80 line_copy.push_back('\0');
81 char *buffer = &line_copy[0];
82 rstrip(buffer);
83 if (g_debug_level > 0) {
84 logger(LG_INFO, "Query: %s", buffer);
85 }
86 if (strncmp(buffer, "Filter:", 7) == 0) {
87 parseFilterLine(lstrip(buffer + 7), true);
88
89 } else if (strncmp(buffer, "Or:", 3) == 0) {
90 parseAndOrLine(lstrip(buffer + 3), ANDOR_OR, true);
91
92 } else if (strncmp(buffer, "And:", 4) == 0) {
93 parseAndOrLine(lstrip(buffer + 4), ANDOR_AND, true);
94
95 } else if (strncmp(buffer, "Negate:", 7) == 0) {
96 parseNegateLine(lstrip(buffer + 7), true);
97
98 } else if (strncmp(buffer, "StatsOr:", 8) == 0) {
99 parseStatsAndOrLine(lstrip(buffer + 8), ANDOR_OR);
100
101 } else if (strncmp(buffer, "StatsAnd:", 9) == 0) {
102 parseStatsAndOrLine(lstrip(buffer + 9), ANDOR_AND);
103
104 } else if (strncmp(buffer, "StatsNegate:", 12) == 0) {
105 parseStatsNegateLine(lstrip(buffer + 12));
106
107 } else if (strncmp(buffer, "Stats:", 6) == 0) {
108 parseStatsLine(lstrip(buffer + 6));
109
110 } else if (strncmp(buffer, "StatsGroupBy:", 13) == 0) {
111 parseStatsGroupLine(lstrip(buffer + 13));
112
113 } else if (strncmp(buffer, "Columns:", 8) == 0) {
114 parseColumnsLine(lstrip(buffer + 8));
115
116 } else if (strncmp(buffer, "ColumnHeaders:", 14) == 0) {
117 parseColumnHeadersLine(lstrip(buffer + 14));
118
119 } else if (strncmp(buffer, "Limit:", 6) == 0) {
120 parseLimitLine(lstrip(buffer + 6));
121
122 } else if (strncmp(buffer, "Timelimit:", 10) == 0) {
123 parseTimelimitLine(lstrip(buffer + 10));
124
125 } else if (strncmp(buffer, "AuthUser:", 9) == 0) {
126 parseAuthUserHeader(lstrip(buffer + 9));
127
128 } else if (strncmp(buffer, "Separators:", 11) == 0) {
129 parseSeparatorsLine(lstrip(buffer + 11));
130
131 } else if (strncmp(buffer, "OutputFormat:", 13) == 0) {
132 parseOutputFormatLine(lstrip(buffer + 13));
133
134 } else if (strncmp(buffer, "ResponseHeader:", 15) == 0) {
135 parseResponseHeaderLine(lstrip(buffer + 15));
136
137 } else if (strncmp(buffer, "KeepAlive:", 10) == 0) {
138 parseKeepAliveLine(lstrip(buffer + 10));
139
140 } else if (strncmp(buffer, "WaitCondition:", 14) == 0) {
141 parseFilterLine(lstrip(buffer + 14), false);
142
143 } else if (strncmp(buffer, "WaitConditionAnd:", 17) == 0) {
144 parseAndOrLine(lstrip(buffer + 17), ANDOR_AND, false);
145
146 } else if (strncmp(buffer, "WaitConditionOr:", 16) == 0) {
147 parseAndOrLine(lstrip(buffer + 16), ANDOR_OR, false);
148
149 } else if (strncmp(buffer, "WaitConditionNegate:", 20) == 0) {
150 parseNegateLine(lstrip(buffer + 20), false);
151
152 } else if (strncmp(buffer, "WaitTrigger:", 12) == 0) {
153 parseWaitTriggerLine(lstrip(buffer + 12));
154
155 } else if (strncmp(buffer, "WaitObject:", 11) == 0) {
156 parseWaitObjectLine(lstrip(buffer + 11));
157
158 } else if (strncmp(buffer, "WaitTimeout:", 12) == 0) {
159 parseWaitTimeoutLine(lstrip(buffer + 12));
160
161 } else if (strncmp(buffer, "Localtime:", 10) == 0) {
162 parseLocaltimeLine(lstrip(buffer + 10));
163
164 } else if (buffer[0] == 0) {
165 break;
166
167 } else {
168 output->setError(RESPONSE_CODE_INVALID_HEADER,
169 "Undefined request header '%s'", buffer);
170 break;
171 }
172 }
173 }
174
~Query()175 Query::~Query() {
176 // delete dynamic columns
177 for (auto column : _columns) {
178 if (column->mustDelete()) {
179 delete column;
180 }
181 }
182
183 for (auto &dummy_column : _dummy_columns) {
184 delete dummy_column;
185 }
186 for (auto &stats_column : _stats_columns) {
187 delete stats_column;
188 }
189 }
190
createDummyColumn(const char * name)191 Column *Query::createDummyColumn(const char *name) {
192 Column *col = new NullColumn(name, "Non existing column");
193 _dummy_columns.push_back(col);
194 return col;
195 }
196
addColumn(Column * column)197 void Query::addColumn(Column *column) { _columns.push_back(column); }
198
setError(int error_code,const char * msg)199 void Query::setError(int error_code, const char *msg) {
200 _output->setError(error_code, msg);
201 }
202
hasNoColumns()203 bool Query::hasNoColumns() { return _columns.empty() && !doStats(); }
204
lookupOperator(const char * opname)205 int Query::lookupOperator(const char *opname) {
206 int opid;
207 int negate = 1;
208 if (opname[0] == '!') {
209 negate = -1;
210 opname++;
211 }
212
213 if (strcmp(opname, "=") == 0) {
214 opid = OP_EQUAL;
215 } else if (strcmp(opname, "~") == 0) {
216 opid = OP_REGEX;
217 } else if (strcmp(opname, "=~") == 0) {
218 opid = OP_EQUAL_ICASE;
219 } else if (strcmp(opname, "~~") == 0) {
220 opid = OP_REGEX_ICASE;
221 } else if (strcmp(opname, ">") == 0) {
222 opid = OP_GREATER;
223 } else if (strcmp(opname, "<") == 0) {
224 opid = OP_LESS;
225 } else if (strcmp(opname, ">=") == 0) {
226 opid = OP_LESS;
227 negate = -negate;
228 } else if (strcmp(opname, "<=") == 0) {
229 opid = OP_GREATER;
230 negate = -negate;
231 } else {
232 opid = OP_INVALID;
233 }
234 return negate * opid;
235 }
236
createFilter(Column * column,int operator_id,char * value)237 Filter *Query::createFilter(Column *column, int operator_id, char *value) {
238 Filter *filter = column->createFilter(operator_id, value);
239 if (filter == nullptr) {
240 _output->setError(RESPONSE_CODE_INVALID_HEADER,
241 "cannot create filter on table %s", _table->name());
242 } else if (filter->hasError()) {
243 _output->setError(filter->errorCode(), "error in Filter header: %s",
244 filter->errorMessage().c_str());
245 delete filter;
246 filter = nullptr;
247 } else {
248 filter->setQuery(this);
249 filter->setColumn(column);
250 }
251 return filter;
252 }
253
parseAndOrLine(char * line,int andor,bool filter)254 void Query::parseAndOrLine(char *line, int andor, bool filter) {
255 char *value = next_field(&line);
256 if (value == nullptr) {
257 _output->setError(
258 RESPONSE_CODE_INVALID_HEADER,
259 "Missing value for %s%s: need non-zero integer number",
260 filter ? "" : "WaitCondition", andor == ANDOR_OR ? "Or" : "And");
261 return;
262 }
263
264 int number = atoi(value);
265 if ((isdigit(value[0]) == 0) || number <= 0) {
266 _output->setError(
267 RESPONSE_CODE_INVALID_HEADER,
268 "Invalid value for %s%s: need non-zero integer number",
269 filter ? "" : "WaitCondition", andor == ANDOR_OR ? "Or" : "And");
270 return;
271 }
272 if (filter) {
273 _filter.combineFilters(number, andor);
274 } else {
275 _wait_condition.combineFilters(number, andor);
276 }
277 }
278
parseNegateLine(char * line,bool filter)279 void Query::parseNegateLine(char *line, bool filter) {
280 if (next_field(&line) != nullptr) {
281 _output->setError(
282 RESPONSE_CODE_INVALID_HEADER,
283 filter ? "Negate: does not take any arguments"
284 : "WaitConditionNegate: does not take any arguments");
285 return;
286 }
287
288 Filter *to_negate;
289 if (filter) {
290 to_negate = _filter.stealLastSubfiler();
291 if (to_negate == nullptr) {
292 _output->setError(RESPONSE_CODE_INVALID_HEADER,
293 "Negate: no Filter: header to negate");
294 return;
295 }
296 } else {
297 to_negate = _wait_condition.stealLastSubfiler();
298 if (to_negate == nullptr) {
299 _output->setError(RESPONSE_CODE_INVALID_HEADER,
300 "Negate: no Wait:-condition negate");
301 return;
302 }
303 }
304 Filter *negated = new NegatingFilter(to_negate);
305 if (filter) {
306 _filter.addSubfilter(negated);
307 } else {
308 _wait_condition.addSubfilter(negated);
309 }
310 }
311
parseStatsAndOrLine(char * line,int andor)312 void Query::parseStatsAndOrLine(char *line, int andor) {
313 char *value = next_field(&line);
314 if (value == nullptr) {
315 _output->setError(
316 RESPONSE_CODE_INVALID_HEADER,
317 "Missing value for Stats%s: need non-zero integer number",
318 andor == ANDOR_OR ? "Or" : "And");
319 return;
320 }
321
322 int number = atoi(value);
323 if ((isdigit(value[0]) == 0) || number <= 0) {
324 _output->setError(
325 RESPONSE_CODE_INVALID_HEADER,
326 "Invalid value for Stats%s: need non-zero integer number",
327 andor == ANDOR_OR ? "Or" : "And");
328 return;
329 }
330
331 // The last 'number' StatsColumns must be of type STATS_OP_COUNT
332 AndingFilter *anding =
333 (andor == ANDOR_OR) ? new OringFilter() : new AndingFilter();
334 while (number > 0) {
335 if (_stats_columns.empty()) {
336 _output->setError(
337 RESPONSE_CODE_INVALID_HEADER,
338 "Invalid count for Stats%s: too few Stats: headers available",
339 andor == ANDOR_OR ? "Or" : "And");
340 delete anding;
341 return;
342 }
343
344 StatsColumn *col = _stats_columns.back();
345 if (col->operation() != STATS_OP_COUNT) {
346 _output->setError(
347 RESPONSE_CODE_INVALID_HEADER,
348 "Can use Stats%s only on Stats: headers of filter type",
349 andor == ANDOR_OR ? "Or" : "And");
350 delete anding;
351 return;
352 }
353 anding->addSubfilter(col->stealFilter());
354 delete col;
355 _stats_columns.pop_back();
356 number--;
357 }
358 _stats_columns.push_back(new StatsColumn(nullptr, anding, STATS_OP_COUNT));
359 }
360
parseStatsNegateLine(char * line)361 void Query::parseStatsNegateLine(char *line) {
362 if (next_field(&line) != nullptr) {
363 _output->setError(RESPONSE_CODE_INVALID_HEADER,
364 "StatsNegate: does not take any arguments");
365 return;
366 }
367 if (_stats_columns.empty()) {
368 _output->setError(RESPONSE_CODE_INVALID_HEADER,
369 "StatsNegate: no Stats: headers available");
370 return;
371 }
372 StatsColumn *col = _stats_columns.back();
373 if (col->operation() != STATS_OP_COUNT) {
374 _output->setError(
375 RESPONSE_CODE_INVALID_HEADER,
376 "Can use StatsNegate only on Stats: headers of filter type");
377 return;
378 }
379 auto negated = new NegatingFilter(col->stealFilter());
380 delete col;
381 _stats_columns.pop_back();
382 _stats_columns.push_back(new StatsColumn(nullptr, negated, STATS_OP_COUNT));
383 }
384
parseStatsLine(char * line)385 void Query::parseStatsLine(char *line) {
386 if (_table == nullptr) {
387 return;
388 }
389
390 // first token is either aggregation operator or column name
391 char *col_or_op = next_field(&line);
392 if (col_or_op == nullptr) {
393 _output->setError(RESPONSE_CODE_INVALID_HEADER, "empty stats line");
394 return;
395 }
396
397 int operation = STATS_OP_COUNT;
398 if (strcmp(col_or_op, "sum") == 0) {
399 operation = STATS_OP_SUM;
400 } else if (strcmp(col_or_op, "min") == 0) {
401 operation = STATS_OP_MIN;
402 } else if (strcmp(col_or_op, "max") == 0) {
403 operation = STATS_OP_MAX;
404 } else if (strcmp(col_or_op, "avg") == 0) {
405 operation = STATS_OP_AVG;
406 } else if (strcmp(col_or_op, "std") == 0) {
407 operation = STATS_OP_STD;
408 } else if (strcmp(col_or_op, "suminv") == 0) {
409 operation = STATS_OP_SUMINV;
410 } else if (strcmp(col_or_op, "avginv") == 0) {
411 operation = STATS_OP_AVGINV;
412 }
413
414 char *column_name;
415 if (operation == STATS_OP_COUNT) {
416 column_name = col_or_op;
417 } else {
418 // aggregation operator is followed by column name
419 column_name = next_field(&line);
420 if (column_name == nullptr) {
421 _output->setError(RESPONSE_CODE_INVALID_HEADER,
422 "missing column name in stats header");
423 return;
424 }
425 }
426
427 Column *column = _table->column(column_name);
428 if (column == nullptr) {
429 _output->setError(RESPONSE_CODE_INVALID_HEADER,
430 "invalid stats header: table '%s' has no column '%s'",
431 _table->name(), column_name);
432 return;
433 }
434
435 StatsColumn *stats_col;
436 if (operation == STATS_OP_COUNT) {
437 char *operator_name = next_field(&line);
438 if (operator_name == nullptr) {
439 _output->setError(
440 RESPONSE_CODE_INVALID_HEADER,
441 "invalid stats header: missing operator after table '%s'",
442 column_name);
443 return;
444 }
445 int operator_id = lookupOperator(operator_name);
446 if (operator_id == OP_INVALID) {
447 _output->setError(RESPONSE_CODE_INVALID_HEADER,
448 "invalid stats operator '%s'", operator_name);
449 return;
450 }
451 char *value = lstrip(line);
452 if (value == nullptr) {
453 _output->setError(
454 RESPONSE_CODE_INVALID_HEADER,
455 "invalid stats: missing value after operator '%s'",
456 operator_name);
457 return;
458 }
459
460 Filter *filter = createFilter(column, operator_id, value);
461 if (filter == nullptr) {
462 return;
463 }
464 stats_col = new StatsColumn(column, filter, operation);
465 } else {
466 stats_col = new StatsColumn(column, nullptr, operation);
467 }
468 _stats_columns.push_back(stats_col);
469
470 /* Default to old behaviour: do not output column headers if we
471 do Stats queries */
472 _show_column_headers = false;
473 }
474
parseFilterLine(char * line,bool is_filter)475 void Query::parseFilterLine(char *line, bool is_filter) {
476 if (_table == nullptr) {
477 return;
478 }
479
480 char *column_name = next_field(&line);
481 if (column_name == nullptr) {
482 _output->setError(RESPONSE_CODE_INVALID_HEADER, "empty filter line");
483 return;
484 }
485
486 Column *column = _table->column(column_name);
487 if (column == nullptr) {
488 _output->setError(RESPONSE_CODE_INVALID_HEADER,
489 "invalid filter: table '%s' has no column '%s'",
490 _table->name(), column_name);
491 return;
492 }
493
494 char *operator_name = next_field(&line);
495 if (operator_name == nullptr) {
496 _output->setError(
497 RESPONSE_CODE_INVALID_HEADER,
498 "invalid filter header: missing operator after table '%s'",
499 column_name);
500 return;
501 }
502 int operator_id = lookupOperator(operator_name);
503 if (operator_id == OP_INVALID) {
504 _output->setError(RESPONSE_CODE_INVALID_HEADER,
505 "invalid filter operator '%s'", operator_name);
506 return;
507 }
508 char *value = lstrip(line);
509 if (value == nullptr) {
510 _output->setError(RESPONSE_CODE_INVALID_HEADER,
511 "invalid filter: missing value after operator '%s'",
512 operator_name);
513 return;
514 }
515
516 Filter *filter = createFilter(column, operator_id, value);
517 if (filter != nullptr) {
518 if (is_filter) {
519 _filter.addSubfilter(filter);
520 } else {
521 _wait_condition.addSubfilter(filter);
522 }
523 }
524 }
525
parseAuthUserHeader(char * line)526 void Query::parseAuthUserHeader(char *line) {
527 if (_table == nullptr) {
528 return;
529 }
530 _auth_user = find_contact(line);
531 if (_auth_user == nullptr) {
532 // Do not handle this as error any more. In a multi site setup
533 // not all users might be present on all sites by design.
534 _auth_user = UNKNOWN_AUTH_USER;
535 // _output->setError(RESPONSE_CODE_UNAUTHORIZED, "AuthUser: no such user
536 // '%s'", line);
537 }
538 }
539
parseStatsGroupLine(char * line)540 void Query::parseStatsGroupLine(char *line) {
541 logger(LOG_WARNING,
542 "Warning: StatsGroupBy is deprecated. "
543 "Please use Columns instead.");
544 parseColumnsLine(line);
545 }
546
parseColumnsLine(char * line)547 void Query::parseColumnsLine(char *line) {
548 if (_table == nullptr) {
549 return;
550 }
551 char *column_name;
552 while (nullptr != (column_name = next_field(&line))) {
553 Column *column = _table->column(column_name);
554 if (column != nullptr) {
555 _columns.push_back(column);
556 } else {
557 logger(LOG_WARNING,
558 "Replacing non-existing column '%s' with null column",
559 column_name);
560 // Do not fail any longer. We might want to make this configurable.
561 // But not failing has the advantage that an updated GUI, that
562 // expects new columns,
563 // will be able to keep compatibility with older Livestatus
564 // versions.
565 // _output->setError(RESPONSE_CODE_INVALID_HEADER,
566 // "Table '%s' has no column '%s'", _table->name(),
567 // column_name);
568 Column *col = createDummyColumn(column_name);
569 _columns.push_back(col);
570 }
571 }
572 _show_column_headers = false;
573 }
574
parseSeparatorsLine(char * line)575 void Query::parseSeparatorsLine(char *line) {
576 char dssep = 0, fieldsep = 0, listsep = 0, hssep = 0;
577 char *token = next_field(&line);
578 if (token != nullptr) {
579 dssep = atoi(token);
580 }
581 token = next_field(&line);
582 if (token != nullptr) {
583 fieldsep = atoi(token);
584 }
585 token = next_field(&line);
586 if (token != nullptr) {
587 listsep = atoi(token);
588 }
589 token = next_field(&line);
590 if (token != nullptr) {
591 hssep = atoi(token);
592 }
593
594 // if (dssep == fieldsep
595 // || dssep == listsep
596 // || fieldsep == listsep
597 // || dssep == hssep
598 // || fieldsep == hssep
599 // || listsep == hssep)
600 // {
601 // _output->setError(RESPONSE_CODE_INVALID_HEADER, "invalid Separators:
602 // need four different integers");
603 // return;
604 // }
605 _dataset_separator = string(&dssep, 1);
606 _field_separator = string(&fieldsep, 1);
607 _list_separator = string(&listsep, 1);
608 _host_service_separator = string(&hssep, 1);
609 }
610
parseOutputFormatLine(char * line)611 void Query::parseOutputFormatLine(char *line) {
612 char *format = next_field(&line);
613 if (format == nullptr) {
614 _output->setError(
615 RESPONSE_CODE_INVALID_HEADER,
616 "Missing output format. Only 'csv' and 'json' are available.");
617 return;
618 }
619
620 if (strcmp(format, "csv") == 0) {
621 _output_format = OUTPUT_FORMAT_CSV;
622 } else if (strcmp(format, "json") == 0) {
623 _output_format = OUTPUT_FORMAT_JSON;
624 } else if (strcmp(format, "python") == 0) {
625 _output_format = OUTPUT_FORMAT_PYTHON;
626 } else {
627 _output->setError(
628 RESPONSE_CODE_INVALID_HEADER,
629 "Invalid output format. Only 'csv' and 'json' are available.");
630 }
631 }
632
parseColumnHeadersLine(char * line)633 void Query::parseColumnHeadersLine(char *line) {
634 char *value = next_field(&line);
635 if (value == nullptr) {
636 _output->setError(
637 RESPONSE_CODE_INVALID_HEADER,
638 "Missing value for ColumnHeaders: must be 'on' or 'off'");
639 return;
640 }
641
642 if (strcmp(value, "on") == 0) {
643 _show_column_headers = true;
644 } else if (strcmp(value, "off") == 0) {
645 _show_column_headers = false;
646 } else {
647 _output->setError(
648 RESPONSE_CODE_INVALID_HEADER,
649 "Invalid value for ColumnHeaders: must be 'on' or 'off'");
650 }
651 }
652
parseKeepAliveLine(char * line)653 void Query::parseKeepAliveLine(char *line) {
654 char *value = next_field(&line);
655 if (value == nullptr) {
656 _output->setError(RESPONSE_CODE_INVALID_HEADER,
657 "Missing value for KeepAlive: must be 'on' or 'off'");
658 return;
659 }
660
661 if (strcmp(value, "on") == 0) {
662 _output->setDoKeepalive(true);
663 } else if (strcmp(value, "off") == 0) {
664 _output->setDoKeepalive(false);
665 } else {
666 _output->setError(RESPONSE_CODE_INVALID_HEADER,
667 "Invalid value for KeepAlive: must be 'on' or 'off'");
668 }
669 }
670
parseResponseHeaderLine(char * line)671 void Query::parseResponseHeaderLine(char *line) {
672 char *value = next_field(&line);
673 if (value == nullptr) {
674 _output->setError(
675 RESPONSE_CODE_INVALID_HEADER,
676 "Missing value for ResponseHeader: must be 'off' or 'fixed16'");
677 return;
678 }
679
680 if (strcmp(value, "off") == 0) {
681 _output->setResponseHeader(RESPONSE_HEADER_OFF);
682 } else if (strcmp(value, "fixed16") == 0) {
683 _output->setResponseHeader(RESPONSE_HEADER_FIXED16);
684 } else {
685 _output->setError(
686 RESPONSE_CODE_INVALID_HEADER,
687 "Invalid value '%s' for ResponseHeader: must be 'off' or 'fixed16'",
688 value);
689 }
690 }
691
parseLimitLine(char * line)692 void Query::parseLimitLine(char *line) {
693 char *value = next_field(&line);
694 if (value == nullptr) {
695 _output->setError(RESPONSE_CODE_INVALID_HEADER,
696 "Header Limit: missing value");
697 } else {
698 int limit = atoi(value);
699 if ((isdigit(value[0]) == 0) || limit < 0) {
700 _output->setError(
701 RESPONSE_CODE_INVALID_HEADER,
702 "Invalid value for Limit: must be non-negative integer");
703 } else {
704 _limit = limit;
705 }
706 }
707 }
708
parseTimelimitLine(char * line)709 void Query::parseTimelimitLine(char *line) {
710 char *value = next_field(&line);
711 if (value == nullptr) {
712 _output->setError(RESPONSE_CODE_INVALID_HEADER,
713 "Header Timelimit: missing value");
714 } else {
715 int timelimit = atoi(value);
716 if ((isdigit(value[0]) == 0) || timelimit < 0) {
717 _output->setError(RESPONSE_CODE_INVALID_HEADER,
718 "Invalid value for Timelimit: must be "
719 "non-negative integer (seconds)");
720 } else {
721 _time_limit = timelimit;
722 _time_limit_timeout = time(nullptr) + _time_limit;
723 }
724 }
725 }
726
parseWaitTimeoutLine(char * line)727 void Query::parseWaitTimeoutLine(char *line) {
728 char *value = next_field(&line);
729 if (value == nullptr) {
730 _output->setError(RESPONSE_CODE_INVALID_HEADER,
731 "WaitTimeout: missing value");
732 } else {
733 int timeout = atoi(value);
734 if ((isdigit(value[0]) == 0) || timeout < 0) {
735 _output->setError(
736 RESPONSE_CODE_INVALID_HEADER,
737 "Invalid value for WaitTimeout: must be non-negative integer");
738 } else {
739 _wait_timeout = timeout;
740 }
741 }
742 }
743
parseWaitTriggerLine(char * line)744 void Query::parseWaitTriggerLine(char *line) {
745 char *value = next_field(&line);
746 if (value == nullptr) {
747 _output->setError(RESPONSE_CODE_INVALID_HEADER,
748 "WaitTrigger: missing keyword");
749 return;
750 }
751 struct trigger *t = trigger_find(value);
752 if (t == nullptr) {
753 _output->setError(RESPONSE_CODE_INVALID_HEADER,
754 "WaitTrigger: invalid trigger '%s'. Allowed are %s.",
755 value, trigger_all_names());
756 return;
757 }
758 _wait_trigger = t;
759 }
760
parseWaitObjectLine(char * line)761 void Query::parseWaitObjectLine(char *line) {
762 if (_table == nullptr) {
763 return;
764 }
765
766 char *objectspec = lstrip(line);
767 _wait_object = _table->findObject(objectspec);
768 if (_wait_object == nullptr) {
769 _output->setError(
770 RESPONSE_CODE_INVALID_HEADER,
771 "WaitObject: object '%s' not found or not supported by this table",
772 objectspec);
773 }
774 }
775
parseLocaltimeLine(char * line)776 void Query::parseLocaltimeLine(char *line) {
777 char *value = next_field(&line);
778 if (value == nullptr) {
779 _output->setError(RESPONSE_CODE_INVALID_HEADER,
780 "Header Localtime: missing value");
781 return;
782 }
783 time_t their_time = atoi(value);
784 time_t our_time = time(nullptr);
785
786 // compute offset to be *added* each time we output our time and
787 // *substracted* from reference value by filter headers
788 int dif = their_time - our_time;
789
790 // Round difference to half hour. We assume, that both clocks are more
791 // or less synchronized and that the time offset is only due to being
792 // in different time zones.
793 int full = dif / 1800;
794 int rem = dif % 1800;
795 if (rem <= -900) {
796 full--;
797 } else if (rem >= 900) {
798 full++;
799 }
800 if (full >= 48 || full <= -48) {
801 _output->setError(RESPONSE_CODE_INVALID_HEADER,
802 "Invalid Localtime header: timezone difference "
803 "greater then 24 hours");
804 return;
805 }
806 _timezone_offset = full * 1800;
807 if (g_debug_level >= 2) {
808 logger(LG_INFO, "Timezone difference is %.1f hours",
809 _timezone_offset / 3600.0);
810 }
811 }
812
doStats()813 bool Query::doStats() { return !_stats_columns.empty(); }
814
start()815 void Query::start() {
816 doWait();
817
818 _need_ds_separator = false;
819
820 if (_output_format != OUTPUT_FORMAT_CSV) {
821 _output->addChar('[');
822 }
823
824 if (doStats()) {
825 // if we have no StatsGroupBy: column, we allocate one only row of
826 // Aggregators,
827 // directly in _stats_aggregators. When grouping the rows of aggregators
828 // will be created each time a new group is found.
829 if (_columns.empty()) {
830 _stats_aggregators = new Aggregator *[_stats_columns.size()];
831 for (unsigned i = 0; i < _stats_columns.size(); i++) {
832 _stats_aggregators[i] = _stats_columns[i]->createAggregator();
833 }
834 }
835 }
836
837 if (_show_column_headers) {
838 outputDatasetBegin();
839 bool first = true;
840
841 for (const auto &column : _columns) {
842 if (first) {
843 first = false;
844 } else {
845 outputFieldSeparator();
846 }
847 outputString(column->name());
848 }
849
850 // Output dummy headers for stats columns
851 int col = 1;
852 for (const auto &stats_column : _stats_columns) {
853 (void)stats_column;
854 if (first) {
855 first = false;
856 } else {
857 outputFieldSeparator();
858 }
859 char colheader[32];
860 snprintf(colheader, 32, "stats_%d", col);
861 outputString(colheader);
862 col++;
863 }
864
865 outputDatasetEnd();
866 _need_ds_separator = true;
867 }
868 }
869
timelimitReached()870 bool Query::timelimitReached() {
871 if (_time_limit >= 0 && time(nullptr) >= _time_limit_timeout) {
872 logger(LG_INFO, "Maximum query time of %d seconds exceeded!",
873 _time_limit);
874 _output->setError(RESPONSE_CODE_LIMIT_EXCEEDED,
875 "Maximum query time of %d seconds exceeded!",
876 _time_limit);
877 return true;
878 }
879 return false;
880 }
881
processDataset(void * data)882 bool Query::processDataset(void *data) {
883 if (_output->size() > g_max_response_size) {
884 logger(LG_INFO, "Maximum response size of %lu bytes exceeded!",
885 g_max_response_size);
886 // _output->setError(RESPONSE_CODE_LIMIT_EXCEEDED, "Maximum response
887 // size of %d reached", g_max_response_size);
888 // currently we only log an error into the log file and do
889 // not abort the query. We handle it like Limit:
890 return false;
891 }
892
893 if (_filter.accepts(data) &&
894 ((_auth_user == nullptr) || _table->isAuthorized(_auth_user, data))) {
895 _current_line++;
896 if (_limit >= 0 && static_cast<int>(_current_line) > _limit) {
897 return false;
898 }
899
900 // When we reach the time limit we let the query fail. Otherwise the
901 // user will
902 // not know that the answer is incomplete.
903 if (timelimitReached()) {
904 return false;
905 }
906
907 if (doStats()) {
908 Aggregator **aggr;
909 // When doing grouped stats, we need to fetch/create a row
910 // of aggregators for the current group
911 if (!_columns.empty()) {
912 _stats_group_spec_t groupspec;
913 computeStatsGroupSpec(groupspec, data);
914 aggr = getStatsGroup(groupspec);
915 } else {
916 aggr = _stats_aggregators;
917 }
918
919 for (unsigned i = 0; i < _stats_columns.size(); i++) {
920 aggr[i]->consume(data, this);
921 }
922
923 // No output is done while processing the data, we only
924 // collect stats.
925 } else {
926 // output data of current row
927 if (_need_ds_separator && _output_format != OUTPUT_FORMAT_CSV) {
928 _output->addBuffer(",\n", 2);
929 } else {
930 _need_ds_separator = true;
931 }
932
933 outputDatasetBegin();
934 bool first = true;
935 for (auto column : _columns) {
936 if (first) {
937 first = false;
938 } else {
939 outputFieldSeparator();
940 }
941 column->output(data, this);
942 }
943 outputDatasetEnd();
944 }
945 }
946 return true;
947 }
948
finish()949 void Query::finish() {
950 // grouped stats
951 if (doStats() && !_columns.empty()) {
952 // output values of all stats groups (output has been post poned until
953 // now)
954 for (auto &stats_group : _stats_groups) {
955 if (_need_ds_separator && _output_format != OUTPUT_FORMAT_CSV) {
956 _output->addBuffer(",\n", 2);
957 } else {
958 _need_ds_separator = true;
959 }
960
961 outputDatasetBegin();
962
963 // output group columns first
964 _stats_group_spec_t groupspec = stats_group.first;
965 bool first = true;
966 for (auto &iit : groupspec) {
967 if (!first) {
968 outputFieldSeparator();
969 } else {
970 first = false;
971 }
972 outputString(iit.c_str());
973 }
974
975 Aggregator **aggr = stats_group.second;
976 for (unsigned i = 0; i < _stats_columns.size(); i++) {
977 outputFieldSeparator();
978 aggr[i]->output(this);
979 delete aggr[i]; // not needed any more
980 }
981 outputDatasetEnd();
982 delete[] aggr;
983 }
984 }
985
986 // stats without group column
987 else if (doStats()) {
988 if (_need_ds_separator && _output_format != OUTPUT_FORMAT_CSV) {
989 _output->addBuffer(",\n", 2);
990 } else {
991 _need_ds_separator = true;
992 }
993
994 outputDatasetBegin();
995 for (unsigned i = 0; i < _stats_columns.size(); i++) {
996 if (i > 0) {
997 outputFieldSeparator();
998 }
999 _stats_aggregators[i]->output(this);
1000 delete _stats_aggregators[i];
1001 }
1002 outputDatasetEnd();
1003 delete[] _stats_aggregators;
1004 }
1005
1006 // normal query
1007 if (_output_format != OUTPUT_FORMAT_CSV) {
1008 _output->addBuffer("]\n", 2);
1009 }
1010 }
1011
findIndexFilter(const char * columnname)1012 void *Query::findIndexFilter(const char *columnname) {
1013 return _filter.findIndexFilter(columnname);
1014 }
1015
findIntLimits(const char * columnname,int * lower,int * upper)1016 void Query::findIntLimits(const char *columnname, int *lower, int *upper) {
1017 return _filter.findIntLimits(columnname, lower, upper);
1018 }
1019
optimizeBitmask(const char * columnname,uint32_t * bitmask)1020 void Query::optimizeBitmask(const char *columnname, uint32_t *bitmask) {
1021 _filter.optimizeBitmask(columnname, bitmask);
1022 }
1023
1024 // output helpers, called from columns
outputDatasetBegin()1025 void Query::outputDatasetBegin() {
1026 if (_output_format != OUTPUT_FORMAT_CSV) {
1027 _output->addChar('[');
1028 }
1029 }
1030
outputDatasetEnd()1031 void Query::outputDatasetEnd() {
1032 if (_output_format == OUTPUT_FORMAT_CSV) {
1033 _output->addBuffer(_dataset_separator.c_str(),
1034 _dataset_separator.size());
1035 } else {
1036 _output->addChar(']');
1037 }
1038 }
1039
outputFieldSeparator()1040 void Query::outputFieldSeparator() {
1041 if (_output_format == OUTPUT_FORMAT_CSV) {
1042 _output->addBuffer(_field_separator.c_str(), _field_separator.size());
1043 } else {
1044 _output->addChar(',');
1045 }
1046 }
1047
outputInteger(int32_t value)1048 void Query::outputInteger(int32_t value) {
1049 char buf[32];
1050 int l = snprintf(buf, 32, "%d", value);
1051 _output->addBuffer(buf, l);
1052 }
1053
outputInteger64(int64_t value)1054 void Query::outputInteger64(int64_t value) {
1055 char buf[32];
1056 int l = snprintf(buf, 32, "%lld", static_cast<long long int>(value));
1057 _output->addBuffer(buf, l);
1058 }
1059
outputTime(int32_t value)1060 void Query::outputTime(int32_t value) {
1061 value += _timezone_offset;
1062 outputInteger(value);
1063 }
1064
outputUnsignedLong(unsigned long value)1065 void Query::outputUnsignedLong(unsigned long value) {
1066 char buf[64];
1067 int l = snprintf(buf, sizeof(buf), "%lu", value);
1068 _output->addBuffer(buf, l);
1069 }
1070
outputCounter(counter_t value)1071 void Query::outputCounter(counter_t value) {
1072 char buf[64];
1073 int l = snprintf(buf, sizeof(buf), "%llu",
1074 static_cast<unsigned long long>(value));
1075 _output->addBuffer(buf, l);
1076 }
1077
outputDouble(double value)1078 void Query::outputDouble(double value) {
1079 if (isnan(value)) {
1080 outputNull();
1081 } else {
1082 char buf[64];
1083 int l = snprintf(buf, sizeof(buf), "%.10e", value);
1084 _output->addBuffer(buf, l);
1085 }
1086 }
1087
outputNull()1088 void Query::outputNull() {
1089 if (_output_format == OUTPUT_FORMAT_CSV) {
1090 // output empty cell
1091 } else if (_output_format == OUTPUT_FORMAT_PYTHON) {
1092 _output->addBuffer("None", 4);
1093 } else {
1094 _output->addBuffer("null", 4); // JSON
1095 }
1096 }
1097
outputAsciiEscape(char value)1098 void Query::outputAsciiEscape(char value) {
1099 char buf[8];
1100 snprintf(buf, sizeof(buf), "\\%03o", value);
1101 _output->addBuffer(buf, 4);
1102 }
1103
outputUnicodeEscape(unsigned value)1104 void Query::outputUnicodeEscape(unsigned value) {
1105 char buf[8];
1106 snprintf(buf, sizeof(buf), "\\u%04x", value);
1107 _output->addBuffer(buf, 6);
1108 }
1109
outputBlob(const char * buffer,int size)1110 void Query::outputBlob(const char *buffer, int size) {
1111 if (_output_format != OUTPUT_FORMAT_CSV) {
1112 outputString(buffer, size);
1113 } else {
1114 _output->addBuffer(buffer, size);
1115 }
1116 }
1117
1118 // len = -1 -> use strlen(), len >= 0: consider
1119 // output as blob, do not handle UTF-8.
outputString(const char * value,int len)1120 void Query::outputString(const char *value, int len) {
1121 if (value == nullptr) {
1122 if (_output_format != OUTPUT_FORMAT_CSV) {
1123 _output->addBuffer("\"\"", 2);
1124 }
1125 return;
1126 }
1127
1128 if (_output_format == OUTPUT_FORMAT_CSV) {
1129 _output->addString(value);
1130
1131 } else // JSON
1132 {
1133 if (_output_format == OUTPUT_FORMAT_PYTHON && len < 0) {
1134 _output->addChar('u'); // mark strings as unicode
1135 }
1136 _output->addChar('"');
1137 const char *r = value;
1138 int chars_left = len >= 0 ? len : strlen(r);
1139 while (*r != 0) {
1140 // Always escape control characters (1..31)
1141 if (*r < 32 && *r >= 0) {
1142 if (len < 0) {
1143 outputUnicodeEscape(static_cast<unsigned>(*r));
1144 } else {
1145 outputAsciiEscape(*r);
1146 }
1147 }
1148
1149 // Output ASCII characters unencoded
1150 else if (*r >= 32 || len >= 0) {
1151 if (*r == '"' || *r == '\\') {
1152 _output->addChar('\\');
1153 }
1154 _output->addChar(*r);
1155 }
1156
1157 // interprete two-Byte UTF-8 sequences in mode 'utf8' and 'mixed'
1158 else if ((g_data_encoding == ENCODING_UTF8 ||
1159 g_data_encoding == ENCODING_MIXED) &&
1160 ((*r & 0xE0) == 0xC0)) {
1161 outputUnicodeEscape(((*r & 31) << 6) |
1162 (*(r + 1) & 0x3F)); // 2 byte encoding
1163 r++;
1164 chars_left--;
1165 }
1166
1167 // interprete 3/4-Byte UTF-8 sequences only in mode 'utf8'
1168 else if (g_data_encoding == ENCODING_UTF8) {
1169 // three-byte sequences (avoid buffer overflow!)
1170 if ((*r & 0xF0) == 0xE0) {
1171 if (chars_left < 3) {
1172 if (g_debug_level >= 2) {
1173 logger(LG_INFO,
1174 "Ignoring invalid UTF-8 sequence in string "
1175 "'%s'",
1176 value);
1177 }
1178 break; // end of string. No use in continuing
1179 } else {
1180 outputUnicodeEscape(((*r & 0x0F) << 12 |
1181 (*(r + 1) & 0x3F) << 6 |
1182 (*(r + 2) & 0x3F)));
1183 r += 2;
1184 chars_left -= 2;
1185 }
1186 }
1187 // four-byte sequences
1188 else if ((*r & 0xF8) == 0xF0) {
1189 if (chars_left < 4) {
1190 if (g_debug_level >= 2) {
1191 logger(LG_INFO,
1192 "Ignoring invalid UTF-8 sequence in string "
1193 "'%s'",
1194 value);
1195 }
1196 break; // end of string. No use in continuing
1197 } else {
1198 outputUnicodeEscape(
1199 ((*r & 0x07) << 18 | (*(r + 1) & 0x3F) << 6 |
1200 (*(r + 2) & 0x3F) << 6 | (*(r + 3) & 0x3F)));
1201 r += 3;
1202 chars_left -= 3;
1203 }
1204 } else {
1205 if (g_debug_level >= 2) {
1206 logger(LG_INFO,
1207 "Ignoring invalid UTF-8 sequence in string '%s'",
1208 value);
1209 }
1210 }
1211 }
1212
1213 // in latin1 and mixed mode interprete all other non-ASCII
1214 // characters as latin1
1215 else {
1216 outputUnicodeEscape(static_cast<unsigned>(
1217 static_cast<int>(*r) + 256)); // assume latin1 encoding
1218 }
1219
1220 r++;
1221 chars_left--;
1222 }
1223 _output->addChar('"');
1224 }
1225 }
1226
outputBeginList()1227 void Query::outputBeginList() {
1228 if (_output_format != OUTPUT_FORMAT_CSV) {
1229 _output->addChar('[');
1230 }
1231 }
1232
outputListSeparator()1233 void Query::outputListSeparator() {
1234 if (_output_format == OUTPUT_FORMAT_CSV) {
1235 _output->addBuffer(_list_separator.c_str(), _list_separator.size());
1236 } else {
1237 _output->addChar(',');
1238 }
1239 }
1240
outputEndList()1241 void Query::outputEndList() {
1242 if (_output_format != OUTPUT_FORMAT_CSV) {
1243 _output->addChar(']');
1244 }
1245 }
1246
outputBeginSublist()1247 void Query::outputBeginSublist() {
1248 if (_output_format != OUTPUT_FORMAT_CSV) {
1249 _output->addChar('[');
1250 }
1251 }
1252
outputSublistSeparator()1253 void Query::outputSublistSeparator() {
1254 if (_output_format == OUTPUT_FORMAT_CSV) {
1255 _output->addBuffer(_host_service_separator.c_str(),
1256 _host_service_separator.size());
1257 } else {
1258 _output->addChar(',');
1259 }
1260 }
1261
outputEndSublist()1262 void Query::outputEndSublist() {
1263 if (_output_format != OUTPUT_FORMAT_CSV) {
1264 _output->addChar(']');
1265 }
1266 }
1267
outputBeginDict()1268 void Query::outputBeginDict() {
1269 if (_output_format != OUTPUT_FORMAT_CSV) {
1270 _output->addChar('{');
1271 }
1272 }
1273
outputDictSeparator()1274 void Query::outputDictSeparator() { outputListSeparator(); }
1275
outputDictValueSeparator()1276 void Query::outputDictValueSeparator() {
1277 if (_output_format == OUTPUT_FORMAT_CSV) {
1278 _output->addBuffer(_host_service_separator.c_str(),
1279 _host_service_separator.size());
1280 } else {
1281 _output->addChar(':');
1282 }
1283 }
1284
outputEndDict()1285 void Query::outputEndDict() {
1286 if (_output_format != OUTPUT_FORMAT_CSV) {
1287 _output->addChar('}');
1288 }
1289 }
1290
getStatsGroup(Query::_stats_group_spec_t & groupspec)1291 Aggregator **Query::getStatsGroup(Query::_stats_group_spec_t &groupspec) {
1292 auto it = _stats_groups.find(groupspec);
1293 if (it == _stats_groups.end()) {
1294 auto aggr = new Aggregator *[_stats_columns.size()];
1295 for (unsigned i = 0; i < _stats_columns.size(); i++) {
1296 aggr[i] = _stats_columns[i]->createAggregator();
1297 }
1298 _stats_groups.insert(make_pair(groupspec, aggr));
1299 return aggr;
1300 }
1301 return it->second;
1302 }
1303
computeStatsGroupSpec(Query::_stats_group_spec_t & groupspec,void * data)1304 void Query::computeStatsGroupSpec(Query::_stats_group_spec_t &groupspec,
1305 void *data) {
1306 for (auto column : _columns) {
1307 groupspec.push_back(column->valueAsString(data, this));
1308 }
1309 }
1310
doWait()1311 void Query::doWait() {
1312 // If no wait condition and no trigger is set,
1313 // we do not wait at all.
1314 if (_wait_condition.numFilters() == 0 && _wait_trigger == nullptr) {
1315 return;
1316 }
1317
1318 // If a condition is set, we check the condition. If it
1319 // is already true, we do not need to way
1320 if (_wait_condition.numFilters() > 0 &&
1321 _wait_condition.accepts(_wait_object)) {
1322 if (g_debug_level >= 2) {
1323 logger(LG_INFO, "Wait condition true, no waiting neccessary");
1324 }
1325 return;
1326 }
1327
1328 // No wait on specified trigger. If no trigger was specified
1329 // we use WT_ALL as default trigger.
1330 if (_wait_trigger == nullptr) {
1331 _wait_trigger = trigger_all();
1332 }
1333
1334 struct timeval now;
1335 gettimeofday(&now, nullptr);
1336 struct timespec timeout;
1337 timeout.tv_sec = now.tv_sec + (_wait_timeout / 1000);
1338 timeout.tv_nsec = now.tv_usec * 1000 + 1000 * 1000 * (_wait_timeout % 1000);
1339 if (timeout.tv_nsec > 1000000000) {
1340 timeout.tv_sec++;
1341 timeout.tv_nsec -= 1000000000;
1342 }
1343
1344 do {
1345 if (_wait_timeout == 0) {
1346 if (g_debug_level >= 2) {
1347 logger(LG_INFO,
1348 "Waiting unlimited until condition becomes true");
1349 }
1350 trigger_wait(_wait_trigger);
1351 } else {
1352 if (g_debug_level >= 2) {
1353 logger(LG_INFO, "Waiting %d ms or until condition becomes true",
1354 _wait_timeout);
1355 }
1356 if (trigger_wait_until(_wait_trigger, &timeout) == 0) {
1357 if (g_debug_level >= 2) {
1358 logger(LG_INFO, "WaitTimeout after %d ms", _wait_timeout);
1359 }
1360 return; // timeout occurred. do not wait any longer
1361 }
1362 }
1363 } while (!_wait_condition.accepts(_wait_object));
1364 }
1365