1 /**
2 * Copyright (c) 2013-2016, Timothy Stack
3 *
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * * Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 * * Neither the name of Timothy Stack nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
25 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * @file log_format_loader.cc
30 */
31
32 #include "config.h"
33
34 #include <errno.h>
35 #include <fcntl.h>
36 #include <glob.h>
37 #include <libgen.h>
38 #include <sys/stat.h>
39
40 #include <map>
41 #include <string>
42
43 #include "fmt/format.h"
44
45 #include "base/paths.hh"
46 #include "base/string_util.hh"
47 #include "yajlpp/yajlpp.hh"
48 #include "yajlpp/yajlpp_def.hh"
49 #include "lnav_config.hh"
50 #include "log_format_ext.hh"
51 #include "auto_fd.hh"
52 #include "sql_util.hh"
53 #include "lnav_util.hh"
54 #include "builtin-scripts.h"
55 #include "builtin-sh-scripts.h"
56 #include "default-formats.h"
57
58 #include "log_format_loader.hh"
59 #include "bin2c.hh"
60
61 using namespace std;
62
63 static void extract_metadata(const char *contents, size_t len, struct script_metadata &meta_out);
64
65 typedef map<intern_string_t, std::shared_ptr<external_log_format>> log_formats_map_t;
66
67 static auto intern_lifetime = intern_string::get_table_lifetime();
68 static log_formats_map_t LOG_FORMATS;
69
70 struct userdata {
71 ghc::filesystem::path ud_format_path;
72 vector<intern_string_t> *ud_format_names{nullptr};
73 std::vector<std::string> *ud_errors{nullptr};
74 };
75
ensure_format(const yajlpp_provider_context & ypc,userdata * ud)76 static external_log_format *ensure_format(const yajlpp_provider_context &ypc, userdata *ud)
77 {
78 const intern_string_t name = ypc.get_substr_i(0);
79 vector<intern_string_t> *formats = ud->ud_format_names;
80 external_log_format *retval;
81
82 retval = LOG_FORMATS[name].get();
83 if (retval == nullptr) {
84 LOG_FORMATS[name] = std::make_shared<external_log_format>(name);
85 retval = LOG_FORMATS[name].get();
86 log_debug("Loading format -- %s", name.get());
87 }
88 retval->elf_source_path.insert(ud->ud_format_path.filename().string());
89
90 if (find(formats->begin(), formats->end(), name) == formats->end()) {
91 formats->push_back(name);
92 }
93
94 if (ud->ud_format_path.empty()) {
95 retval->elf_builtin_format = true;
96 }
97
98 return retval;
99 }
100
pattern_provider(const yajlpp_provider_context & ypc,external_log_format * elf)101 static external_log_format::pattern *pattern_provider(const yajlpp_provider_context &ypc, external_log_format *elf)
102 {
103 string regex_name = ypc.get_substr(0);
104 auto &pat = elf->elf_patterns[regex_name];
105
106 if (pat.get() == nullptr) {
107 pat = make_shared<external_log_format::pattern>();
108 }
109
110 if (pat->p_config_path.empty()) {
111 pat->p_config_path = elf->get_name().to_string() + "/regex/" + regex_name;
112 }
113
114 return pat.get();
115 }
116
value_def_provider(const yajlpp_provider_context & ypc,external_log_format * elf)117 static external_log_format::value_def *value_def_provider(const yajlpp_provider_context &ypc, external_log_format *elf)
118 {
119 const intern_string_t value_name = ypc.get_substr_i(0);
120
121 auto iter = elf->elf_value_defs.find(value_name);
122 shared_ptr<external_log_format::value_def> retval;
123
124 if (iter == elf->elf_value_defs.end()) {
125 retval = make_shared<external_log_format::value_def>(
126 value_name, value_kind_t::VALUE_TEXT, -1, elf);
127 elf->elf_value_defs[value_name] = retval;
128 elf->elf_value_def_order.emplace_back(retval);
129 } else {
130 retval = iter->second;
131 }
132
133 return retval.get();
134 }
135
scaling_factor_provider(const yajlpp_provider_context & ypc,external_log_format::value_def * value_def)136 static scaling_factor *scaling_factor_provider(const yajlpp_provider_context &ypc, external_log_format::value_def *value_def)
137 {
138 auto scale_name = ypc.get_substr_i(0);
139 scaling_factor &retval = value_def->vd_unit_scaling[scale_name];
140
141 return &retval;
142 }
143
144 static external_log_format::json_format_element &
ensure_json_format_element(external_log_format * elf,int index)145 ensure_json_format_element(external_log_format *elf, int index)
146 {
147 elf->jlf_line_format.resize(index + 1);
148
149 return elf->jlf_line_format[index];
150 }
151
line_format_provider(const yajlpp_provider_context & ypc,external_log_format * elf)152 static external_log_format::json_format_element *line_format_provider(
153 const yajlpp_provider_context &ypc, external_log_format *elf)
154 {
155 auto &jfe = ensure_json_format_element(elf, ypc.ypc_index);
156
157 jfe.jfe_type = external_log_format::JLF_VARIABLE;
158
159 return &jfe;
160 }
161
read_format_bool(yajlpp_parse_context * ypc,int val)162 static int read_format_bool(yajlpp_parse_context *ypc, int val)
163 {
164 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
165 string field_name = ypc->get_path_fragment(1);
166
167 if (field_name == "convert-to-local-time")
168 elf->lf_date_time.dts_local_time = val;
169 else if (field_name == "json") {
170 if (val) {
171 elf->elf_type = external_log_format::ELF_TYPE_JSON;
172 }
173 }
174 else if (field_name == "hide-extra")
175 elf->jlf_hide_extra = val;
176 else if (field_name == "multiline")
177 elf->elf_multiline = val;
178
179 return 1;
180 }
181
read_format_double(yajlpp_parse_context * ypc,double val)182 static int read_format_double(yajlpp_parse_context *ypc, double val)
183 {
184 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
185 string field_name = ypc->get_path_fragment(1);
186
187 if (field_name == "timestamp-divisor") {
188 if (val <= 0) {
189 fprintf(stderr, "error:%s: timestamp-divisor cannot be less "
190 "than or equal to zero\n",
191 ypc->get_path_fragment(0).c_str());
192 return 0;
193 }
194 elf->elf_timestamp_divisor = val;
195 }
196
197 return 1;
198 }
199
read_format_int(yajlpp_parse_context * ypc,long long val)200 static int read_format_int(yajlpp_parse_context *ypc, long long val)
201 {
202 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
203 string field_name = ypc->get_path_fragment(1);
204
205 if (field_name == "timestamp-divisor") {
206 if (val <= 0) {
207 fprintf(stderr, "error:%s: timestamp-divisor cannot be less "
208 "than or equal to zero\n",
209 ypc->get_path_fragment(0).c_str());
210 return 0;
211 }
212 elf->elf_timestamp_divisor = val;
213 }
214
215 return 1;
216 }
217
read_format_field(yajlpp_parse_context * ypc,const unsigned char * str,size_t len)218 static int read_format_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
219 {
220 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
221 auto leading_slash = len > 0 && str[0] == '/';
222 auto value = string((const char *) (leading_slash ? str + 1 : str),
223 leading_slash ? len - 1 : len);
224 auto field_name = ypc->get_path_fragment(1);
225
226 if (field_name == "file-pattern") {
227 elf->elf_file_pattern = value;
228 }
229 else if (field_name == "level-field") {
230 elf->elf_level_field = intern_string::lookup(value);
231 }
232 else if (field_name == "timestamp-field") {
233 elf->lf_timestamp_field = intern_string::lookup(value);
234 }
235 else if (field_name == "body-field") {
236 elf->elf_body_field = intern_string::lookup(value);
237 }
238 else if (field_name == "timestamp-format") {
239 elf->lf_timestamp_format.push_back(intern_string::lookup(value)->get());
240 }
241 else if (field_name == "module-field") {
242 elf->elf_module_id_field = intern_string::lookup(value);
243 elf->elf_container = true;
244 }
245 else if (field_name == "opid-field") {
246 elf->elf_opid_field = intern_string::lookup(value);
247 }
248
249 return 1;
250 }
251
read_levels(yajlpp_parse_context * ypc,const unsigned char * str,size_t len)252 static int read_levels(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
253 {
254 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
255 string regex = string((const char *)str, len);
256 string level_name_or_number = ypc->get_path_fragment(2);
257 log_level_t level = string2level(level_name_or_number.c_str());
258 elf->elf_level_patterns[level].lp_regex = regex;
259
260 return 1;
261 }
262
read_level_int(yajlpp_parse_context * ypc,long long val)263 static int read_level_int(yajlpp_parse_context *ypc, long long val)
264 {
265 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
266 string level_name_or_number = ypc->get_path_fragment(2);
267 log_level_t level = string2level(level_name_or_number.c_str());
268
269 elf->elf_level_pairs.emplace_back(val, level);
270
271 return 1;
272 }
273
read_action_def(yajlpp_parse_context * ypc,const unsigned char * str,size_t len)274 static int read_action_def(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
275 {
276 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
277 string action_name = ypc->get_path_fragment(2);
278 string field_name = ypc->get_path_fragment(3);
279 string val = string((const char *)str, len);
280
281 elf->lf_action_defs[action_name].ad_name = action_name;
282 if (field_name == "label")
283 elf->lf_action_defs[action_name].ad_label = val;
284
285 return 1;
286 }
287
read_action_bool(yajlpp_parse_context * ypc,int val)288 static int read_action_bool(yajlpp_parse_context *ypc, int val)
289 {
290 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
291 string action_name = ypc->get_path_fragment(2);
292 string field_name = ypc->get_path_fragment(3);
293
294 elf->lf_action_defs[action_name].ad_capture_output = val;
295
296 return 1;
297 }
298
read_action_cmd(yajlpp_parse_context * ypc,const unsigned char * str,size_t len)299 static int read_action_cmd(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
300 {
301 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
302 string action_name = ypc->get_path_fragment(2);
303 string field_name = ypc->get_path_fragment(3);
304 string val = string((const char *)str, len);
305
306 elf->lf_action_defs[action_name].ad_name = action_name;
307 elf->lf_action_defs[action_name].ad_cmdline.push_back(val);
308
309 return 1;
310 }
311
ensure_sample(external_log_format * elf,int index)312 static external_log_format::sample &ensure_sample(external_log_format *elf,
313 int index)
314 {
315 elf->elf_samples.resize(index + 1);
316
317 return elf->elf_samples[index];
318 }
319
sample_provider(const yajlpp_provider_context & ypc,external_log_format * elf)320 static external_log_format::sample *sample_provider(const yajlpp_provider_context &ypc, external_log_format *elf)
321 {
322 external_log_format::sample &sample = ensure_sample(elf, ypc.ypc_index);
323
324 return &sample;
325 }
326
read_json_constant(yajlpp_parse_context * ypc,const unsigned char * str,size_t len)327 static int read_json_constant(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
328 {
329 auto val = string((const char *) str, len);
330 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
331
332 ypc->ypc_array_index.back() += 1;
333 auto &jfe = ensure_json_format_element(elf, ypc->ypc_array_index.back());
334 jfe.jfe_type = external_log_format::JLF_CONSTANT;
335 jfe.jfe_default_value = val;
336
337 return 1;
338 }
339
create_search_table(yajlpp_parse_context * ypc,const unsigned char * str,size_t len)340 static int create_search_table(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
341 {
342 auto elf = (external_log_format *) ypc->ypc_obj_stack.top();
343 const intern_string_t table_name = ypc->get_path_fragment_i(2);
344 string regex = string((const char *) str, len);
345
346 elf->elf_search_tables.emplace_back(table_name, regex);
347
348 return 1;
349 }
350
351
352 static struct json_path_container pattern_handlers = {
353 yajlpp::property_handler("pattern")
354 .with_synopsis("<message-regex>")
355 .with_description(
356 "The regular expression to match a log message and capture fields.")
357 .with_min_length(1)
358 .FOR_FIELD(external_log_format::pattern, p_string),
359 yajlpp::property_handler("module-format")
360 .with_synopsis("<bool>")
361 .with_description(
362 "If true, this pattern will only be used to parse message bodies "
363 "of container formats, like syslog")
364 .for_field(&external_log_format::pattern::p_module_format)
365 };
366
367 static const json_path_handler_base::enum_value_t ALIGN_ENUM[] = {
368 { "left", external_log_format::json_format_element::align_t::LEFT },
369 { "right", external_log_format::json_format_element::align_t::RIGHT },
370
371 json_path_handler_base::ENUM_TERMINATOR
372 };
373
374 static const json_path_handler_base::enum_value_t OVERFLOW_ENUM[] = {
375 { "abbrev", external_log_format::json_format_element::overflow_t::ABBREV },
376 { "truncate", external_log_format::json_format_element::overflow_t::TRUNCATE },
377 { "dot-dot", external_log_format::json_format_element::overflow_t::DOTDOT },
378
379 json_path_handler_base::ENUM_TERMINATOR
380 };
381
382 static const json_path_handler_base::enum_value_t TRANSFORM_ENUM[] = {
383 { "none", external_log_format::json_format_element::transform_t::NONE },
384 { "uppercase", external_log_format::json_format_element::transform_t::UPPERCASE },
385 { "lowercase", external_log_format::json_format_element::transform_t::LOWERCASE },
386 { "capitalize", external_log_format::json_format_element::transform_t::CAPITALIZE },
387
388 json_path_handler_base::ENUM_TERMINATOR
389 };
390
391 static struct json_path_container line_format_handlers = {
392 yajlpp::property_handler("field")
393 .with_synopsis("<field-name>")
394 .with_description("The name of the field to substitute at this position")
395 .with_min_length(1)
396 .FOR_FIELD(external_log_format::json_format_element, jfe_value),
397
398 yajlpp::property_handler("default-value")
399 .with_synopsis("<string>")
400 .with_description("The default value for this position if the field is null")
401 .FOR_FIELD(external_log_format::json_format_element, jfe_default_value),
402
403 yajlpp::property_handler("timestamp-format")
404 .with_synopsis("<string>")
405 .with_min_length(1)
406 .with_description("The strftime(3) format for this field")
407 .FOR_FIELD(external_log_format::json_format_element, jfe_ts_format),
408
409 yajlpp::property_handler("min-width")
410 .with_min_value(0)
411 .with_synopsis("<size>")
412 .with_description("The minimum width of the field")
413 .FOR_FIELD(external_log_format::json_format_element, jfe_min_width),
414
415 yajlpp::property_handler("max-width")
416 .with_min_value(0)
417 .with_synopsis("<size>")
418 .with_description("The maximum width of the field")
419 .FOR_FIELD(external_log_format::json_format_element, jfe_max_width),
420
421 yajlpp::property_handler("align")
422 .with_synopsis("left|right")
423 .with_description("Align the text in the column to the left or right side")
424 .with_enum_values(ALIGN_ENUM)
425 .FOR_FIELD(external_log_format::json_format_element, jfe_align),
426
427 yajlpp::property_handler("overflow")
428 .with_synopsis("abbrev|truncate|dot-dot")
429 .with_description("Overflow style")
430 .with_enum_values(OVERFLOW_ENUM)
431 .FOR_FIELD(external_log_format::json_format_element, jfe_overflow),
432
433 yajlpp::property_handler("text-transform")
434 .with_synopsis("none|uppercase|lowercase|capitalize")
435 .with_description("Text transformation")
436 .with_enum_values(TRANSFORM_ENUM)
437 .FOR_FIELD(external_log_format::json_format_element, jfe_text_transform)
438 };
439
440 static const json_path_handler_base::enum_value_t KIND_ENUM[] = {
441 {"string", value_kind_t::VALUE_TEXT},
442 {"integer", value_kind_t::VALUE_INTEGER},
443 {"float", value_kind_t::VALUE_FLOAT},
444 {"boolean", value_kind_t::VALUE_BOOLEAN},
445 {"json", value_kind_t::VALUE_JSON},
446 {"struct", value_kind_t::VALUE_STRUCT},
447 {"quoted", value_kind_t::VALUE_QUOTED},
448 {"xml", value_kind_t::VALUE_XML},
449
450 json_path_handler_base::ENUM_TERMINATOR
451 };
452
453 static const json_path_handler_base::enum_value_t SCALE_OP_ENUM[] = {
454 {"identity", scale_op_t::SO_IDENTITY},
455 {"multiply", scale_op_t::SO_MULTIPLY},
456 {"divide", scale_op_t::SO_DIVIDE},
457
458 json_path_handler_base::ENUM_TERMINATOR
459 };
460
461 static struct json_path_container scaling_factor_handlers = {
462 yajlpp::pattern_property_handler("op")
463 .with_enum_values(SCALE_OP_ENUM)
464 .FOR_FIELD(scaling_factor, sf_op),
465
466 yajlpp::pattern_property_handler("value")
467 .FOR_FIELD(scaling_factor, sf_value)
468 };
469
470 static struct json_path_container scale_handlers = {
471 yajlpp::pattern_property_handler("(?<scale>[^/]+)")
472 .with_obj_provider(scaling_factor_provider)
473 .with_children(scaling_factor_handlers),
474 };
475
476 static struct json_path_container unit_handlers = {
477 yajlpp::property_handler("field")
478 .with_synopsis("<field-name>")
479 .with_description("The name of the field that contains the units for this field")
480 .FOR_FIELD(external_log_format::value_def, vd_unit_field),
481
482 yajlpp::property_handler("scaling-factor")
483 .with_description("Transforms the numeric value by the given factor")
484 .with_children(scale_handlers),
485 };
486
487 static struct json_path_container value_def_handlers = {
488 yajlpp::property_handler("kind")
489 .with_synopsis("string|integer|float|boolean|json|quoted")
490 .with_description("The type of data in the field")
491 .with_enum_values(KIND_ENUM)
492 .for_field(&external_log_format::value_def::vd_meta,
493 &logline_value_meta::lvm_kind),
494
495 yajlpp::property_handler("collate")
496 .with_synopsis("<function>")
497 .with_description("The collating function to use for this column")
498 .FOR_FIELD(external_log_format::value_def, vd_collate),
499
500 yajlpp::property_handler("unit")
501 .with_description("Unit definitions for this field")
502 .with_children(unit_handlers),
503
504 yajlpp::property_handler("identifier")
505 .with_synopsis("<bool>")
506 .with_description("Indicates whether or not this field contains an identifier that should be highlighted")
507 .for_field(&external_log_format::value_def::vd_meta,
508 &logline_value_meta::lvm_identifier),
509
510 yajlpp::property_handler("foreign-key")
511 .with_synopsis("<bool>")
512 .with_description("Indicates whether or not this field should be treated as a foreign key for row in another table")
513 .for_field(&external_log_format::value_def::vd_foreign_key),
514
515 yajlpp::property_handler("hidden")
516 .with_synopsis("<bool>")
517 .with_description("Indicates whether or not this field should be hidden")
518 .for_field(&external_log_format::value_def::vd_meta,
519 &logline_value_meta::lvm_hidden),
520
521 yajlpp::property_handler("action-list#")
522 .with_synopsis("<string>")
523 .with_description("Actions to execute when this field is clicked on")
524 .FOR_FIELD(external_log_format::value_def, vd_action_list),
525
526 yajlpp::property_handler("rewriter")
527 .with_synopsis("<command>")
528 .with_description("A command that will rewrite this field when pretty-printing")
529 .FOR_FIELD(external_log_format::value_def, vd_rewriter),
530
531 yajlpp::property_handler("description")
532 .with_synopsis("<string>")
533 .with_description("A description of the field")
534 .FOR_FIELD(external_log_format::value_def, vd_description)
535 };
536
537 static struct json_path_container highlighter_def_handlers = {
538 yajlpp::property_handler("pattern")
539 .with_synopsis("<regex>")
540 .with_description("A regular expression to highlight in logs of this format.")
541 .FOR_FIELD(external_log_format::highlighter_def, hd_pattern),
542
543 yajlpp::property_handler("color")
544 .with_synopsis("#<hex>|<name>")
545 .with_description("The color to use when highlighting this pattern.")
546 .FOR_FIELD(external_log_format::highlighter_def, hd_color),
547
548 yajlpp::property_handler("background-color")
549 .with_synopsis("#<hex>|<name>")
550 .with_description("The background color to use when highlighting this pattern.")
551 .FOR_FIELD(external_log_format::highlighter_def, hd_background_color),
552
553 yajlpp::property_handler("underline")
554 .with_synopsis("<enabled>")
555 .with_description("Highlight this pattern with an underline.")
556 .for_field(&external_log_format::highlighter_def::hd_underline),
557
558 yajlpp::property_handler("blink")
559 .with_synopsis("<enabled>")
560 .with_description("Highlight this pattern by blinking.")
561 .for_field(&external_log_format::highlighter_def::hd_blink)
562 };
563
564 static const json_path_handler_base::enum_value_t LEVEL_ENUM[] = {
565 { level_names[LEVEL_TRACE], LEVEL_TRACE },
566 { level_names[LEVEL_DEBUG5], LEVEL_DEBUG5 },
567 { level_names[LEVEL_DEBUG4], LEVEL_DEBUG4 },
568 { level_names[LEVEL_DEBUG3], LEVEL_DEBUG3 },
569 { level_names[LEVEL_DEBUG2], LEVEL_DEBUG2 },
570 { level_names[LEVEL_DEBUG], LEVEL_DEBUG },
571 { level_names[LEVEL_INFO], LEVEL_INFO },
572 { level_names[LEVEL_STATS], LEVEL_STATS },
573 { level_names[LEVEL_NOTICE], LEVEL_NOTICE },
574 { level_names[LEVEL_WARNING], LEVEL_WARNING },
575 { level_names[LEVEL_ERROR], LEVEL_ERROR },
576 { level_names[LEVEL_CRITICAL], LEVEL_CRITICAL },
577 { level_names[LEVEL_FATAL], LEVEL_FATAL },
578
579 json_path_handler_base::ENUM_TERMINATOR
580 };
581
582 static struct json_path_container sample_handlers = {
583 yajlpp::property_handler("line")
584 .with_synopsis("<log-line>")
585 .with_description("A sample log line that should match a pattern in this format.")
586 .FOR_FIELD(external_log_format::sample, s_line),
587
588 yajlpp::property_handler("level")
589 .with_enum_values(LEVEL_ENUM)
590 .with_description("The expected level for this sample log line.")
591 .FOR_FIELD(external_log_format::sample, s_level)
592 };
593
594 static const json_path_handler_base::enum_value_t TYPE_ENUM[] = {
595 { "text", external_log_format::elf_type_t::ELF_TYPE_TEXT },
596 { "json", external_log_format::elf_type_t::ELF_TYPE_JSON },
597 { "csv", external_log_format::elf_type_t::ELF_TYPE_CSV },
598
599 json_path_handler_base::ENUM_TERMINATOR
600 };
601
602 static struct json_path_container regex_handlers = {
603 yajlpp::pattern_property_handler(R"((?<pattern_name>[^/]+))")
604 .with_description("The set of patterns used to match log messages")
605 .with_obj_provider(pattern_provider)
606 .with_children(pattern_handlers),
607 };
608
609 static struct json_path_container level_handlers = {
610 yajlpp::pattern_property_handler(
611 "(?<level>trace|debug[2345]?|info|stats|notice|warning|error|critical|fatal)")
612 .add_cb(read_levels)
613 .add_cb(read_level_int)
614 .with_synopsis("<pattern|integer>")
615 .with_description("The regular expression used to match the log text for this level. "
616 "For JSON logs with numeric levels, this should be the number for the corresponding level.")
617 };
618
619 static struct json_path_container value_handlers = {
620 yajlpp::pattern_property_handler("(?<value_name>[^/]+)")
621 .with_description("The set of values captured by the log message patterns")
622 .with_obj_provider(value_def_provider)
623 .with_children(value_def_handlers)
624 };
625
626 static struct json_path_container highlight_handlers = {
627 yajlpp::pattern_property_handler(R"((?<highlight_name>[^/]+))")
628 .with_description("The definition of a highlight")
__anond6026f9f0102(const yajlpp_provider_context &ypc, external_log_format *root) 629 .with_obj_provider<external_log_format::highlighter_def, external_log_format>([](const yajlpp_provider_context &ypc, external_log_format *root) {
630 return &(root->elf_highlighter_patterns[ypc.get_substr_i(0)]);
631 })
632 .with_children(highlighter_def_handlers)
633 };
634
635 static struct json_path_container action_def_handlers = {
636 json_path_handler("label", read_action_def),
637 json_path_handler("capture-output", read_action_bool),
638 json_path_handler("cmd#", read_action_cmd)
639 };
640
641 static struct json_path_container action_handlers = {
642 json_path_handler(pcrepp("(?<action_name>\\w+)"), read_action_def)
643 .with_children(action_def_handlers)
644 };
645
646 static struct json_path_container search_table_def_handlers = {
647 json_path_handler("pattern", create_search_table)
648 .with_synopsis("<regex>")
649 .with_description("The regular expression for this search table."),
650 };
651
652 static struct json_path_container search_table_handlers = {
653 yajlpp::pattern_property_handler("\\w+")
654 .with_description("The set of search tables to be automatically defined")
655 .with_children(search_table_def_handlers)
656 };
657
658 struct json_path_container format_handlers = {
659 yajlpp::property_handler("regex")
660 .with_description("The set of regular expressions used to match log messages")
661 .with_children(regex_handlers),
662
663 json_path_handler("json", read_format_bool)
664 .with_description(R"(Indicates that log files are JSON-encoded (deprecated, use "file-type": "json"))"),
665 json_path_handler("convert-to-local-time", read_format_bool)
666 .with_description("Indicates that displayed timestamps should automatically be converted to local time"),
667 json_path_handler("hide-extra", read_format_bool)
668 .with_description("Specifies whether extra values in JSON logs should be displayed"),
669 json_path_handler("multiline", read_format_bool)
670 .with_description("Indicates that log messages can span multiple lines"),
671 json_path_handler("timestamp-divisor", read_format_double)
672 .add_cb(read_format_int)
673 .with_synopsis("<number>")
674 .with_description("The value to divide a numeric timestamp by in a JSON log."),
675 json_path_handler("file-pattern", read_format_field)
676 .with_description("A regular expression that restricts this format to log files with a matching name"),
677 json_path_handler("level-field", read_format_field)
678 .with_description("The name of the level field in the log message pattern"),
679 json_path_handler("timestamp-field", read_format_field)
680 .with_description("The name of the timestamp field in the log message pattern"),
681 json_path_handler("body-field", read_format_field)
682 .with_description("The name of the body field in the log message pattern"),
683 json_path_handler("url", pcrepp("url#?"))
684 .add_cb(read_format_field)
685 .with_description("A URL with more information about this log format"),
686 json_path_handler("title", read_format_field)
687 .with_description("The human-readable name for this log format"),
688 json_path_handler("description", read_format_field)
689 .with_description("A longer description of this log format"),
690 json_path_handler("timestamp-format#", read_format_field)
691 .with_description("An array of strptime(3)-like timestamp formats"),
692 json_path_handler("module-field", read_format_field)
693 .with_description("The name of the module field in the log message pattern"),
694 json_path_handler("opid-field", read_format_field)
695 .with_description("The name of the operation-id field in the log message pattern"),
696 yajlpp::property_handler("ordered-by-time")
697 .with_synopsis("<bool>")
698 .with_description("Indicates that the order of messages in the file is time-based.")
699 .for_field(&log_format::lf_time_ordered),
700 yajlpp::property_handler("level")
701 .with_description("The map of level names to patterns or integer values")
702 .with_children(level_handlers),
703
704 yajlpp::property_handler("value")
705 .with_description("The set of value definitions")
706 .with_children(value_handlers),
707
708 yajlpp::property_handler("action")
709 .with_children(action_handlers),
710 yajlpp::property_handler("sample#")
711 .with_description("An array of sample log messages to be tested against the log message patterns")
712 .with_obj_provider(sample_provider)
713 .with_children(sample_handlers),
714
715 yajlpp::property_handler("line-format#")
716 .with_description("The display format for JSON-encoded log messages")
717 .with_obj_provider(line_format_provider)
718 .add_cb(read_json_constant)
719 .with_children(line_format_handlers),
720 json_path_handler("search-table", create_search_table)
721 .with_description("Search tables to automatically define for this log format")
722 .with_children(search_table_handlers),
723
724 yajlpp::property_handler("highlights")
725 .with_description("The set of highlight definitions")
726 .with_children(highlight_handlers),
727
728 yajlpp::property_handler("file-type")
729 .with_synopsis("text|json|csv")
730 .with_description("The type of file that contains the log messages")
731 .with_enum_values(TYPE_ENUM)
732 .FOR_FIELD(external_log_format, elf_type)
733 };
734
read_id(yajlpp_parse_context * ypc,const unsigned char * str,size_t len)735 static int read_id(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
736 {
737 auto file_id = string((const char *) str, len);
738
739 if (find(SUPPORTED_FORMAT_SCHEMAS.begin(),
740 SUPPORTED_FORMAT_SCHEMAS.end(),
741 file_id) == SUPPORTED_FORMAT_SCHEMAS.end()) {
742 fprintf(stderr, "%s:%d: error: unsupported format $schema -- %s\n",
743 ypc->ypc_source.c_str(), ypc->get_line_number(), file_id.c_str());
744 return 0;
745 }
746
747 return 1;
748 }
749
750 struct json_path_container root_format_handler = json_path_container {
751 json_path_handler("$schema", read_id)
752 .with_synopsis("The URI of the schema for this file")
753 .with_description("Specifies the type of this file"),
754
755 yajlpp::pattern_property_handler("(?<format_name>\\w+)")
756 .with_description("The definition of a log file format.")
757 .with_obj_provider(ensure_format)
758 .with_children(format_handlers)
759 }
760 .with_schema_id(DEFAULT_FORMAT_SCHEMA);
761
write_sample_file()762 static void write_sample_file()
763 {
764 for (const auto& bsf : lnav_format_json) {
765 auto sample_path = lnav::paths::dotlnav() /
766 fmt::format("formats/default/{}.sample", bsf.get_name());
767 auto sf = bsf.to_string_fragment();
768 auto_fd sample_fd;
769
770 if ((sample_fd = openp(sample_path,
771 O_WRONLY | O_TRUNC | O_CREAT,
772 0644)) == -1 ||
773 (write(sample_fd.get(), sf.data(), sf.length()) == -1)) {
774 fprintf(stderr,
775 "error:unable to write default format file: %s -- %s\n",
776 sample_path.c_str(),
777 strerror(errno));
778 }
779 }
780
781 for (const auto& bsf : lnav_sh_scripts) {
782 auto sh_path = lnav::paths::dotlnav() / fmt::format("formats/default/{}", bsf.get_name());
783 auto sf = bsf.to_string_fragment();
784 auto_fd sh_fd;
785
786 if ((sh_fd = openp(sh_path, O_WRONLY|O_TRUNC|O_CREAT, 0755)) == -1 ||
787 write(sh_fd.get(), sf.data(), sf.length()) == -1) {
788 fprintf(stderr,
789 "error:unable to write default text file: %s -- %s\n",
790 sh_path.c_str(),
791 strerror(errno));
792 }
793 }
794
795 for (const auto& bsf : lnav_scripts) {
796 struct script_metadata meta;
797 auto sf = bsf.to_string_fragment();
798 auto_fd script_fd;
799 char path[2048];
800 struct stat st;
801
802 extract_metadata(sf.data(), sf.length(), meta);
803 snprintf(path, sizeof(path), "formats/default/%s.lnav", meta.sm_name.c_str());
804 auto script_path = lnav::paths::dotlnav() / path;
805 if (statp(script_path, &st) == 0 && st.st_size == sf.length()) {
806 // Assume it's the right contents and move on...
807 continue;
808 }
809 if ((script_fd = openp(script_path, O_WRONLY|O_TRUNC|O_CREAT, 0755)) == -1 ||
810 write(script_fd.get(), sf.data(), sf.length()) == -1) {
811 fprintf(stderr,
812 "error:unable to write default text file: %s -- %s\n",
813 script_path.c_str(),
814 strerror(errno));
815 }
816 }
817 }
818
format_error_reporter(const yajlpp_parse_context & ypc,lnav_log_level_t level,const char * msg)819 static void format_error_reporter(const yajlpp_parse_context &ypc,
820 lnav_log_level_t level,
821 const char *msg)
822 {
823 if (level >= lnav_log_level_t::ERROR) {
824 struct userdata *ud = (userdata *) ypc.ypc_userdata;
825
826 ud->ud_errors->push_back(msg);
827 } else {
828 fprintf(stderr, "warning:%s\n", msg);
829 }
830 }
831
832 std::vector<intern_string_t>
load_format_file(const ghc::filesystem::path & filename,std::vector<string> & errors)833 load_format_file(const ghc::filesystem::path &filename,
834 std::vector<string> &errors)
835 {
836 std::vector<intern_string_t> retval;
837 struct userdata ud;
838 auto_fd fd;
839
840 log_info("loading formats from file: %s", filename.c_str());
841 ud.ud_format_path = filename;
842 ud.ud_format_names = &retval;
843 ud.ud_errors = &errors;
844 yajlpp_parse_context ypc(filename, &root_format_handler);
845 ypc.ypc_userdata = &ud;
846 ypc.with_obj(ud);
847 if ((fd = openp(filename, O_RDONLY)) == -1) {
848 errors.emplace_back(fmt::format(
849 "error: unable to open format file '{}' -- {}",
850 filename.string(), strerror(errno)));
851 }
852 else {
853 auto_mem<yajl_handle_t> handle(yajl_free);
854 char buffer[2048];
855 off_t offset = 0;
856 ssize_t rc = -1;
857
858 handle = yajl_alloc(&ypc.ypc_callbacks, nullptr, &ypc);
859 ypc.with_handle(handle)
860 .with_error_reporter(format_error_reporter);
861 yajl_config(handle, yajl_allow_comments, 1);
862 while (true) {
863 rc = read(fd, buffer, sizeof(buffer));
864 if (rc == 0) {
865 break;
866 }
867 else if (rc == -1) {
868 errors.push_back(fmt::format(
869 "error:{}:unable to read file -- {}",
870 filename.string(),
871 strerror(errno)));
872 break;
873 }
874 if (offset == 0 && (rc > 2) &&
875 (buffer[0] == '#') && (buffer[1] == '!')) {
876 // Turn it into a JavaScript comment.
877 buffer[0] = buffer[1] = '/';
878 }
879 if (ypc.parse((const unsigned char *)buffer, rc) != yajl_status_ok) {
880 break;
881 }
882 offset += rc;
883 }
884 if (rc == 0) {
885 ypc.complete_parse();
886 }
887 }
888
889 return retval;
890 }
891
load_from_path(const ghc::filesystem::path & path,std::vector<string> & errors)892 static void load_from_path(const ghc::filesystem::path &path, std::vector<string> &errors)
893 {
894 auto format_path = path / "formats/*/*.json";
895 static_root_mem<glob_t, globfree> gl;
896
897 log_info("loading formats from path: %s", format_path.c_str());
898 if (glob(format_path.c_str(), 0, nullptr, gl.inout()) == 0) {
899 for (int lpc = 0; lpc < (int)gl->gl_pathc; lpc++) {
900 const char *base = basename(gl->gl_pathv[lpc]);
901
902 if (startswith(base, "config.")) {
903 continue;
904 }
905
906 string filename(gl->gl_pathv[lpc]);
907 vector<intern_string_t> format_list;
908
909 format_list = load_format_file(filename, errors);
910 if (format_list.empty()) {
911 log_warning("Empty format file: %s", filename.c_str());
912 }
913 else {
914 for (auto iter = format_list.begin();
915 iter != format_list.end();
916 ++iter) {
917 log_info(" found format: %s", iter->get());
918 }
919 }
920 }
921 }
922 }
923
load_formats(const std::vector<ghc::filesystem::path> & extra_paths,std::vector<std::string> & errors)924 void load_formats(const std::vector<ghc::filesystem::path> &extra_paths,
925 std::vector<std::string> &errors)
926 {
927 auto default_source = lnav::paths::dotlnav() / "default";
928 yajlpp_parse_context ypc_builtin(default_source.string(), &root_format_handler);
929 std::vector<intern_string_t> retval;
930 struct userdata ud;
931 yajl_handle handle;
932
933 write_sample_file();
934
935 log_debug("Loading default formats");
936 for (const auto& bsf : lnav_format_json) {
937 handle = yajl_alloc(&ypc_builtin.ypc_callbacks, nullptr, &ypc_builtin);
938 ud.ud_format_names = &retval;
939 ud.ud_errors = &errors;
940 ypc_builtin
941 .with_obj(ud)
942 .with_handle(handle)
943 .with_error_reporter(format_error_reporter)
944 .ypc_userdata = &ud;
945 yajl_config(handle, yajl_allow_comments, 1);
946 auto sf = bsf.to_string_fragment();
947 if (ypc_builtin.parse(sf) != yajl_status_ok) {
948 unsigned char *msg = yajl_get_error(handle, 1,
949 (const unsigned char *) sf.data(),
950 sf.length());
951
952 errors.push_back(fmt::format(
953 FMT_STRING("builtin: invalid json -- {}"), msg));
954 yajl_free_error(handle, msg);
955 }
956 ypc_builtin.complete_parse();
957 yajl_free(handle);
958 }
959
960 for (const auto & extra_path : extra_paths) {
961 load_from_path(extra_path, errors);
962 }
963
964 uint8_t mod_counter = 0;
965
966 vector<std::shared_ptr<external_log_format>> alpha_ordered_formats;
967 for (auto iter = LOG_FORMATS.begin(); iter != LOG_FORMATS.end(); ++iter) {
968 auto& elf = iter->second;
969 elf->build(errors);
970
971 if (elf->elf_has_module_format) {
972 mod_counter += 1;
973 elf->lf_mod_index = mod_counter;
974 }
975
976 for (auto & check_iter : LOG_FORMATS) {
977 if (iter->first == check_iter.first) {
978 continue;
979 }
980
981 auto& check_elf = check_iter.second;
982 if (elf->match_samples(check_elf->elf_samples)) {
983 log_warning("Format collision, format '%s' matches sample from '%s'",
984 elf->get_name().get(),
985 check_elf->get_name().get());
986 elf->elf_collision.push_back(check_elf->get_name());
987 }
988 }
989
990 if (errors.empty()) {
991 alpha_ordered_formats.push_back(elf);
992 }
993 }
994
995 if (!errors.empty()) {
996 return;
997 }
998
999 auto& graph_ordered_formats = external_log_format::GRAPH_ORDERED_FORMATS;
1000
1001 while (!alpha_ordered_formats.empty()) {
1002 vector<intern_string_t> popped_formats;
1003
1004 for (auto iter = alpha_ordered_formats.begin();
1005 iter != alpha_ordered_formats.end();) {
1006 auto elf = *iter;
1007 if (elf->elf_collision.empty()) {
1008 iter = alpha_ordered_formats.erase(iter);
1009 popped_formats.push_back(elf->get_name());
1010 graph_ordered_formats.push_back(elf);
1011 }
1012 else {
1013 ++iter;
1014 }
1015 }
1016
1017 if (popped_formats.empty() && !alpha_ordered_formats.empty()) {
1018 bool broke_cycle = false;
1019
1020 log_warning("Detected a cycle...");
1021 for (const auto& elf : alpha_ordered_formats) {
1022 if (elf->elf_builtin_format) {
1023 log_warning(" Skipping builtin format -- %s",
1024 elf->get_name().get());
1025 } else {
1026 log_warning(" Breaking cycle by picking -- %s",
1027 elf->get_name().get());
1028 elf->elf_collision.clear();
1029 broke_cycle = true;
1030 break;
1031 }
1032 }
1033 if (!broke_cycle) {
1034 alpha_ordered_formats.front()->elf_collision.clear();
1035 }
1036 }
1037
1038 for (const auto& elf : alpha_ordered_formats) {
1039 for (auto & popped_format : popped_formats) {
1040 elf->elf_collision.remove(popped_format);
1041 }
1042 }
1043 }
1044
1045 log_info("Format order:")
1046 for (auto & graph_ordered_format : graph_ordered_formats) {
1047 log_info(" %s", graph_ordered_format->get_name().get());
1048 }
1049
1050 auto &roots = log_format::get_root_formats();
1051 auto iter = std::find_if(roots.begin(), roots.end(), [](const auto& elem) {
1052 return elem->get_name() == "generic_log";
1053 });
1054 roots.insert(iter, graph_ordered_formats.begin(), graph_ordered_formats.end());
1055 }
1056
exec_sql_in_path(sqlite3 * db,const ghc::filesystem::path & path,std::vector<string> & errors)1057 static void exec_sql_in_path(sqlite3 *db, const ghc::filesystem::path &path, std::vector<string> &errors)
1058 {
1059 auto format_path = path / "formats/*/*.sql";
1060 static_root_mem<glob_t, globfree> gl;
1061
1062 log_info("executing SQL files in path: %s", format_path.c_str());
1063 if (glob(format_path.c_str(), 0, nullptr, gl.inout()) == 0) {
1064 for (int lpc = 0; lpc < (int)gl->gl_pathc; lpc++) {
1065 auto filename = ghc::filesystem::path(gl->gl_pathv[lpc]);
1066 auto read_res = read_file(filename);
1067
1068 if (read_res.isOk()) {
1069 log_info("Executing SQL file: %s", filename.c_str());
1070 auto content = read_res.unwrap();
1071
1072 sql_execute_script(db, filename.c_str(), content.c_str(), errors);
1073 }
1074 else {
1075 errors.push_back(fmt::format(
1076 "error:unable to read file '{}' -- {}",
1077 filename.string(), read_res.unwrapErr()));
1078 }
1079 }
1080 }
1081 }
1082
load_format_extra(sqlite3 * db,const std::vector<ghc::filesystem::path> & extra_paths,std::vector<std::string> & errors)1083 void load_format_extra(sqlite3 *db,
1084 const std::vector<ghc::filesystem::path> &extra_paths,
1085 std::vector<std::string> &errors)
1086 {
1087 for (const auto & extra_path : extra_paths) {
1088 exec_sql_in_path(db, extra_path, errors);
1089 }
1090 }
1091
extract_metadata(const char * contents,size_t len,struct script_metadata & meta_out)1092 static void extract_metadata(const char *contents, size_t len, struct script_metadata &meta_out)
1093 {
1094 static const pcrepp SYNO_RE("^#\\s+@synopsis:(.*)$", PCRE_MULTILINE);
1095 static const pcrepp DESC_RE("^#\\s+@description:(.*)$", PCRE_MULTILINE);
1096
1097 pcre_input pi(contents, 0, len);
1098 pcre_context_static<16> pc;
1099
1100 pi.reset(contents, 0, len);
1101 if (SYNO_RE.match(pc, pi)) {
1102 meta_out.sm_synopsis = trim(pi.get_substr(pc[0]));
1103 }
1104 pi.reset(contents, 0, len);
1105 if (DESC_RE.match(pc, pi)) {
1106 meta_out.sm_description = trim(pi.get_substr(pc[0]));
1107 }
1108
1109 if (!meta_out.sm_synopsis.empty()) {
1110 size_t space = meta_out.sm_synopsis.find(' ');
1111
1112 if (space == string::npos) {
1113 space = meta_out.sm_synopsis.size();
1114 }
1115 meta_out.sm_name = meta_out.sm_synopsis.substr(0, space);
1116 }
1117 }
1118
extract_metadata_from_file(struct script_metadata & meta_inout)1119 void extract_metadata_from_file(struct script_metadata &meta_inout)
1120 {
1121 char buffer[8 * 1024];
1122 auto_mem<FILE> fp(fclose);
1123 struct stat st;
1124
1125 if (statp(meta_inout.sm_path, &st) == -1) {
1126 log_warning("unable to open script -- %s", meta_inout.sm_path.c_str());
1127 } else if (!S_ISREG(st.st_mode)) {
1128 log_warning("not a regular file -- %s", meta_inout.sm_path.c_str());
1129 } else if ((fp = fopen(meta_inout.sm_path.c_str(), "r")) != nullptr) {
1130 size_t len;
1131
1132 len = fread(buffer, 1, sizeof(buffer), fp.in());
1133 extract_metadata(buffer, len, meta_inout);
1134 }
1135 }
1136
find_format_in_path(const ghc::filesystem::path & path,available_scripts & scripts)1137 static void find_format_in_path(const ghc::filesystem::path &path,
1138 available_scripts& scripts)
1139 {
1140 auto format_path = path / "formats/*/*.lnav";
1141 static_root_mem<glob_t, globfree> gl;
1142
1143 log_debug("Searching for script in path: %s", format_path.c_str());
1144 if (glob(format_path.c_str(), 0, nullptr, gl.inout()) == 0) {
1145 for (int lpc = 0; lpc < (int)gl->gl_pathc; lpc++) {
1146 const char *filename = basename(gl->gl_pathv[lpc]);
1147 string script_name = string(filename, strlen(filename) - 5);
1148 struct script_metadata meta;
1149
1150 meta.sm_path = gl->gl_pathv[lpc];
1151 meta.sm_name = script_name;
1152 extract_metadata_from_file(meta);
1153 scripts.as_scripts[script_name].push_back(meta);
1154
1155 log_debug(" found script: %s", meta.sm_path.c_str());
1156 }
1157 }
1158 }
1159
find_format_scripts(const vector<ghc::filesystem::path> & extra_paths,available_scripts & scripts)1160 void find_format_scripts(const vector<ghc::filesystem::path> &extra_paths,
1161 available_scripts& scripts)
1162 {
1163 for (const auto &extra_path : extra_paths) {
1164 find_format_in_path(extra_path, scripts);
1165 }
1166 }
1167
load_format_vtabs(log_vtab_manager * vtab_manager,std::vector<std::string> & errors)1168 void load_format_vtabs(log_vtab_manager *vtab_manager,
1169 std::vector<std::string> &errors)
1170 {
1171 auto &root_formats = LOG_FORMATS;
1172
1173 for (auto & root_format : root_formats) {
1174 root_format.second->register_vtabs(vtab_manager, errors);
1175 }
1176 }
1177