1 /**
2 * Copyright (c) 2007-2012, Timothy Stack
3 *
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * * Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 * * Neither the name of Timothy Stack nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
25 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 #include "config.h"
31
32 #include <future>
33 #include <algorithm>
34 #include <sqlite3.h>
35
36 #include "base/humanize.time.hh"
37 #include "base/string_util.hh"
38 #include "k_merge_tree.h"
39 #include "lnav_util.hh"
40 #include "log_accel.hh"
41 #include "relative_time.hh"
42 #include "logfile_sub_source.hh"
43 #include "command_executor.hh"
44 #include "ansi_scrubber.hh"
45 #include "sql_util.hh"
46 #include "yajlpp/yajlpp.hh"
47
48 using namespace std;
49
50 bookmark_type_t logfile_sub_source::BM_ERRORS("error");
51 bookmark_type_t logfile_sub_source::BM_WARNINGS("warning");
52 bookmark_type_t logfile_sub_source::BM_FILES("");
53
pretty_sql_callback(exec_context & ec,sqlite3_stmt * stmt)54 static int pretty_sql_callback(exec_context &ec, sqlite3_stmt *stmt)
55 {
56 if (!sqlite3_stmt_busy(stmt)) {
57 return 0;
58 }
59
60 int ncols = sqlite3_column_count(stmt);
61
62 for (int lpc = 0; lpc < ncols; lpc++) {
63 if (!ec.ec_accumulator.empty()) {
64 ec.ec_accumulator.append(", ");
65 }
66
67 const char *res = (const char *)sqlite3_column_text(stmt, lpc);
68 if (res == nullptr) {
69 continue;
70 }
71
72 ec.ec_accumulator.append(res);
73 }
74
75 return 0;
76 }
77
pretty_pipe_callback(exec_context & ec,const string & cmdline,auto_fd & fd)78 static future<string> pretty_pipe_callback(exec_context &ec,
79 const string &cmdline,
80 auto_fd &fd)
81 {
82 auto retval = std::async(std::launch::async, [&]() {
83 char buffer[1024];
84 ostringstream ss;
85 ssize_t rc;
86
87 while ((rc = read(fd, buffer, sizeof(buffer))) > 0) {
88 ss.write(buffer, rc);
89 }
90
91 string retval = ss.str();
92
93 if (endswith(retval, "\n")) {
94 retval.resize(retval.length() - 1);
95 }
96
97 return retval;
98 });
99
100 return retval;
101 }
102
logfile_sub_source()103 logfile_sub_source::logfile_sub_source()
104 : text_sub_source(1),
105 lss_meta_grepper(*this),
106 lss_location_history(*this)
107 {
108 this->tss_supports_filtering = true;
109 this->clear_line_size_cache();
110 this->clear_min_max_log_times();
111 }
112
find(const char * fn,content_line_t & line_base)113 shared_ptr<logfile> logfile_sub_source::find(const char *fn,
114 content_line_t &line_base)
115 {
116 iterator iter;
117 shared_ptr<logfile> retval = nullptr;
118
119 line_base = content_line_t(0);
120 for (iter = this->lss_files.begin();
121 iter != this->lss_files.end() && retval == nullptr;
122 iter++) {
123 auto &ld = *(*iter);
124 auto lf = ld.get_file_ptr();
125
126 if (lf == nullptr) {
127 continue;
128 }
129 if (strcmp(lf->get_filename().c_str(), fn) == 0) {
130 retval = ld.get_file();
131 }
132 else {
133 line_base += content_line_t(MAX_LINES_PER_FILE);
134 }
135 }
136
137 return retval;
138 }
139
find_from_time(const struct timeval & start) const140 nonstd::optional<vis_line_t> logfile_sub_source::find_from_time(const struct timeval &start) const
141 {
142 auto lb = lower_bound(this->lss_filtered_index.begin(),
143 this->lss_filtered_index.end(),
144 start,
145 filtered_logline_cmp(*this));
146 if (lb != this->lss_filtered_index.end()) {
147 return vis_line_t(lb - this->lss_filtered_index.begin());
148 }
149
150 return nonstd::nullopt;
151 }
152
text_value_for_line(textview_curses & tc,int row,string & value_out,line_flags_t flags)153 void logfile_sub_source::text_value_for_line(textview_curses &tc,
154 int row,
155 string &value_out,
156 line_flags_t flags)
157 {
158 content_line_t line(0);
159
160 require(row >= 0);
161 require((size_t)row < this->lss_filtered_index.size());
162
163 line = this->at(vis_line_t(row));
164
165 if (flags & RF_RAW) {
166 shared_ptr<logfile> lf = this->find(line);
167 value_out = lf->read_line(lf->begin() + line)
168 .map([](auto sbr) { return to_string(sbr); })
169 .unwrapOr({});
170 return;
171 }
172
173 require(!this->lss_in_value_for_line);
174
175 this->lss_in_value_for_line = true;
176 this->lss_token_flags = flags;
177 this->lss_token_file_data = this->find_data(line);
178 this->lss_token_file = (*this->lss_token_file_data)->get_file();
179 this->lss_token_line = this->lss_token_file->begin() + line;
180
181 this->lss_token_attrs.clear();
182 this->lss_token_values.clear();
183 this->lss_share_manager.invalidate_refs();
184 if (flags & text_sub_source::RF_FULL) {
185 shared_buffer_ref sbr;
186
187 this->lss_token_file->read_full_message(this->lss_token_line, sbr);
188 this->lss_token_value = to_string(sbr);
189 } else {
190 this->lss_token_value =
191 this->lss_token_file->read_line(this->lss_token_line).map([](auto sbr) {
192 return to_string(sbr);
193 }).unwrapOr({});
194 }
195 this->lss_token_shift_start = 0;
196 this->lss_token_shift_size = 0;
197
198 auto format = this->lss_token_file->get_format();
199
200 value_out = this->lss_token_value;
201 if (this->lss_flags & F_SCRUB) {
202 format->scrub(value_out);
203 }
204
205 shared_buffer_ref sbr;
206
207 sbr.share(this->lss_share_manager,
208 (char *)this->lss_token_value.c_str(), this->lss_token_value.size());
209 if (this->lss_token_line->is_continued()) {
210 this->lss_token_attrs.emplace_back(
211 line_range{0, (int) this->lss_token_value.length()},
212 &SA_BODY);
213 } else {
214 format->annotate(line, sbr, this->lss_token_attrs, this->lss_token_values);
215 }
216 if (this->lss_token_line->get_sub_offset() != 0) {
217 this->lss_token_attrs.clear();
218 }
219 if (flags & RF_REWRITE) {
220 exec_context ec(&this->lss_token_values, pretty_sql_callback, pretty_pipe_callback);
221 string rewritten_line;
222
223 ec.with_perms(exec_context::perm_t::READ_ONLY);
224 ec.ec_local_vars.push(map<string, string>());
225 ec.ec_top_line = vis_line_t(row);
226 add_ansi_vars(ec.ec_global_vars);
227 add_global_vars(ec);
228 format->rewrite(ec, sbr, this->lss_token_attrs, rewritten_line);
229 this->lss_token_value.assign(rewritten_line);
230 value_out = this->lss_token_value;
231 }
232
233 if ((this->lss_token_file->is_time_adjusted() ||
234 format->lf_timestamp_flags & ETF_MACHINE_ORIENTED) &&
235 format->lf_date_time.dts_fmt_lock != -1) {
236 auto time_attr = find_string_attr(
237 this->lss_token_attrs, &logline::L_TIMESTAMP);
238 if (time_attr != this->lss_token_attrs.end()) {
239 const struct line_range time_range = time_attr->sa_range;
240 struct timeval adjusted_time;
241 struct exttm adjusted_tm;
242 char buffer[128];
243 const char *fmt;
244 ssize_t len;
245
246 if (format->lf_timestamp_flags & ETF_MACHINE_ORIENTED) {
247 format->lf_date_time.convert_to_timeval(
248 &this->lss_token_value.c_str()[time_range.lr_start],
249 time_range.length(),
250 format->get_timestamp_formats(),
251 adjusted_time);
252 fmt = "%Y-%m-%d %H:%M:%S.%f";
253 gmtime_r(&adjusted_time.tv_sec, &adjusted_tm.et_tm);
254 adjusted_tm.et_nsec = adjusted_time.tv_usec * 1000;
255 len = ftime_fmt(buffer, sizeof(buffer), fmt, adjusted_tm);
256 } else {
257 adjusted_time = this->lss_token_line->get_timeval();
258 gmtime_r(&adjusted_time.tv_sec, &adjusted_tm.et_tm);
259 adjusted_tm.et_nsec = adjusted_time.tv_usec * 1000;
260 len = format->lf_date_time.ftime(buffer, sizeof(buffer),
261 adjusted_tm);
262 }
263
264 value_out.replace(time_range.lr_start,
265 time_range.length(),
266 buffer,
267 len);
268 this->lss_token_shift_start = time_range.lr_start;
269 this->lss_token_shift_size = len - time_range.length();
270 }
271 }
272
273 if (this->lss_flags & F_FILENAME || this->lss_flags & F_BASENAME) {
274 size_t file_offset_end;
275 std::string name;
276 if (this->lss_flags & F_FILENAME) {
277 file_offset_end = this->lss_filename_width;
278 name = this->lss_token_file->get_filename();
279 if (file_offset_end < name.size()) {
280 file_offset_end = name.size();
281 this->lss_filename_width = name.size();
282 }
283 } else {
284 file_offset_end = this->lss_basename_width;
285 name = this->lss_token_file->get_unique_path();
286 if (file_offset_end < name.size()) {
287 file_offset_end = name.size();
288 this->lss_basename_width = name.size();
289 }
290 }
291 value_out.insert(0, 1, '|');
292 value_out.insert(0, file_offset_end - name.size(), ' ');
293 value_out.insert(0, name);
294 } else {
295 // Insert space for the file/search-hit markers.
296 value_out.insert(0, 1, ' ');
297 }
298
299 if (this->lss_flags & F_TIME_OFFSET) {
300 auto curr_tv = this->lss_token_line->get_timeval();
301 struct timeval diff_tv;
302
303 vis_line_t prev_mark =
304 tc.get_bookmarks()[&textview_curses::BM_USER].prev(vis_line_t(row));
305 vis_line_t next_mark =
306 tc.get_bookmarks()[&textview_curses::BM_USER].next(vis_line_t(row));
307 if (prev_mark == -1 && next_mark != -1) {
308 auto next_line = this->find_line(this->at(next_mark));
309
310 diff_tv = curr_tv - next_line->get_timeval();
311 } else {
312 if (prev_mark == -1_vl) {
313 prev_mark = 0_vl;
314 }
315
316 auto first_line = this->find_line(this->at(prev_mark));
317 auto start_tv = first_line->get_timeval();
318 diff_tv = curr_tv - start_tv;
319 }
320
321 auto relstr = humanize::time::duration::from_tv(diff_tv).to_string();
322 value_out = fmt::format(FMT_STRING("{: >12}|{}"), relstr, value_out);
323 }
324 this->lss_in_value_for_line = false;
325 }
326
text_attrs_for_line(textview_curses & lv,int row,string_attrs_t & value_out)327 void logfile_sub_source::text_attrs_for_line(textview_curses &lv,
328 int row,
329 string_attrs_t &value_out)
330 {
331 view_colors & vc = view_colors::singleton();
332 logline * next_line = nullptr;
333 struct line_range lr;
334 int time_offset_end = 0;
335 int attrs = 0;
336
337 value_out = this->lss_token_attrs;
338
339 attrs = vc.vc_level_attrs[this->lss_token_line->get_msg_level()].first;
340
341 if ((row + 1) < (int)this->lss_filtered_index.size()) {
342 next_line = this->find_line(this->at(vis_line_t(row + 1)));
343 }
344
345 if (next_line != nullptr &&
346 (day_num(next_line->get_time()) >
347 day_num(this->lss_token_line->get_time()))) {
348 attrs |= A_UNDERLINE;
349 }
350
351 const std::vector<logline_value> &line_values = this->lss_token_values;
352
353 lr.lr_start = 0;
354 lr.lr_end = this->lss_token_value.length();
355 value_out.emplace_back(lr, &SA_ORIGINAL_LINE);
356
357 lr.lr_start = time_offset_end;
358 lr.lr_end = -1;
359
360 value_out.emplace_back(lr, &view_curses::VC_STYLE, attrs);
361
362 if (this->lss_token_line->get_msg_level() == log_level_t::LEVEL_INVALID) {
363 for (auto& token_attr : this->lss_token_attrs) {
364 if (token_attr.sa_type != &SA_INVALID) {
365 continue;
366 }
367
368
369 value_out.emplace_back(token_attr.sa_range,
370 &view_curses::VC_ROLE,
371 view_colors::VCR_INVALID_MSG);
372 }
373 }
374
375 for (const auto &line_value : line_values) {
376 if ((!(this->lss_token_flags & RF_FULL) &&
377 line_value.lv_sub_offset != this->lss_token_line->get_sub_offset()) ||
378 !line_value.lv_origin.is_valid()) {
379 continue;
380 }
381
382 if (line_value.lv_meta.is_hidden()) {
383 value_out.emplace_back(
384 line_value.lv_origin, &SA_HIDDEN);
385 }
386
387 if (!line_value.lv_meta.lvm_identifier || !line_value.lv_origin.is_valid()) {
388 continue;
389 }
390
391 line_range ident_range = line_value.lv_origin;
392 if (this->lss_token_flags & RF_FULL) {
393 ident_range = line_value.origin_in_full_msg(
394 this->lss_token_value.c_str(), this->lss_token_value.length());
395 }
396
397 value_out.emplace_back(ident_range,
398 &view_curses::VC_ROLE,
399 view_colors::VCR_IDENTIFIER);
400 }
401
402 if (this->lss_token_shift_size) {
403 shift_string_attrs(value_out, this->lss_token_shift_start + 1,
404 this->lss_token_shift_size);
405 }
406
407 shift_string_attrs(value_out, 0, 1);
408
409 lr.lr_start = 0;
410 lr.lr_end = 1;
411 {
412 auto &bm = lv.get_bookmarks();
413 const auto &bv = bm[&BM_FILES];
414 bool is_first_for_file = binary_search(
415 bv.begin(), bv.end(), vis_line_t(row));
416 bool is_last_for_file = binary_search(
417 bv.begin(), bv.end(), vis_line_t(row + 1));
418 chtype graph = ACS_VLINE;
419 if (is_first_for_file) {
420 if (is_last_for_file) {
421 graph = ACS_HLINE;
422 }
423 else {
424 graph = ACS_ULCORNER;
425 }
426 }
427 else if (is_last_for_file) {
428 graph = ACS_LLCORNER;
429 }
430 value_out.push_back(
431 string_attr(lr, &view_curses::VC_GRAPHIC, graph));
432
433 if (!(this->lss_token_flags & RF_FULL)) {
434 bookmark_vector<vis_line_t> &bv_search = bm[&textview_curses::BM_SEARCH];
435
436 if (binary_search(::begin(bv_search), ::end(bv_search),
437 vis_line_t(row))) {
438 lr.lr_start = 0;
439 lr.lr_end = 1;
440 value_out.emplace_back(lr, &view_curses::VC_STYLE, A_REVERSE);
441 }
442 }
443 }
444
445 value_out.emplace_back(lr, &view_curses::VC_STYLE, vc.attrs_for_ident(
446 this->lss_token_file->get_filename()));
447
448 if (this->lss_flags & F_FILENAME || this->lss_flags & F_BASENAME) {
449 size_t file_offset_end = (this->lss_flags & F_FILENAME) ?
450 this->lss_filename_width :
451 this->lss_basename_width ;
452
453 shift_string_attrs(value_out, 0, file_offset_end);
454
455 lr.lr_start = 0;
456 lr.lr_end = file_offset_end + 1;
457 value_out.emplace_back(lr, &view_curses::VC_STYLE, vc.attrs_for_ident(
458 this->lss_token_file->get_filename()));
459 }
460
461 if (this->lss_flags & F_TIME_OFFSET) {
462 time_offset_end = 13;
463 lr.lr_start = 0;
464 lr.lr_end = time_offset_end;
465
466 shift_string_attrs(value_out, 0, time_offset_end);
467
468 value_out.emplace_back(lr,
469 &view_curses::VC_ROLE,
470 view_colors::VCR_OFFSET_TIME);
471 value_out.emplace_back(line_range(12, 13),
472 &view_curses::VC_GRAPHIC, ACS_VLINE);
473
474 view_colors::role_t bar_role = view_colors::VCR_NONE;
475
476 switch (this->get_line_accel_direction(vis_line_t(row))) {
477 case log_accel::A_STEADY:
478 break;
479 case log_accel::A_DECEL:
480 bar_role = view_colors::VCR_DIFF_DELETE;
481 break;
482 case log_accel::A_ACCEL:
483 bar_role = view_colors::VCR_DIFF_ADD;
484 break;
485 }
486 if (bar_role != view_colors::VCR_NONE) {
487 value_out.emplace_back(
488 line_range(12, 13), &view_curses::VC_ROLE, bar_role);
489 }
490 }
491
492 lr.lr_start = 0;
493 lr.lr_end = -1;
494 value_out.emplace_back(lr, &logline::L_FILE, this->lss_token_file.get());
495 value_out.emplace_back(lr, &SA_FORMAT,
496 this->lss_token_file->get_format()->get_name());
497
498 {
499 const auto &bv = lv.get_bookmarks()[&textview_curses::BM_META];
500 bookmark_vector<vis_line_t>::const_iterator bv_iter;
501
502 bv_iter = lower_bound(bv.begin(), bv.end(), vis_line_t(row + 1));
503 if (bv_iter != bv.begin()) {
504 --bv_iter;
505 content_line_t part_start_line = this->at(*bv_iter);
506 std::map<content_line_t, bookmark_metadata>::iterator bm_iter;
507
508 if ((bm_iter = this->lss_user_mark_metadata.find(part_start_line))
509 != this->lss_user_mark_metadata.end() &&
510 !bm_iter->second.bm_name.empty()) {
511 lr.lr_start = 0;
512 lr.lr_end = -1;
513 value_out.emplace_back(lr, &logline::L_PARTITION, &bm_iter->second);
514 }
515 }
516
517 auto bm_iter = this->lss_user_mark_metadata.find(this->at(vis_line_t(row)));
518
519 if (bm_iter != this->lss_user_mark_metadata.end()) {
520 lr.lr_start = 0;
521 lr.lr_end = -1;
522 value_out.emplace_back(lr, &logline::L_META, &bm_iter->second);
523 }
524 }
525
526 if (this->lss_token_file->is_time_adjusted()) {
527 struct line_range time_range = find_string_attr_range(
528 value_out, &logline::L_TIMESTAMP);
529
530 if (time_range.lr_end != -1) {
531 value_out.emplace_back(time_range, &view_curses::VC_ROLE,
532 view_colors::VCR_ADJUSTED_TIME);
533 }
534 }
535
536 if (this->lss_token_line->is_time_skewed()) {
537 struct line_range time_range = find_string_attr_range(
538 value_out, &logline::L_TIMESTAMP);
539
540 if (time_range.lr_end != -1) {
541 value_out.emplace_back(time_range, &view_curses::VC_ROLE,
542 view_colors::VCR_SKEWED_TIME);
543 }
544 }
545
546 if (!this->lss_token_line->is_continued()) {
547 if (this->lss_preview_filter_stmt != nullptr) {
548 int color;
549 auto eval_res = this->eval_sql_filter(this->lss_preview_filter_stmt.in(),
550 this->lss_token_file_data,
551 this->lss_token_line);
552 if (eval_res.isErr()) {
553 color = COLOR_YELLOW;
554 value_out.emplace_back(line_range{0, -1},
555 &SA_ERROR,
556 eval_res.unwrapErr());
557 } else {
558 auto matched = eval_res.unwrap();
559
560 if (matched) {
561 color = COLOR_GREEN;
562 } else {
563 color = COLOR_RED;
564 value_out.emplace_back(line_range{0, 1}, &view_curses::VC_STYLE,
565 A_BLINK);
566 }
567 }
568 value_out.emplace_back(line_range{0, 1}, &view_curses::VC_BACKGROUND, color);
569 }
570
571 auto sql_filter_opt = this->get_sql_filter();
572 if (sql_filter_opt) {
573 auto sf = (sql_filter *) sql_filter_opt.value().get();
574 int color;
575 auto eval_res = this->eval_sql_filter(sf->sf_filter_stmt.in(),
576 this->lss_token_file_data,
577 this->lss_token_line);
578 if (eval_res.isErr()) {
579 auto msg = fmt::format(
580 "filter expression evaluation failed with -- {}",
581 eval_res.unwrapErr());
582 color = COLOR_YELLOW;
583 value_out.emplace_back(line_range{0, -1},
584 &SA_ERROR,
585 msg);
586 value_out.emplace_back(line_range{0, 1}, &view_curses::VC_BACKGROUND, color);
587 }
588 }
589 }
590 }
591
rebuild_index(nonstd::optional<ui_clock::time_point> deadline)592 logfile_sub_source::rebuild_result logfile_sub_source::rebuild_index(nonstd::optional<ui_clock::time_point> deadline)
593 {
594 iterator iter;
595 size_t total_lines = 0;
596 bool full_sort = false;
597 int file_count = 0;
598 bool force = this->lss_force_rebuild;
599 auto retval = rebuild_result::rr_no_change;
600 nonstd::optional<struct timeval> lowest_tv = nonstd::nullopt;
601 vis_line_t search_start = 0_vl;
602
603 this->lss_force_rebuild = false;
604 if (force) {
605 log_debug("forced to full rebuild");
606 retval = rebuild_result::rr_full_rebuild;
607 }
608
609 std::vector<size_t> file_order(this->lss_files.size());
610
611 for (size_t lpc = 0; lpc < file_order.size(); lpc++) {
612 file_order[lpc] = lpc;
613 }
614 if (!this->lss_index.empty()) {
615 std::stable_sort(file_order.begin(), file_order.end(),
616 [this](const auto &left, const auto &right) {
617 const auto &left_ld = this->lss_files[left];
618 const auto &right_ld = this->lss_files[right];
619
620 if (left_ld->get_file_ptr() == nullptr) {
621 return true;
622 }
623 if (right_ld->get_file_ptr() == nullptr) {
624 return false;
625 }
626
627 return left_ld->get_file_ptr()->back() <
628 right_ld->get_file_ptr()->back();
629 });
630 }
631
632 bool time_left = true;
633 for (const auto file_index : file_order) {
634 auto &ld = *(this->lss_files[file_index]);
635 auto lf = ld.get_file_ptr();
636
637 if (lf == nullptr) {
638 if (ld.ld_lines_indexed > 0) {
639 log_debug("%d: file closed, doing full rebuild",
640 ld.ld_file_index);
641 force = true;
642 retval = rebuild_result::rr_full_rebuild;
643 }
644 }
645 else {
646 if (time_left && deadline && ui_clock::now() > deadline.value()) {
647 log_debug("no time left, skipping %s", lf->get_filename().c_str());
648 time_left = false;
649 }
650
651 if (!this->tss_view->is_paused() && time_left) {
652 switch (lf->rebuild_index(deadline)) {
653 case logfile::rebuild_result_t::NO_NEW_LINES:
654 // No changes
655 break;
656 case logfile::rebuild_result_t::NEW_LINES:
657 if (retval == rebuild_result::rr_no_change) {
658 retval = rebuild_result::rr_appended_lines;
659 }
660 log_debug("new lines for %s:%d", lf->get_filename().c_str(), lf->size());
661 if (!this->lss_index.empty() &&
662 lf->size() > ld.ld_lines_indexed) {
663 logline &new_file_line = (*lf)[ld.ld_lines_indexed];
664 content_line_t cl = this->lss_index.back();
665 logline *last_indexed_line = this->find_line(cl);
666
667 // If there are new lines that are older than what we
668 // have in the index, we need to resort.
669 if (last_indexed_line == nullptr ||
670 new_file_line <
671 last_indexed_line->get_timeval()) {
672 log_debug("%s:%ld: found older lines, full "
673 "rebuild: %p %lld < %lld",
674 lf->get_filename().c_str(),
675 ld.ld_lines_indexed,
676 last_indexed_line,
677 new_file_line.get_time_in_millis(),
678 last_indexed_line == nullptr ?
679 (uint64_t) -1 :
680 last_indexed_line->get_time_in_millis());
681 if (retval <= rebuild_result::rr_partial_rebuild) {
682 retval = rebuild_result::rr_partial_rebuild;
683 if (!lowest_tv) {
684 lowest_tv = new_file_line.get_timeval();
685 } else if (new_file_line.get_timeval() <
686 lowest_tv.value()) {
687 lowest_tv = new_file_line.get_timeval();
688 }
689 }
690 }
691 }
692 break;
693 case logfile::rebuild_result_t::INVALID:
694 case logfile::rebuild_result_t::NEW_ORDER:
695 log_debug("%s: log file has a new order, full rebuild",
696 lf->get_filename().c_str());
697 retval = rebuild_result::rr_full_rebuild;
698 force = true;
699 full_sort = true;
700 break;
701 }
702 }
703 file_count += 1;
704 total_lines += lf->size();
705 }
706 }
707
708 if (this->lss_index.empty() && !time_left) {
709 return rebuild_result::rr_appended_lines;
710 }
711
712 if (this->lss_index.reserve(total_lines)) {
713 force = true;
714 retval = rebuild_result::rr_full_rebuild;
715 }
716
717 auto& vis_bm = this->tss_view->get_bookmarks();
718
719 if (force) {
720 for (iter = this->lss_files.begin();
721 iter != this->lss_files.end();
722 iter++) {
723 (*iter)->ld_lines_indexed = 0;
724 }
725
726 this->lss_index.clear();
727 this->lss_filtered_index.clear();
728 this->lss_longest_line = 0;
729 this->lss_basename_width = 0;
730 this->lss_filename_width = 0;
731 vis_bm[&textview_curses::BM_USER_EXPR].clear();
732 } else if (retval == rebuild_result::rr_partial_rebuild) {
733 size_t remaining = 0;
734
735 log_debug("partial rebuild with lowest time: %ld",
736 lowest_tv.value().tv_sec);
737 for (iter = this->lss_files.begin();
738 iter != this->lss_files.end();
739 iter++) {
740 logfile_data &ld = *(*iter);
741 auto lf = ld.get_file_ptr();
742
743 if (lf == nullptr) {
744 continue;
745 }
746
747 auto line_iter = lf->find_from_time(lowest_tv.value());
748
749 if (line_iter) {
750 log_debug("%s: lowest line time %ld; line %ld; size %ld",
751 lf->get_filename().c_str(),
752 line_iter.value()->get_timeval().tv_sec,
753 std::distance(lf->cbegin(), line_iter.value()),
754 lf->size());
755 }
756 ld.ld_lines_indexed = std::distance(
757 lf->cbegin(), line_iter.value_or(lf->cend()));
758 remaining += lf->size() - ld.ld_lines_indexed;
759 }
760
761 auto row_iter = lower_bound(this->lss_index.begin(),
762 this->lss_index.end(),
763 *lowest_tv,
764 logline_cmp(*this));
765 this->lss_index.shrink_to(std::distance(
766 this->lss_index.begin(), row_iter));
767 log_debug("new index size %ld/%ld; remain %ld",
768 this->lss_index.ba_size,
769 this->lss_index.ba_capacity,
770 remaining);
771 auto filt_row_iter = lower_bound(this->lss_filtered_index.begin(),
772 this->lss_filtered_index.end(),
773 *lowest_tv,
774 filtered_logline_cmp(*this));
775 this->lss_filtered_index.resize(std::distance(
776 this->lss_filtered_index.begin(), filt_row_iter));
777 search_start = vis_line_t(this->lss_filtered_index.size());
778
779 auto bm_range = vis_bm[&textview_curses::BM_USER_EXPR].equal_range(
780 search_start, -1_vl);
781 auto bm_new_size = std::distance(vis_bm[&textview_curses::BM_USER_EXPR]
782 .begin(), bm_range.first);
783 vis_bm[&textview_curses::BM_USER_EXPR].resize(bm_new_size);
784
785 if (this->lss_index_delegate) {
786 this->lss_index_delegate->index_start(*this);
787 for (const auto row_in_full_index : this->lss_filtered_index) {
788 auto cl = this->lss_index[row_in_full_index];
789 uint64_t line_number;
790 auto ld_iter = this->find_data(cl, line_number);
791 auto& ld = *ld_iter;
792 auto line_iter = ld->get_file_ptr()->begin() + line_number;
793
794 this->lss_index_delegate->index_line(
795 *this, ld->get_file_ptr(), line_iter);
796 }
797 }
798 }
799
800 if (retval != rebuild_result::rr_no_change || force) {
801 size_t index_size = 0, start_size = this->lss_index.size();
802 logline_cmp line_cmper(*this);
803
804 for (auto& ld : this->lss_files) {
805 auto lf = ld->get_file_ptr();
806
807 if (lf == nullptr) {
808 continue;
809 }
810 this->lss_longest_line = std::max(
811 this->lss_longest_line, lf->get_longest_line_length());
812 this->lss_basename_width = std::max(
813 this->lss_basename_width, lf->get_unique_path().size());
814 this->lss_filename_width = std::max(
815 this->lss_filename_width, lf->get_filename().size());
816 }
817
818 if (full_sort) {
819 for (auto& ld : this->lss_files) {
820 auto lf = ld->get_file_ptr();
821
822 if (lf == nullptr) {
823 continue;
824 }
825
826 for (size_t line_index = 0; line_index < lf->size(); line_index++) {
827 content_line_t con_line(ld->ld_file_index * MAX_LINES_PER_FILE +
828 line_index);
829
830 this->lss_index.push_back(con_line);
831 }
832 }
833
834 // XXX get rid of this full sort on the initial run, it's not
835 // needed unless the file is not in time-order
836 if (this->lss_sorting_observer) {
837 this->lss_sorting_observer(*this, 0, this->lss_index.size());
838 }
839 sort(this->lss_index.begin(), this->lss_index.end(), line_cmper);
840 if (this->lss_sorting_observer) {
841 this->lss_sorting_observer(*this, this->lss_index.size(),
842 this->lss_index.size());
843 }
844 } else {
845 kmerge_tree_c<logline, logfile_data, logfile::iterator> merge(
846 file_count);
847
848 for (iter = this->lss_files.begin();
849 iter != this->lss_files.end();
850 iter++) {
851 logfile_data *ld = iter->get();
852 auto lf = ld->get_file_ptr();
853 if (lf == nullptr) {
854 continue;
855 }
856
857 merge.add(ld,
858 lf->begin() + ld->ld_lines_indexed,
859 lf->end());
860 index_size += lf->size();
861 }
862
863 file_off_t index_off = 0;
864 merge.execute();
865 if (this->lss_sorting_observer) {
866 this->lss_sorting_observer(*this, index_off, index_size);
867 }
868 for (;;) {
869 logfile::iterator lf_iter;
870 logfile_data *ld;
871
872 if (!merge.get_top(ld, lf_iter)) {
873 break;
874 }
875
876 int file_index = ld->ld_file_index;
877 int line_index = lf_iter - ld->get_file_ptr()->begin();
878
879 content_line_t con_line(file_index * MAX_LINES_PER_FILE +
880 line_index);
881
882 this->lss_index.push_back(con_line);
883
884 merge.next();
885 index_off += 1;
886 if (index_off % 10000 == 0 && this->lss_sorting_observer) {
887 this->lss_sorting_observer(*this, index_off, index_size);
888 }
889 }
890 if (this->lss_sorting_observer) {
891 this->lss_sorting_observer(*this, index_size, index_size);
892 }
893 }
894
895 for (iter = this->lss_files.begin();
896 iter != this->lss_files.end();
897 iter++) {
898 auto lf = (*iter)->get_file_ptr();
899
900 if (lf == nullptr) {
901 continue;
902 }
903
904 (*iter)->ld_lines_indexed = lf->size();
905 }
906
907 this->lss_filtered_index.reserve(this->lss_index.size());
908
909 uint32_t filter_in_mask, filter_out_mask;
910 this->get_filters().get_enabled_mask(filter_in_mask, filter_out_mask);
911
912 if (start_size == 0 && this->lss_index_delegate != nullptr) {
913 this->lss_index_delegate->index_start(*this);
914 }
915
916 for (size_t index_index = start_size;
917 index_index < this->lss_index.size();
918 index_index++) {
919 content_line_t cl = (content_line_t) this->lss_index[index_index];
920 uint64_t line_number;
921 auto ld = this->find_data(cl, line_number);
922
923 if (!(*ld)->is_visible()) {
924 continue;
925 }
926
927 auto lf = (*ld)->get_file_ptr();
928 auto line_iter = lf->begin() + line_number;
929
930 if (line_iter->is_ignored()) {
931 continue;
932 }
933
934 if (!this->tss_apply_filters ||
935 (!(*ld)->ld_filter_state.excluded(filter_in_mask, filter_out_mask,
936 line_number) &&
937 this->check_extra_filters(ld, line_iter))) {
938 auto eval_res = this->eval_sql_filter(this->lss_marker_stmt.in(),
939 ld, line_iter);
940 if (eval_res.isErr()) {
941 line_iter->set_expr_mark(false);
942 } else {
943 auto matched = eval_res.unwrap();
944
945 if (matched) {
946 line_iter->set_expr_mark(true);
947 vis_bm[&textview_curses::BM_USER_EXPR]
948 .insert_once(vis_line_t(this->lss_filtered_index.size()));
949 }
950 else {
951 line_iter->set_expr_mark(false);
952 }
953 }
954 this->lss_filtered_index.push_back(index_index);
955 if (this->lss_index_delegate != nullptr) {
956 this->lss_index_delegate->index_line(
957 *this, lf, lf->begin() + line_number);
958 }
959 }
960 }
961
962 if (this->lss_index_delegate != nullptr) {
963 this->lss_index_delegate->index_complete(*this);
964 }
965 }
966
967 switch (retval) {
968 case rebuild_result::rr_no_change:
969 break;
970 case rebuild_result::rr_full_rebuild:
971 log_debug("redoing search");
972 this->tss_view->redo_search();
973 break;
974 case rebuild_result::rr_partial_rebuild:
975 log_debug("redoing search from: %d", (int) search_start);
976 this->tss_view->search_new_data(search_start);
977 break;
978 case rebuild_result::rr_appended_lines:
979 this->tss_view->search_new_data();
980 break;
981 }
982
983 return retval;
984 }
985
text_update_marks(vis_bookmarks & bm)986 void logfile_sub_source::text_update_marks(vis_bookmarks &bm)
987 {
988 shared_ptr<logfile> last_file = nullptr;
989 vis_line_t vl;
990
991 bm[&BM_WARNINGS].clear();
992 bm[&BM_ERRORS].clear();
993 bm[&BM_FILES].clear();
994
995 for (auto &lss_user_mark : this->lss_user_marks) {
996 bm[lss_user_mark.first].clear();
997 }
998
999 for (; vl < (int)this->lss_filtered_index.size(); ++vl) {
1000 const content_line_t orig_cl = this->at(vl);
1001 content_line_t cl = orig_cl;
1002 shared_ptr<logfile> lf;
1003
1004 lf = this->find(cl);
1005
1006 for (auto &lss_user_mark : this->lss_user_marks) {
1007 if (binary_search(lss_user_mark.second.begin(),
1008 lss_user_mark.second.end(),
1009 orig_cl)) {
1010 bm[lss_user_mark.first].insert_once(vl);
1011
1012 if (lss_user_mark.first == &textview_curses::BM_USER) {
1013 auto ll = lf->begin() + cl;
1014
1015 ll->set_mark(true);
1016 }
1017 }
1018 }
1019
1020 if (lf != last_file) {
1021 bm[&BM_FILES].insert_once(vl);
1022 }
1023
1024 auto line_iter = lf->begin() + cl;
1025 if (line_iter->is_message()) {
1026 switch (line_iter->get_msg_level()) {
1027 case LEVEL_WARNING:
1028 bm[&BM_WARNINGS].insert_once(vl);
1029 break;
1030
1031 case LEVEL_FATAL:
1032 case LEVEL_ERROR:
1033 case LEVEL_CRITICAL:
1034 bm[&BM_ERRORS].insert_once(vl);
1035 break;
1036
1037 default:
1038 break;
1039 }
1040 }
1041
1042 last_file = lf;
1043 }
1044 }
1045
get_line_accel_direction(vis_line_t vl)1046 log_accel::direction_t logfile_sub_source::get_line_accel_direction(
1047 vis_line_t vl)
1048 {
1049 log_accel la;
1050
1051 while (vl >= 0) {
1052 logline *curr_line = this->find_line(this->at(vl));
1053
1054 if (!curr_line->is_message()) {
1055 --vl;
1056 continue;
1057 }
1058
1059 if (!la.add_point(curr_line->get_time_in_millis())) {
1060 break;
1061 }
1062
1063 --vl;
1064 }
1065
1066 return la.get_direction();
1067 }
1068
text_filters_changed()1069 void logfile_sub_source::text_filters_changed()
1070 {
1071 if (this->lss_line_meta_changed) {
1072 this->invalidate_sql_filter();
1073 this->lss_line_meta_changed = false;
1074 }
1075
1076 for (auto& ld : *this) {
1077 auto lf = ld->get_file_ptr();
1078
1079 if (lf != nullptr) {
1080 ld->ld_filter_state.clear_deleted_filter_state();
1081 lf->reobserve_from(lf->begin() + ld->ld_filter_state.get_min_count(lf->size()));
1082 }
1083 }
1084
1085 auto& vis_bm = this->tss_view->get_bookmarks();
1086 uint32_t filtered_in_mask, filtered_out_mask;
1087
1088 this->get_filters().get_enabled_mask(filtered_in_mask, filtered_out_mask);
1089
1090 if (this->lss_index_delegate != nullptr) {
1091 this->lss_index_delegate->index_start(*this);
1092 }
1093 vis_bm[&textview_curses::BM_USER_EXPR].clear();
1094
1095 this->lss_filtered_index.clear();
1096 for (size_t index_index = 0; index_index < this->lss_index.size(); index_index++) {
1097 content_line_t cl = (content_line_t) this->lss_index[index_index];
1098 uint64_t line_number;
1099 auto ld = this->find_data(cl, line_number);
1100
1101 if (!(*ld)->is_visible()) {
1102 continue;
1103 }
1104
1105 auto lf = (*ld)->get_file_ptr();
1106 auto line_iter = lf->begin() + line_number;
1107
1108 if (!this->tss_apply_filters ||
1109 (!(*ld)->ld_filter_state.excluded(filtered_in_mask, filtered_out_mask,
1110 line_number) &&
1111 this->check_extra_filters(ld, line_iter))) {
1112 auto eval_res = this->eval_sql_filter(this->lss_marker_stmt.in(),
1113 ld, line_iter);
1114 if (eval_res.isErr()) {
1115 line_iter->set_expr_mark(false);
1116 } else {
1117 auto matched = eval_res.unwrap();
1118
1119 if (matched) {
1120 line_iter->set_expr_mark(true);
1121 vis_bm[&textview_curses::BM_USER_EXPR]
1122 .insert_once(vis_line_t(this->lss_filtered_index.size()));
1123 }
1124 else {
1125 line_iter->set_expr_mark(false);
1126 }
1127 }
1128 this->lss_filtered_index.push_back(index_index);
1129 if (this->lss_index_delegate != nullptr) {
1130 this->lss_index_delegate->index_line(*this, lf, line_iter);
1131 }
1132 }
1133 }
1134
1135 if (this->lss_index_delegate != nullptr) {
1136 this->lss_index_delegate->index_complete(*this);
1137 }
1138
1139 if (this->tss_view != nullptr) {
1140 this->tss_view->reload_data();
1141 this->tss_view->redo_search();
1142 }
1143 }
1144
list_input_handle_key(listview_curses & lv,int ch)1145 bool logfile_sub_source::list_input_handle_key(listview_curses &lv, int ch)
1146 {
1147 switch (ch) {
1148 case 'h':
1149 case 'H':
1150 case KEY_SLEFT:
1151 case KEY_LEFT:
1152 if (lv.get_left() == 0) {
1153 this->increase_line_context();
1154 lv.set_needs_update();
1155 return true;
1156 }
1157 break;
1158 case 'l':
1159 case 'L':
1160 case KEY_SRIGHT:
1161 case KEY_RIGHT:
1162 if (this->decrease_line_context()) {
1163 lv.set_needs_update();
1164 return true;
1165 }
1166 break;
1167 }
1168 return false;
1169 }
1170
1171 nonstd::optional<pair<grep_proc_source<vis_line_t> *, grep_proc_sink<vis_line_t> *>>
get_grepper()1172 logfile_sub_source::get_grepper()
1173 {
1174 return make_pair(
1175 (grep_proc_source<vis_line_t> *) &this->lss_meta_grepper,
1176 (grep_proc_sink<vis_line_t> *) &this->lss_meta_grepper
1177 );
1178 }
1179
insert_file(const shared_ptr<logfile> & lf)1180 bool logfile_sub_source::insert_file(const shared_ptr<logfile> &lf)
1181 {
1182 iterator existing;
1183
1184 require(lf->size() < MAX_LINES_PER_FILE);
1185
1186 existing = std::find_if(this->lss_files.begin(),
1187 this->lss_files.end(),
1188 logfile_data_eq(nullptr));
1189 if (existing == this->lss_files.end()) {
1190 if (this->lss_files.size() >= MAX_FILES) {
1191 return false;
1192 }
1193
1194 auto ld = std::make_unique<logfile_data>(
1195 this->lss_files.size(), this->get_filters(), lf);
1196 ld->set_visibility(lf->get_open_options().loo_is_visible);
1197 this->lss_files.push_back(std::move(ld));
1198 }
1199 else {
1200 (*existing)->set_file(lf);
1201 }
1202 this->lss_force_rebuild = true;
1203
1204 return true;
1205 }
1206
set_sql_filter(std::string stmt_str,sqlite3_stmt * stmt)1207 Result<void, std::string> logfile_sub_source::set_sql_filter(std::string stmt_str, sqlite3_stmt *stmt)
1208 {
1209 if (stmt != nullptr && !this->lss_filtered_index.empty()) {
1210 auto top_cl = this->at(0_vl);
1211 auto ld = this->find_data(top_cl);
1212 auto eval_res = this->eval_sql_filter(stmt, ld, (*ld)->get_file_ptr()->begin());
1213
1214 if (eval_res.isErr()) {
1215 sqlite3_finalize(stmt);
1216 return Err(eval_res.unwrapErr());
1217 }
1218 }
1219
1220 for (auto& ld : *this) {
1221 ld->ld_filter_state.lfo_filter_state.clear_filter_state(0);
1222 }
1223
1224 auto old_filter = this->get_sql_filter();
1225 if (stmt != nullptr) {
1226 auto new_filter = std::make_shared<sql_filter>(*this, stmt_str, stmt);
1227
1228 if (old_filter) {
1229 auto existing_iter = std::find(this->tss_filters.begin(),
1230 this->tss_filters.end(),
1231 old_filter.value());
1232 *existing_iter = new_filter;
1233 } else {
1234 this->tss_filters.add_filter(new_filter);
1235 }
1236 } else if (old_filter) {
1237 this->tss_filters.delete_filter(old_filter.value()->get_id());
1238 }
1239
1240 return Ok();
1241 }
1242
1243 Result<void, std::string>
set_sql_marker(std::string stmt_str,sqlite3_stmt * stmt)1244 logfile_sub_source::set_sql_marker(std::string stmt_str, sqlite3_stmt *stmt)
1245 {
1246 if (stmt != nullptr && !this->lss_filtered_index.empty()) {
1247 auto top_cl = this->at(0_vl);
1248 auto ld = this->find_data(top_cl);
1249 auto eval_res = this->eval_sql_filter(stmt, ld, (*ld)->get_file_ptr()->begin());
1250
1251 if (eval_res.isErr()) {
1252 sqlite3_finalize(stmt);
1253 return Err(eval_res.unwrapErr());
1254 }
1255 }
1256
1257 auto& vis_bm = this->tss_view->get_bookmarks();
1258 auto& expr_marks_bv = vis_bm[&textview_curses::BM_USER_EXPR];
1259
1260 expr_marks_bv.clear();
1261 this->lss_marker_stmt_text = std::move(stmt_str);
1262 this->lss_marker_stmt = stmt;
1263 if (this->lss_index_delegate) {
1264 this->lss_index_delegate->index_start(*this);
1265 }
1266 for (auto row = 0_vl; row < this->lss_filtered_index.size(); row += 1_vl) {
1267 auto cl = this->at(row);
1268 auto ld = this->find_data(cl);
1269 auto ll = (*ld)->get_file()->begin() + cl;
1270 auto eval_res = this->eval_sql_filter(this->lss_marker_stmt.in(), ld, ll);
1271
1272 if (eval_res.isErr()) {
1273 ll->set_expr_mark(false);
1274 } else {
1275 auto matched = eval_res.unwrap();
1276
1277 if (matched) {
1278 ll->set_expr_mark(true);
1279 expr_marks_bv.insert_once(row);
1280 } else {
1281 ll->set_expr_mark(false);
1282 }
1283 }
1284 if (this->lss_index_delegate) {
1285 this->lss_index_delegate->index_line(*this, (*ld)->get_file_ptr(), ll);
1286 }
1287 }
1288 if (this->lss_index_delegate) {
1289 this->lss_index_delegate->index_complete(*this);
1290 }
1291
1292 return Ok();
1293 }
1294
1295 Result<void, std::string>
set_preview_sql_filter(sqlite3_stmt * stmt)1296 logfile_sub_source::set_preview_sql_filter(sqlite3_stmt *stmt)
1297 {
1298 if (stmt != nullptr && !this->lss_filtered_index.empty()) {
1299 auto top_cl = this->at(0_vl);
1300 auto ld = this->find_data(top_cl);
1301 auto eval_res = this->eval_sql_filter(stmt, ld, (*ld)->get_file_ptr()->begin());
1302
1303 if (eval_res.isErr()) {
1304 sqlite3_finalize(stmt);
1305 return Err(eval_res.unwrapErr());
1306 }
1307 }
1308
1309 this->lss_preview_filter_stmt = stmt;
1310
1311 return Ok();
1312 }
1313
1314 Result<bool, std::string>
eval_sql_filter(sqlite3_stmt * stmt,iterator ld,logfile::const_iterator ll)1315 logfile_sub_source::eval_sql_filter(sqlite3_stmt *stmt, iterator ld, logfile::const_iterator ll)
1316 {
1317 if (stmt == nullptr) {
1318 return Ok(false);
1319 }
1320
1321 auto lf = (*ld)->get_file_ptr();
1322 char timestamp_buffer[64];
1323 shared_buffer_ref sbr, raw_sbr;
1324 lf->read_full_message(ll, sbr);
1325 auto format = lf->get_format();
1326 string_attrs_t sa;
1327 vector<logline_value> values;
1328 format->annotate(std::distance(lf->cbegin(), ll), sbr, sa, values);
1329
1330 sqlite3_reset(stmt);
1331 sqlite3_clear_bindings(stmt);
1332
1333 auto count = sqlite3_bind_parameter_count(stmt);
1334 for (int lpc = 0; lpc < count; lpc++) {
1335 auto *name = sqlite3_bind_parameter_name(stmt, lpc + 1);
1336
1337 if (name[0] == '$') {
1338 const char *env_value;
1339
1340 if ((env_value = getenv(&name[1])) != nullptr) {
1341 sqlite3_bind_text(stmt, lpc + 1, env_value, -1, SQLITE_STATIC);
1342 }
1343 continue;
1344 }
1345 if (strcmp(name, ":log_level") == 0) {
1346 sqlite3_bind_text(stmt,
1347 lpc + 1,
1348 ll->get_level_name(), -1,
1349 SQLITE_STATIC);
1350 continue;
1351 }
1352 if (strcmp(name, ":log_time") == 0) {
1353 auto len = sql_strftime(timestamp_buffer, sizeof(timestamp_buffer),
1354 ll->get_timeval(),
1355 'T');
1356 sqlite3_bind_text(stmt,
1357 lpc + 1,
1358 timestamp_buffer, len,
1359 SQLITE_STATIC);
1360 continue;
1361 }
1362 if (strcmp(name, ":log_time_msecs") == 0) {
1363 sqlite3_bind_int64(stmt, lpc + 1, ll->get_time_in_millis());
1364 continue;
1365 }
1366 if (strcmp(name, ":log_mark") == 0) {
1367 sqlite3_bind_int(stmt, lpc + 1, ll->is_marked());
1368 continue;
1369 }
1370 if (strcmp(name, ":log_comment") == 0) {
1371 const auto &bm = this->get_user_bookmark_metadata();
1372 auto cl = this->get_file_base_content_line(ld);
1373 cl += content_line_t(std::distance(lf->cbegin(), ll));
1374 auto bm_iter = bm.find(cl);
1375 if (bm_iter != bm.end() && !bm_iter->second.bm_comment.empty()) {
1376 const auto &meta = bm_iter->second;
1377 sqlite3_bind_text(stmt,
1378 lpc + 1,
1379 meta.bm_comment.c_str(),
1380 meta.bm_comment.length(),
1381 SQLITE_STATIC);
1382 }
1383 continue;
1384 }
1385 if (strcmp(name, ":log_tags") == 0) {
1386 const auto &bm = this->get_user_bookmark_metadata();
1387 auto cl = this->get_file_base_content_line(ld);
1388 cl += content_line_t(std::distance(lf->cbegin(), ll));
1389 auto bm_iter = bm.find(cl);
1390 if (bm_iter != bm.end() && !bm_iter->second.bm_tags.empty()) {
1391 const auto &meta = bm_iter->second;
1392 yajlpp_gen gen;
1393
1394 yajl_gen_config(gen, yajl_gen_beautify, false);
1395
1396 {
1397 yajlpp_array arr(gen);
1398
1399 for (const auto &str : meta.bm_tags) {
1400 arr.gen(str);
1401 }
1402 }
1403
1404 string_fragment sf = gen.to_string_fragment();
1405
1406 sqlite3_bind_text(stmt,
1407 lpc + 1,
1408 sf.data(),
1409 sf.length(),
1410 SQLITE_TRANSIENT);
1411 }
1412 continue;
1413 }
1414 if (strcmp(name, ":log_path") == 0) {
1415 const auto& filename = lf->get_filename();
1416 sqlite3_bind_text(stmt,
1417 lpc + 1,
1418 filename.c_str(), filename.length(),
1419 SQLITE_STATIC);
1420 continue;
1421 }
1422 if (strcmp(name, ":log_text") == 0) {
1423 sqlite3_bind_text(stmt,
1424 lpc + 1,
1425 sbr.get_data(), sbr.length(),
1426 SQLITE_STATIC);
1427 continue;
1428 }
1429 if (strcmp(name, ":log_body") == 0) {
1430 auto iter = find_string_attr(sa, &SA_BODY);
1431 sqlite3_bind_text(stmt,
1432 lpc + 1,
1433 &(sbr.get_data()[iter->sa_range.lr_start]),
1434 iter->sa_range.length(),
1435 SQLITE_STATIC);
1436 continue;
1437 }
1438 if (strcmp(name, ":log_raw_text") == 0) {
1439 auto res = lf->read_raw_message(ll);
1440
1441 if (res.isOk()) {
1442 raw_sbr = res.unwrap();
1443 sqlite3_bind_text(stmt,
1444 lpc + 1,
1445 raw_sbr.get_data(),
1446 raw_sbr.length(),
1447 SQLITE_STATIC);
1448 }
1449 continue;
1450 }
1451 for (auto& lv : values) {
1452 if (lv.lv_meta.lvm_name != &name[1]) {
1453 continue;
1454 }
1455
1456 switch (lv.lv_meta.lvm_kind) {
1457 case value_kind_t::VALUE_BOOLEAN:
1458 sqlite3_bind_int64(stmt, lpc + 1, lv.lv_value.i);
1459 break;
1460 case value_kind_t::VALUE_FLOAT:
1461 sqlite3_bind_double(stmt, lpc + 1, lv.lv_value.d);
1462 break;
1463 case value_kind_t::VALUE_INTEGER:
1464 sqlite3_bind_int64(stmt, lpc + 1, lv.lv_value.i);
1465 break;
1466 case value_kind_t::VALUE_NULL:
1467 sqlite3_bind_null(stmt, lpc + 1);
1468 break;
1469 default:
1470 sqlite3_bind_text(stmt,
1471 lpc + 1,
1472 lv.text_value(),
1473 lv.text_length(),
1474 SQLITE_TRANSIENT);
1475 break;
1476 }
1477 break;
1478 }
1479 }
1480
1481 auto step_res = sqlite3_step(stmt);
1482
1483 sqlite3_reset(stmt);
1484 sqlite3_clear_bindings(stmt);
1485 switch (step_res) {
1486 case SQLITE_OK:
1487 case SQLITE_DONE:
1488 return Ok(false);
1489 case SQLITE_ROW:
1490 return Ok(true);
1491 default:
1492 return Err(std::string(sqlite3_errmsg(sqlite3_db_handle(stmt))));
1493 }
1494
1495 return Ok(true);
1496 }
1497
check_extra_filters(iterator ld,logfile::iterator ll)1498 bool logfile_sub_source::check_extra_filters(iterator ld, logfile::iterator ll)
1499 {
1500 if (this->lss_marked_only && !(ll->is_marked() || ll->is_expr_marked())) {
1501 return false;
1502 }
1503
1504 if (ll->get_msg_level() < this->lss_min_log_level) {
1505 return false;
1506 }
1507
1508 if (*ll < this->lss_min_log_time) {
1509 return false;
1510 }
1511
1512 if (!(*ll <= this->lss_max_log_time)) {
1513 return false;
1514 }
1515
1516 return true;
1517 }
1518
invalidate_sql_filter()1519 void logfile_sub_source::invalidate_sql_filter()
1520 {
1521 for (auto& ld : *this) {
1522 ld->ld_filter_state.lfo_filter_state.clear_filter_state(0);
1523 }
1524 }
1525
loc_history_append(vis_line_t top)1526 void log_location_history::loc_history_append(vis_line_t top)
1527 {
1528 if (top >= vis_line_t(this->llh_log_source.text_line_count())) {
1529 return;
1530 }
1531
1532 content_line_t cl = this->llh_log_source.at(top);
1533
1534 auto iter = this->llh_history.begin();
1535 iter += this->llh_history.size() - this->lh_history_position;
1536 this->llh_history.erase_from(iter);
1537 this->lh_history_position = 0;
1538 this->llh_history.push_back(cl);
1539 }
1540
loc_history_back(vis_line_t current_top)1541 nonstd::optional<vis_line_t> log_location_history::loc_history_back(vis_line_t current_top)
1542 {
1543 while (this->lh_history_position < this->llh_history.size()) {
1544 auto iter = this->llh_history.rbegin();
1545
1546 auto vis_for_pos = this->llh_log_source.find_from_content(*iter);
1547
1548 if (this->lh_history_position == 0 && vis_for_pos != current_top) {
1549 return vis_for_pos;
1550 }
1551
1552 if ((this->lh_history_position + 1) >= this->llh_history.size()) {
1553 break;
1554 }
1555
1556 this->lh_history_position += 1;
1557
1558 iter += this->lh_history_position;
1559
1560 vis_for_pos = this->llh_log_source.find_from_content(*iter);
1561
1562 if (vis_for_pos) {
1563 return vis_for_pos;
1564 }
1565 }
1566
1567 return nonstd::nullopt;
1568 }
1569
1570 nonstd::optional<vis_line_t>
loc_history_forward(vis_line_t current_top)1571 log_location_history::loc_history_forward(vis_line_t current_top)
1572 {
1573 while (this->lh_history_position > 0) {
1574 this->lh_history_position -= 1;
1575
1576 auto iter = this->llh_history.rbegin();
1577
1578 iter += this->lh_history_position;
1579
1580 auto vis_for_pos = this->llh_log_source.find_from_content(*iter);
1581
1582 if (vis_for_pos) {
1583 return vis_for_pos;
1584 }
1585 }
1586
1587 return nonstd::nullopt;
1588 }
1589
matches(const logfile & lf,logfile::const_iterator ll,shared_buffer_ref & line)1590 bool sql_filter::matches(const logfile &lf, logfile::const_iterator ll,
1591 shared_buffer_ref &line)
1592 {
1593 if (!ll->is_message()) {
1594 return false;
1595 }
1596 if (this->sf_filter_stmt == nullptr) {
1597 return false;
1598 }
1599
1600 auto lfp = lf.shared_from_this();
1601 auto ld = this->sf_log_source.find_data_i(lfp);
1602 if (ld == this->sf_log_source.end()) {
1603 return false;
1604 }
1605
1606 auto eval_res = this->sf_log_source.eval_sql_filter(this->sf_filter_stmt, ld, ll);
1607 if (eval_res.unwrapOr(true)) {
1608 return false;
1609 }
1610
1611 return true;
1612 }
1613
to_command()1614 std::string sql_filter::to_command()
1615 {
1616 return fmt::format("filter-expr {}", this->lf_id);
1617 }
1618