1 /**
2 * Copyright (c) 2007-2012, Timothy Stack
3 *
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * * Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 * * Neither the name of Timothy Stack nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
25 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 #include "config.h"
31
32 #include "base/lnav_log.hh"
33 #include "base/string_util.hh"
34 #include "sql_util.hh"
35 #include "log_vtab_impl.hh"
36 #include "yajlpp/json_op.hh"
37 #include "yajlpp/yajlpp_def.hh"
38 #include "vtab_module.hh"
39
40 #include "logfile_sub_source.hh"
41
42 using namespace std;
43
44 static auto intern_lifetime = intern_string::get_table_lifetime();
45
46 static struct log_cursor log_cursor_latest;
47
48 struct _log_vtab_data log_vtab_data;
49
50 static const char *LOG_COLUMNS = R"( (
51 log_line INTEGER PRIMARY KEY, -- The line number for the log message
52 log_part TEXT COLLATE naturalnocase, -- The partition the message is in
53 log_time DATETIME, -- The adjusted timestamp for the log message
54 log_actual_time DATETIME HIDDEN, -- The timestamp from the original log file for this message
55 log_idle_msecs INTEGER, -- The difference in time between this messages and the previous
56 log_level TEXT COLLATE loglevel, -- The log message level
57 log_mark BOOLEAN, -- True if the log message was marked
58 log_comment TEXT, -- The comment for this message
59 log_tags TEXT, -- A JSON list of tags for this message
60 log_filters TEXT, -- A JSON list of filter IDs that matched this message
61 -- BEGIN Format-specific fields:
62 )";
63
64 static const char *LOG_FOOTER_COLUMNS = R"(
65 -- END Format-specific fields
66 log_time_msecs INTEGER HIDDEN, -- The adjusted timestamp for the log message as the number of milliseconds from the epoch
67 log_path TEXT HIDDEN COLLATE naturalnocase, -- The path to the log file this message is from
68 log_text TEXT HIDDEN, -- The full text of the log message
69 log_body TEXT HIDDEN, -- The body of the log message
70 log_raw_text TEXT HIDDEN -- The raw text from the log file
71 );
72 )";
73
type_to_string(int type)74 static const char *type_to_string(int type)
75 {
76 switch (type) {
77 case SQLITE_FLOAT:
78 return "FLOAT";
79
80 case SQLITE_INTEGER:
81 return "INTEGER";
82
83 case SQLITE_TEXT:
84 return "TEXT";
85 }
86
87 ensure("Invalid sqlite type");
88
89 return nullptr;
90 }
91
get_table_statement()92 std::string log_vtab_impl::get_table_statement()
93 {
94 std::vector<log_vtab_impl::vtab_column> cols;
95 std::vector<log_vtab_impl::vtab_column>::const_iterator iter;
96 std::ostringstream oss;
97 size_t max_name_len = 15;
98
99 oss << "CREATE TABLE " << this->get_name().to_string() << LOG_COLUMNS;
100 this->get_columns(cols);
101 this->vi_column_count = cols.size();
102 for (iter = cols.begin(); iter != cols.end(); iter++) {
103 max_name_len = std::max(max_name_len, iter->vc_name.length());
104 }
105 for (iter = cols.begin(); iter != cols.end(); iter++) {
106 auto_mem<char, sqlite3_free> coldecl;
107 auto_mem<char, sqlite3_free> colname;
108 string comment;
109
110 require(!iter->vc_name.empty());
111
112 if (!iter->vc_comment.empty()) {
113 comment.append(" -- ")
114 .append(iter->vc_comment);
115 }
116
117 colname = sql_quote_ident(iter->vc_name.c_str());
118 coldecl = sqlite3_mprintf(" %-*s %-7s %s COLLATE %-15Q,%s\n",
119 max_name_len,
120 colname.in(),
121 type_to_string(iter->vc_type),
122 iter->vc_hidden ? "hidden" : "",
123 iter->vc_collator.empty() ?
124 "BINARY" : iter->vc_collator.c_str(),
125 comment.c_str());
126 oss << coldecl;
127 }
128 oss << LOG_FOOTER_COLUMNS;
129
130 log_debug("log_vtab_impl.get_table_statement() -> %s", oss.str().c_str());
131
132 return oss.str();
133 }
134
logline_value_to_sqlite_type(value_kind_t kind)135 pair<int, unsigned int> log_vtab_impl::logline_value_to_sqlite_type(value_kind_t kind)
136 {
137 int type = 0;
138 unsigned int subtype = 0;
139
140 switch (kind) {
141 case value_kind_t::VALUE_JSON:
142 type = SQLITE3_TEXT;
143 subtype = 74;
144 break;
145 case value_kind_t::VALUE_NULL:
146 case value_kind_t::VALUE_TEXT:
147 case value_kind_t::VALUE_STRUCT:
148 case value_kind_t::VALUE_QUOTED:
149 case value_kind_t::VALUE_W3C_QUOTED:
150 case value_kind_t::VALUE_TIMESTAMP:
151 case value_kind_t::VALUE_XML:
152 type = SQLITE3_TEXT;
153 break;
154 case value_kind_t::VALUE_FLOAT:
155 type = SQLITE_FLOAT;
156 break;
157 case value_kind_t::VALUE_BOOLEAN:
158 case value_kind_t::VALUE_INTEGER:
159 type = SQLITE_INTEGER;
160 break;
161 case value_kind_t::VALUE_UNKNOWN:
162 case value_kind_t::VALUE__MAX:
163 ensure(0);
164 break;
165 }
166 return make_pair(type, subtype);
167 }
168
169 struct vtab {
170 sqlite3_vtab base;
171 sqlite3 * db;
172 textview_curses *tc{nullptr};
173 logfile_sub_source *lss{nullptr};
174 std::shared_ptr<log_vtab_impl> vi;
175 };
176
177 struct vtab_cursor {
178 sqlite3_vtab_cursor base;
179 struct log_cursor log_cursor;
180 shared_buffer_ref log_msg;
181 std::vector<logline_value> line_values;
182 };
183
184 static int vt_destructor(sqlite3_vtab *p_svt);
185
vt_create(sqlite3 * db,void * pAux,int argc,const char * const * argv,sqlite3_vtab ** pp_vt,char ** pzErr)186 static int vt_create(sqlite3 *db,
187 void *pAux,
188 int argc, const char *const *argv,
189 sqlite3_vtab **pp_vt,
190 char **pzErr)
191 {
192 auto *vm = (log_vtab_manager *)pAux;
193 int rc = SQLITE_OK;
194 /* Allocate the sqlite3_vtab/vtab structure itself */
195 auto p_vt = std::make_unique<vtab>();
196
197 p_vt->db = db;
198
199 /* Declare the vtable's structure */
200 p_vt->vi = vm->lookup_impl(intern_string::lookup(argv[3]));
201 if (p_vt->vi == nullptr) {
202 return SQLITE_ERROR;
203 }
204 p_vt->tc = vm->get_view();
205 p_vt->lss = vm->get_source();
206 rc = sqlite3_declare_vtab(db, p_vt->vi->get_table_statement().c_str());
207
208 /* Success. Set *pp_vt and return */
209 auto loose_p_vt = p_vt.release();
210 *pp_vt = &loose_p_vt->base;
211
212 log_debug("creating log format table: %s = %p", argv[3], p_vt.get());
213
214 return rc;
215 }
216
vt_destructor(sqlite3_vtab * p_svt)217 static int vt_destructor(sqlite3_vtab *p_svt)
218 {
219 vtab *p_vt = (vtab *)p_svt;
220
221 delete p_vt;
222
223 return SQLITE_OK;
224 }
225
vt_connect(sqlite3 * db,void * p_aux,int argc,const char * const * argv,sqlite3_vtab ** pp_vt,char ** pzErr)226 static int vt_connect(sqlite3 *db, void *p_aux,
227 int argc, const char *const *argv,
228 sqlite3_vtab **pp_vt, char **pzErr)
229 {
230 return vt_create(db, p_aux, argc, argv, pp_vt, pzErr);
231 }
232
vt_disconnect(sqlite3_vtab * pVtab)233 static int vt_disconnect(sqlite3_vtab *pVtab)
234 {
235 return vt_destructor(pVtab);
236 }
237
vt_destroy(sqlite3_vtab * p_vt)238 static int vt_destroy(sqlite3_vtab *p_vt)
239 {
240 return vt_destructor(p_vt);
241 }
242
243 static int vt_next(sqlite3_vtab_cursor *cur);
244
vt_open(sqlite3_vtab * p_svt,sqlite3_vtab_cursor ** pp_cursor)245 static int vt_open(sqlite3_vtab *p_svt, sqlite3_vtab_cursor **pp_cursor)
246 {
247 vtab *p_vt = (vtab *)p_svt;
248
249 p_vt->base.zErrMsg = NULL;
250
251 vtab_cursor *p_cur = new vtab_cursor();
252
253 *pp_cursor = (sqlite3_vtab_cursor *)p_cur;
254
255 p_cur->base.pVtab = p_svt;
256 p_cur->log_cursor.lc_curr_line = -1_vl;
257 p_cur->log_cursor.lc_end_line = vis_line_t(p_vt->lss->text_line_count());
258 p_cur->log_cursor.lc_sub_index = 0;
259 vt_next((sqlite3_vtab_cursor *)p_cur);
260
261 return SQLITE_OK;
262 }
263
vt_close(sqlite3_vtab_cursor * cur)264 static int vt_close(sqlite3_vtab_cursor *cur)
265 {
266 vtab_cursor *p_cur = (vtab_cursor *)cur;
267
268 /* Free cursor struct. */
269 delete p_cur;
270
271 return SQLITE_OK;
272 }
273
vt_eof(sqlite3_vtab_cursor * cur)274 static int vt_eof(sqlite3_vtab_cursor *cur)
275 {
276 vtab_cursor *vc = (vtab_cursor *)cur;
277
278 return vc->log_cursor.is_eof();
279 }
280
vt_next(sqlite3_vtab_cursor * cur)281 static int vt_next(sqlite3_vtab_cursor *cur)
282 {
283 vtab_cursor *vc = (vtab_cursor *)cur;
284 vtab * vt = (vtab *)cur->pVtab;
285 bool done = false;
286
287 vc->line_values.clear();
288 do {
289 log_cursor_latest = vc->log_cursor;
290 if (((log_cursor_latest.lc_curr_line % 1024) == 0) &&
291 (log_vtab_data.lvd_progress != NULL &&
292 log_vtab_data.lvd_progress(log_cursor_latest))) {
293 break;
294 }
295 done = vt->vi->next(vc->log_cursor, *vt->lss);
296 } while (!done);
297
298 return SQLITE_OK;
299 }
300
vt_column(sqlite3_vtab_cursor * cur,sqlite3_context * ctx,int col)301 static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
302 {
303 vtab_cursor *vc = (vtab_cursor *)cur;
304 vtab * vt = (vtab *)cur->pVtab;
305
306 content_line_t cl(vt->lss->at(vc->log_cursor.lc_curr_line));
307 uint64_t line_number;
308 auto ld = vt->lss->find_data(cl, line_number);
309 shared_ptr<logfile> lf = (*ld)->get_file();
310 auto ll = lf->begin() + line_number;
311
312 require(col >= 0);
313
314 /* Just return the ordinal of the column requested. */
315 switch (col) {
316 case VT_COL_LINE_NUMBER:
317 {
318 sqlite3_result_int64(ctx, vc->log_cursor.lc_curr_line);
319 }
320 break;
321
322 case VT_COL_PARTITION:
323 {
324 vis_bookmarks &vb = vt->tc->get_bookmarks();
325 bookmark_vector<vis_line_t> &bv = vb[&textview_curses::BM_META];
326
327 if (bv.empty()) {
328 sqlite3_result_null(ctx);
329 }
330 else {
331 vis_line_t curr_line(vc->log_cursor.lc_curr_line);
332 auto iter = lower_bound(bv.begin(), bv.end(), curr_line + 1_vl);
333
334 if (iter != bv.begin()) {
335 --iter;
336 content_line_t part_line = vt->lss->at(*iter);
337 std::map<content_line_t, bookmark_metadata> &bm_meta = vt->lss->get_user_bookmark_metadata();
338 std::map<content_line_t, bookmark_metadata>::iterator meta_iter;
339
340 meta_iter = bm_meta.find(part_line);
341 if (meta_iter != bm_meta.end() &&
342 !meta_iter->second.bm_name.empty()) {
343 sqlite3_result_text(ctx,
344 meta_iter->second.bm_name.c_str(),
345 meta_iter->second.bm_name.size(),
346 SQLITE_TRANSIENT);
347 } else {
348 sqlite3_result_null(ctx);
349 }
350 } else {
351 sqlite3_result_null(ctx);
352 }
353 }
354 }
355 break;
356
357 case VT_COL_LOG_TIME:
358 {
359 char buffer[64];
360
361 sql_strftime(buffer, sizeof(buffer), ll->get_time(), ll->get_millis());
362 sqlite3_result_text(ctx, buffer, strlen(buffer), SQLITE_TRANSIENT);
363 }
364 break;
365
366 case VT_COL_LOG_ACTUAL_TIME: {
367 char buffer[64];
368
369 if (ll->is_time_skewed()) {
370 if (vc->line_values.empty()) {
371 lf->read_full_message(ll, vc->log_msg);
372 vt->vi->extract(lf, line_number, vc->log_msg, vc->line_values);
373 }
374
375 struct line_range time_range;
376
377 time_range = find_string_attr_range(
378 vt->vi->vi_attrs, &logline::L_TIMESTAMP);
379
380 const char *time_src = vc->log_msg.get_data() + time_range.lr_start;
381 struct timeval actual_tv;
382 struct exttm tm;
383
384 if (lf->get_format()->lf_date_time.scan(
385 time_src, time_range.length(),
386 lf->get_format()->get_timestamp_formats(),
387 &tm, actual_tv,
388 false)) {
389 sql_strftime(buffer, sizeof(buffer), actual_tv);
390 }
391 }
392 else {
393 sql_strftime(buffer, sizeof(buffer), ll->get_time(), ll->get_millis());
394 }
395 sqlite3_result_text(ctx, buffer, strlen(buffer), SQLITE_TRANSIENT);
396 break;
397 }
398
399 case VT_COL_IDLE_MSECS:
400 if (vc->log_cursor.lc_curr_line == 0) {
401 sqlite3_result_int64(ctx, 0);
402 }
403 else {
404 content_line_t prev_cl(vt->lss->at(vis_line_t(
405 vc->log_cursor.lc_curr_line -
406 1)));
407 shared_ptr<logfile> prev_lf = vt->lss->find(prev_cl);
408 logfile::iterator prev_ll = prev_lf->begin() + prev_cl;
409 uint64_t prev_time, curr_line_time;
410
411 prev_time = prev_ll->get_time() * 1000ULL;
412 prev_time += prev_ll->get_millis();
413 curr_line_time = ll->get_time() * 1000ULL;
414 curr_line_time += ll->get_millis();
415 // require(curr_line_time >= prev_time);
416 sqlite3_result_int64(ctx, curr_line_time - prev_time);
417 }
418 break;
419
420 case VT_COL_LEVEL:
421 {
422 const char *level_name = ll->get_level_name();
423
424 sqlite3_result_text(ctx,
425 level_name,
426 strlen(level_name),
427 SQLITE_STATIC);
428 }
429 break;
430
431 case VT_COL_MARK:
432 {
433 sqlite3_result_int(ctx, ll->is_marked());
434 }
435 break;
436
437 case VT_COL_LOG_COMMENT: {
438 const auto &bm = vt->lss->get_user_bookmark_metadata();
439
440 auto bm_iter = bm.find(vt->lss->at(vc->log_cursor.lc_curr_line));
441 if (bm_iter == bm.end() || bm_iter->second.bm_comment.empty()) {
442 sqlite3_result_null(ctx);
443 } else {
444 const bookmark_metadata &meta = bm_iter->second;
445 sqlite3_result_text(ctx,
446 meta.bm_comment.c_str(),
447 meta.bm_comment.length(),
448 SQLITE_TRANSIENT);
449 }
450 break;
451 }
452
453 case VT_COL_LOG_TAGS: {
454 const map<content_line_t, bookmark_metadata> &bm = vt->lss->get_user_bookmark_metadata();
455
456 auto bm_iter = bm.find(vt->lss->at(vc->log_cursor.lc_curr_line));
457 if (bm_iter == bm.end() || bm_iter->second.bm_tags.empty()) {
458 sqlite3_result_null(ctx);
459 } else {
460 const bookmark_metadata &meta = bm_iter->second;
461
462 yajlpp_gen gen;
463
464 yajl_gen_config(gen, yajl_gen_beautify, false);
465
466 {
467 yajlpp_array arr(gen);
468
469 for (const auto &str : meta.bm_tags) {
470 arr.gen(str);
471 }
472 }
473
474 string_fragment sf = gen.to_string_fragment();
475
476 sqlite3_result_text(ctx,
477 sf.data(),
478 sf.length(),
479 SQLITE_TRANSIENT);
480 sqlite3_result_subtype(ctx, 'J');
481 }
482 break;
483 }
484
485 case VT_COL_FILTERS: {
486 auto &filter_mask = (*ld)->ld_filter_state.lfo_filter_state.tfs_mask;
487
488 if (!filter_mask[line_number]) {
489 sqlite3_result_null(ctx);
490 } else {
491 auto &filters = vt->lss->get_filters();
492 yajlpp_gen gen;
493
494 yajl_gen_config(gen, yajl_gen_beautify, false);
495
496 {
497 yajlpp_array arr(gen);
498
499 for (auto &filter : filters) {
500 if (filter->lf_deleted) {
501 continue;
502 }
503
504 uint32_t mask = (1UL << filter->get_index());
505
506 if (filter_mask[line_number] & mask) {
507 arr.gen(filter->get_index());
508 }
509 }
510 }
511
512 to_sqlite(ctx, gen.to_string_fragment());
513 sqlite3_result_subtype(ctx, 'J');
514 }
515 break;
516 }
517
518 default:
519 if (col > (VT_COL_MAX + vt->vi->vi_column_count - 1)) {
520 int post_col_number = col -
521 (VT_COL_MAX + vt->vi->vi_column_count -
522 1) - 1;
523
524 switch (post_col_number) {
525 case 0: {
526 sqlite3_result_int64(ctx, ll->get_time_in_millis());
527 break;
528 }
529 case 1: {
530 const string &fn = lf->get_filename();
531
532 sqlite3_result_text(ctx,
533 fn.c_str(),
534 fn.length(),
535 SQLITE_STATIC);
536 break;
537 }
538 case 2: {
539 shared_buffer_ref line;
540
541 lf->read_full_message(ll, line);
542 sqlite3_result_text(ctx,
543 line.get_data(),
544 line.length(),
545 SQLITE_TRANSIENT);
546 break;
547 }
548 case 3: {
549 if (vc->line_values.empty()) {
550 lf->read_full_message(ll, vc->log_msg);
551 vt->vi->extract(lf, line_number, vc->log_msg, vc->line_values);
552 }
553
554 struct line_range body_range;
555
556 body_range = find_string_attr_range(
557 vt->vi->vi_attrs, &SA_BODY);
558 if (!body_range.is_valid()) {
559 sqlite3_result_null(ctx);
560 }
561 else {
562 const char *msg_start = vc->log_msg.get_data();
563
564 sqlite3_result_text(ctx,
565 &msg_start[body_range.lr_start],
566 body_range.length(),
567 SQLITE_TRANSIENT);
568 }
569 break;
570 }
571 case 4: {
572 auto read_res = lf->read_raw_message(ll);
573
574 if (read_res.isErr()) {
575 auto msg = fmt::format("unable to read line -- {}",
576 read_res.unwrapErr());
577 sqlite3_result_error(ctx, msg.c_str(), msg.length());
578 } else {
579 auto sbr = read_res.unwrap();
580
581 sqlite3_result_text(ctx,
582 sbr.get_data(), sbr.length(),
583 SQLITE_TRANSIENT);
584 }
585 break;
586 }
587 }
588 }
589 else {
590 if (vc->line_values.empty()) {
591 lf->read_full_message(ll, vc->log_msg);
592 vt->vi->extract(lf, line_number, vc->log_msg, vc->line_values);
593 }
594
595 size_t sub_col = col - VT_COL_MAX;
596 std::vector<logline_value>::iterator lv_iter;
597
598 lv_iter = find_if(vc->line_values.begin(), vc->line_values.end(),
599 logline_value_cmp(NULL, sub_col));
600
601 if (lv_iter != vc->line_values.end()) {
602 if (!lv_iter->lv_meta.lvm_struct_name.empty()) {
603 yajlpp_gen gen;
604 yajl_gen_config(gen, yajl_gen_beautify, false);
605
606 {
607 yajlpp_map root(gen);
608
609 for (auto &lv_struct : vc->line_values) {
610 if (lv_struct.lv_meta.lvm_column != sub_col) {
611 continue;
612 }
613
614 root.gen(lv_struct.lv_meta.lvm_name);
615 switch (lv_struct.lv_meta.lvm_kind) {
616 case value_kind_t::VALUE_NULL:
617 root.gen();
618 break;
619 case value_kind_t::VALUE_BOOLEAN:
620 root.gen((bool) lv_struct.lv_value.i);
621 break;
622 case value_kind_t::VALUE_INTEGER:
623 root.gen(lv_struct.lv_value.i);
624 break;
625 case value_kind_t::VALUE_FLOAT:
626 root.gen(lv_struct.lv_value.d);
627 break;
628 case value_kind_t::VALUE_JSON: {
629 auto_mem<yajl_handle_t> parse_handle(yajl_free);
630 json_ptr jp("");
631 json_op jo(jp);
632
633 jo.jo_ptr_callbacks = json_op::gen_callbacks;
634 jo.jo_ptr_data = gen;
635 parse_handle.reset(yajl_alloc(&json_op::ptr_callbacks, nullptr, &jo));
636
637 auto json_in = (const unsigned char *) lv_struct.text_value();
638 auto json_len = lv_struct.text_length();
639
640 if (yajl_parse(parse_handle.in(), json_in, json_len) != yajl_status_ok ||
641 yajl_complete_parse(parse_handle.in()) != yajl_status_ok) {
642 log_error("failed to parse json value: %.*s",
643 lv_struct.text_length(),
644 lv_struct.text_value());
645 root.gen(lv_struct.to_string());
646 }
647 break;
648 }
649 default:
650 root.gen(lv_struct.to_string());
651 break;
652 }
653 }
654 }
655
656 auto sf = gen.to_string_fragment();
657 sqlite3_result_text(ctx,
658 sf.data(),
659 sf.length(),
660 SQLITE_TRANSIENT);
661 sqlite3_result_subtype(ctx, 74);
662 } else {
663 switch (lv_iter->lv_meta.lvm_kind) {
664 case value_kind_t::VALUE_NULL:
665 sqlite3_result_null(ctx);
666 break;
667 case value_kind_t::VALUE_JSON: {
668 sqlite3_result_text(ctx,
669 lv_iter->text_value(),
670 lv_iter->text_length(),
671 SQLITE_TRANSIENT);
672 sqlite3_result_subtype(ctx, 74);
673 break;
674 }
675 case value_kind_t::VALUE_STRUCT:
676 case value_kind_t::VALUE_TEXT:
677 case value_kind_t::VALUE_XML:
678 case value_kind_t::VALUE_TIMESTAMP: {
679 sqlite3_result_text(ctx,
680 lv_iter->text_value(),
681 lv_iter->text_length(),
682 SQLITE_TRANSIENT);
683 break;
684 }
685 case value_kind_t::VALUE_W3C_QUOTED:
686 case value_kind_t::VALUE_QUOTED:
687 if (lv_iter->lv_sbr.empty()) {
688 sqlite3_result_text(ctx, "", 0, SQLITE_STATIC);
689 } else {
690 const char *text_value = lv_iter->lv_sbr.get_data();
691 size_t text_len = lv_iter->lv_sbr.length();
692
693 switch (text_value[0]) {
694 case '\'':
695 case '"': {
696 char *val = (char *) sqlite3_malloc(
697 text_len);
698
699 if (val == nullptr) {
700 sqlite3_result_error_nomem(ctx);
701 } else {
702 auto unquote_func =
703 lv_iter->lv_meta.lvm_kind ==
704 value_kind_t::VALUE_W3C_QUOTED ?
705 unquote_w3c : unquote;
706
707 size_t unquoted_len = unquote_func(
708 val, text_value, text_len);
709 sqlite3_result_text(ctx, val,
710 unquoted_len,
711 sqlite3_free);
712 }
713 break;
714 }
715 default: {
716 sqlite3_result_text(ctx, text_value,
717 lv_iter->lv_sbr.length(),
718 SQLITE_TRANSIENT);
719 break;
720 }
721 }
722 }
723 break;
724
725 case value_kind_t::VALUE_BOOLEAN:
726 case value_kind_t::VALUE_INTEGER:
727 sqlite3_result_int64(ctx, lv_iter->lv_value.i);
728 break;
729
730 case value_kind_t::VALUE_FLOAT:
731 sqlite3_result_double(ctx, lv_iter->lv_value.d);
732 break;
733
734 case value_kind_t::VALUE_UNKNOWN:
735 case value_kind_t::VALUE__MAX:
736 require(0);
737 break;
738 }
739 }
740 }
741 else {
742 sqlite3_result_null(ctx);
743 }
744 }
745 break;
746 }
747
748 return SQLITE_OK;
749 }
750
vt_rowid(sqlite3_vtab_cursor * cur,sqlite_int64 * p_rowid)751 static int vt_rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *p_rowid)
752 {
753 vtab_cursor *p_cur = (vtab_cursor *)cur;
754
755 *p_rowid = (((uint64_t)p_cur->log_cursor.lc_curr_line) << 8) |
756 (p_cur->log_cursor.lc_sub_index & 0xff);
757
758 return SQLITE_OK;
759 }
760
update(unsigned char op,vis_line_t vl,bool exact)761 void log_cursor::update(unsigned char op, vis_line_t vl, bool exact)
762 {
763 if (vl < 0) {
764 vl = -1_vl;
765 }
766 switch (op) {
767 case SQLITE_INDEX_CONSTRAINT_EQ:
768 if (vl < this->lc_end_line) {
769 this->lc_curr_line = vl;
770 this->lc_end_line = vis_line_t(this->lc_curr_line + 1);
771 }
772 break;
773 case SQLITE_INDEX_CONSTRAINT_GE:
774 this->lc_curr_line = vl;
775 break;
776 case SQLITE_INDEX_CONSTRAINT_GT:
777 this->lc_curr_line = vis_line_t(vl + (exact ? 1 : 0));
778 break;
779 case SQLITE_INDEX_CONSTRAINT_LE:
780 this->lc_end_line = vis_line_t(vl + (exact ? 1 : 0));
781 break;
782 case SQLITE_INDEX_CONSTRAINT_LT:
783 this->lc_end_line = vl;
784 break;
785 }
786 }
787
vt_filter(sqlite3_vtab_cursor * p_vtc,int idxNum,const char * idxStr,int argc,sqlite3_value ** argv)788 static int vt_filter(sqlite3_vtab_cursor *p_vtc,
789 int idxNum, const char *idxStr,
790 int argc, sqlite3_value **argv)
791 {
792 vtab_cursor *p_cur = (vtab_cursor *)p_vtc;
793 vtab * vt = (vtab *)p_vtc->pVtab;
794 sqlite3_index_info::sqlite3_index_constraint *index = (
795 sqlite3_index_info::sqlite3_index_constraint *)idxStr;
796
797 log_info("(%p) filter called: %d", vt, idxNum);
798 p_cur->log_cursor.lc_curr_line = -1_vl;
799 p_cur->log_cursor.lc_end_line = vis_line_t(vt->lss->text_line_count());
800 vt_next(p_vtc);
801
802 if (!idxNum) {
803 return SQLITE_OK;
804 }
805
806 for (int lpc = 0; lpc < idxNum; lpc++) {
807 switch (index[lpc].iColumn) {
808 case VT_COL_LINE_NUMBER:
809 p_cur->log_cursor.update(index[lpc].op,
810 vis_line_t(sqlite3_value_int64(argv[lpc])));
811 break;
812
813 case VT_COL_LOG_TIME:
814 if (sqlite3_value_type(argv[lpc]) == SQLITE3_TEXT) {
815 const unsigned char *datestr = sqlite3_value_text(argv[lpc]);
816 date_time_scanner dts;
817 struct timeval tv;
818 struct exttm mytm;
819
820 dts.scan((const char *)datestr, strlen((const char *)datestr), NULL, &mytm, tv);
821 auto vl_opt = vt->lss->find_from_time(tv);
822 if (!vl_opt) {
823 p_cur->log_cursor.lc_curr_line = p_cur->log_cursor.lc_end_line;
824 }
825 else {
826 p_cur->log_cursor.update(index[lpc].op, vl_opt.value(), false);
827 }
828 }
829 break;
830
831 }
832 }
833
834 while (!p_cur->log_cursor.is_eof() && !vt->vi->is_valid(p_cur->log_cursor, *vt->lss)) {
835 p_cur->log_cursor.lc_curr_line += 1_vl;
836 }
837
838 return SQLITE_OK;
839 }
840
vt_best_index(sqlite3_vtab * tab,sqlite3_index_info * p_info)841 static int vt_best_index(sqlite3_vtab *tab, sqlite3_index_info *p_info)
842 {
843 std::vector<sqlite3_index_info::sqlite3_index_constraint> indexes;
844 int argvInUse = 0;
845 vtab *vt = (vtab *) tab;
846
847 log_info("(%p) best index called: nConstraint=%d", tab, p_info->nConstraint);
848 if (!vt->vi->vi_supports_indexes) {
849 return SQLITE_OK;
850 }
851 for (int lpc = 0; lpc < p_info->nConstraint; lpc++) {
852 if (!p_info->aConstraint[lpc].usable ||
853 p_info->aConstraint[lpc].op == SQLITE_INDEX_CONSTRAINT_MATCH) {
854 continue;
855 }
856
857 switch (p_info->aConstraint[lpc].iColumn) {
858 case VT_COL_LINE_NUMBER:
859 argvInUse += 1;
860 indexes.push_back(p_info->aConstraint[lpc]);
861 p_info->aConstraintUsage[lpc].argvIndex = argvInUse;
862 break;
863 }
864 }
865
866 if (!argvInUse) {
867 for (int lpc = 0; lpc < p_info->nConstraint; lpc++) {
868 if (!p_info->aConstraint[lpc].usable ||
869 p_info->aConstraint[lpc].op == SQLITE_INDEX_CONSTRAINT_MATCH) {
870 continue;
871 }
872
873 switch (p_info->aConstraint[lpc].iColumn) {
874 case VT_COL_LOG_TIME:
875 argvInUse += 1;
876 indexes.push_back(p_info->aConstraint[lpc]);
877 p_info->aConstraintUsage[lpc].argvIndex = argvInUse;
878 break;
879 }
880 }
881 }
882
883 if (argvInUse) {
884 sqlite3_index_info::sqlite3_index_constraint *index_copy;
885 size_t len = indexes.size() * sizeof(*index_copy);
886
887 log_info("found index, passing %d args", argvInUse);
888
889 index_copy = (sqlite3_index_info::sqlite3_index_constraint *)
890 sqlite3_malloc(len);
891 if (!index_copy) {
892 return SQLITE_NOMEM;
893 }
894 memcpy(index_copy, &indexes[0], len);
895 p_info->idxNum = argvInUse;
896 p_info->idxStr = (char *) index_copy;
897 p_info->needToFreeIdxStr = 1;
898 p_info->estimatedCost = 10.0;
899 }
900
901 return SQLITE_OK;
902 }
903
904 static struct json_path_container tags_handler = {
905 json_path_handler("#")
906 .with_synopsis("<tag>")
907 .with_description("A tag for the log line")
908 .with_pattern(R"(^#[^\s]+$)")
909 .FOR_FIELD(bookmark_metadata, bm_tags)
910 };
911
vt_update(sqlite3_vtab * tab,int argc,sqlite3_value ** argv,sqlite_int64 * rowid_out)912 static int vt_update(sqlite3_vtab *tab,
913 int argc,
914 sqlite3_value **argv,
915 sqlite_int64 *rowid_out)
916 {
917 vtab *vt = (vtab *)tab;
918 int retval = SQLITE_READONLY;
919
920 if (argc > 1 && sqlite3_value_type(argv[0]) != SQLITE_NULL &&
921 sqlite3_value_int64(argv[0]) == sqlite3_value_int64(argv[1])) {
922 int64_t rowid = sqlite3_value_int64(argv[0]) >> 8;
923 int val = sqlite3_value_int(argv[2 + VT_COL_MARK]);
924 vis_line_t vrowid(rowid);
925
926 std::map<content_line_t, bookmark_metadata> &bm = vt->lss->get_user_bookmark_metadata();
927 const unsigned char *part_name = sqlite3_value_text(argv[2 + VT_COL_PARTITION]);
928 const unsigned char *log_comment = sqlite3_value_text(argv[2 + VT_COL_LOG_COMMENT]);
929 const unsigned char *log_tags = sqlite3_value_text(argv[2 + VT_COL_LOG_TAGS]);
930 bookmark_metadata tmp_bm;
931
932 if (log_tags) {
933 vector<string> errors;
934 yajlpp_parse_context ypc(log_vtab_data.lvd_source, &tags_handler);
935 auto_mem<yajl_handle_t> handle(yajl_free);
936
937 handle = yajl_alloc(&ypc.ypc_callbacks, nullptr, &ypc);
938 ypc.ypc_userdata = &errors;
939 ypc.ypc_line_number = log_vtab_data.lvd_line_number;
940 ypc.with_handle(handle)
941 .with_error_reporter([](const yajlpp_parse_context &ypc,
942 lnav_log_level_t level,
943 const char *msg) {
944 vector<string> &errors = *((vector<string> *) ypc.ypc_userdata);
945 errors.emplace_back(msg);
946 })
947 .with_obj(tmp_bm);
948 ypc.parse(log_tags, strlen((const char *) log_tags));
949 ypc.complete_parse();
950 if (!errors.empty()) {
951 auto all_errors = fmt::format("{}", fmt::join(errors, "\n"));
952 tab->zErrMsg = sqlite3_mprintf("%s", all_errors.c_str());
953 return SQLITE_ERROR;
954 }
955 }
956
957 bookmark_vector<vis_line_t> &bv = vt->tc->get_bookmarks()[
958 &textview_curses::BM_META];
959 bool has_meta = part_name != nullptr || log_comment != nullptr ||
960 log_tags != nullptr;
961
962 if (binary_search(bv.begin(), bv.end(), vrowid) && !has_meta) {
963 vt->tc->set_user_mark(&textview_curses::BM_META, vrowid, false);
964 bm.erase(vt->lss->at(vrowid));
965 vt->lss->set_line_meta_changed();
966 }
967
968 if (has_meta) {
969 bookmark_metadata &line_meta = bm[vt->lss->at(vrowid)];
970
971 vt->tc->set_user_mark(&textview_curses::BM_META, vrowid, true);
972 if (part_name) {
973 line_meta.bm_name = string((const char *) part_name);
974 } else {
975 line_meta.bm_name.clear();
976 }
977 if (log_comment) {
978 line_meta.bm_comment = string((const char *) log_comment);
979 } else {
980 line_meta.bm_comment.clear();
981 }
982 if (log_tags) {
983 line_meta.bm_tags.clear();
984 for (const auto &tag : tmp_bm.bm_tags) {
985 line_meta.add_tag(tag);
986 }
987
988 for (const auto &tag : line_meta.bm_tags) {
989 bookmark_metadata::KNOWN_TAGS.insert(tag);
990 }
991 } else {
992 line_meta.bm_tags.clear();
993 }
994
995 vt->lss->set_line_meta_changed();
996 }
997
998 vt->tc->set_user_mark(&textview_curses::BM_USER, vrowid, val);
999 rowid += 1;
1000 while ((size_t)rowid < vt->lss->text_line_count()) {
1001 vis_line_t vl(rowid);
1002 content_line_t cl = vt->lss->at(vl);
1003 logline *ll = vt->lss->find_line(cl);
1004 if (ll->is_message()) {
1005 break;
1006 }
1007 vt->tc->set_user_mark(&textview_curses::BM_USER, vl, val);
1008 rowid += 1;
1009 }
1010
1011 if (retval != SQLITE_ERROR) {
1012 retval = SQLITE_OK;
1013 }
1014 }
1015
1016 return retval;
1017 }
1018
1019 static sqlite3_module generic_vtab_module = {
1020 0, /* iVersion */
1021 vt_create, /* xCreate - create a vtable */
1022 vt_connect, /* xConnect - associate a vtable with a connection */
1023 vt_best_index, /* xBestIndex - best index */
1024 vt_disconnect, /* xDisconnect - disassociate a vtable with a connection */
1025 vt_destroy, /* xDestroy - destroy a vtable */
1026 vt_open, /* xOpen - open a cursor */
1027 vt_close, /* xClose - close a cursor */
1028 vt_filter, /* xFilter - configure scan constraints */
1029 vt_next, /* xNext - advance a cursor */
1030 vt_eof, /* xEof - inidicate end of result set*/
1031 vt_column, /* xColumn - read data */
1032 vt_rowid, /* xRowid - read data */
1033 vt_update, /* xUpdate - write data */
1034 NULL, /* xBegin - begin transaction */
1035 NULL, /* xSync - sync transaction */
1036 NULL, /* xCommit - commit transaction */
1037 NULL, /* xRollback - rollback transaction */
1038 NULL, /* xFindFunction - function overloading */
1039 };
1040
progress_callback(void * ptr)1041 static int progress_callback(void *ptr)
1042 {
1043 int retval = 0;
1044
1045 if (log_vtab_data.lvd_progress != nullptr) {
1046 retval = log_vtab_data.lvd_progress(log_cursor_latest);
1047 }
1048
1049 return retval;
1050 }
1051
log_vtab_manager(sqlite3 * memdb,textview_curses & tc,logfile_sub_source & lss)1052 log_vtab_manager::log_vtab_manager(sqlite3 *memdb,
1053 textview_curses &tc,
1054 logfile_sub_source &lss)
1055 : vm_db(memdb), vm_textview(tc), vm_source(lss)
1056 {
1057 sqlite3_create_module(this->vm_db, "log_vtab_impl", &generic_vtab_module, this);
1058 sqlite3_progress_handler(memdb, 32, progress_callback, nullptr);
1059 }
1060
~log_vtab_manager()1061 log_vtab_manager::~log_vtab_manager()
1062 {
1063 while (!this->vm_impls.empty()) {
1064 auto first_name = this->vm_impls.begin()->first;
1065
1066 this->unregister_vtab(first_name);
1067 }
1068 }
1069
register_vtab(std::shared_ptr<log_vtab_impl> vi)1070 string log_vtab_manager::register_vtab(std::shared_ptr<log_vtab_impl> vi)
1071 {
1072 string retval;
1073
1074 if (this->vm_impls.find(vi->get_name()) == this->vm_impls.end()) {
1075 auto_mem<char, sqlite3_free> errmsg;
1076 auto_mem<char, sqlite3_free> sql;
1077 int rc;
1078
1079 this->vm_impls[vi->get_name()] = vi;
1080
1081 sql = sqlite3_mprintf("CREATE VIRTUAL TABLE %s "
1082 "USING log_vtab_impl(%s)",
1083 vi->get_name().get(),
1084 vi->get_name().get());
1085 rc = sqlite3_exec(this->vm_db,
1086 sql,
1087 nullptr,
1088 nullptr,
1089 errmsg.out());
1090 if (rc != SQLITE_OK) {
1091 retval = errmsg;
1092 }
1093 }
1094 else {
1095 retval = "a table with the given name already exists";
1096 }
1097
1098 return retval;
1099 }
1100
unregister_vtab(intern_string_t name)1101 string log_vtab_manager::unregister_vtab(intern_string_t name)
1102 {
1103 string retval;
1104
1105 if (this->vm_impls.find(name) == this->vm_impls.end()) {
1106 retval = fmt::format("unknown log line table -- {}", name);
1107 }
1108 else {
1109 auto_mem<char, sqlite3_free> sql;
1110 __attribute((unused))
1111 int rc;
1112
1113 sql = sqlite3_mprintf("DROP TABLE %s ", name.get());
1114 rc = sqlite3_exec(this->vm_db,
1115 sql,
1116 NULL,
1117 NULL,
1118 NULL);
1119
1120 this->vm_impls.erase(name);
1121 }
1122
1123 return retval;
1124 }
1125