1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/trace_processor_impl.h"
18
19 #include <inttypes.h>
20 #include <algorithm>
21
22 #include "perfetto/base/logging.h"
23 #include "perfetto/base/time.h"
24 #include "perfetto/ext/base/string_splitter.h"
25 #include "perfetto/ext/base/string_utils.h"
26 #include "src/trace_processor/dynamic/ancestor_slice_generator.h"
27 #include "src/trace_processor/dynamic/connected_flow_generator.h"
28 #include "src/trace_processor/dynamic/descendant_slice_generator.h"
29 #include "src/trace_processor/dynamic/describe_slice_generator.h"
30 #include "src/trace_processor/dynamic/experimental_counter_dur_generator.h"
31 #include "src/trace_processor/dynamic/experimental_flamegraph_generator.h"
32 #include "src/trace_processor/dynamic/experimental_sched_upid_generator.h"
33 #include "src/trace_processor/dynamic/experimental_slice_layout_generator.h"
34 #include "src/trace_processor/dynamic/thread_state_generator.h"
35 #include "src/trace_processor/export_json.h"
36 #include "src/trace_processor/importers/additional_modules.h"
37 #include "src/trace_processor/importers/ftrace/sched_event_tracker.h"
38 #include "src/trace_processor/importers/fuchsia/fuchsia_trace_parser.h"
39 #include "src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.h"
40 #include "src/trace_processor/importers/gzip/gzip_trace_parser.h"
41 #include "src/trace_processor/importers/json/json_trace_parser.h"
42 #include "src/trace_processor/importers/json/json_trace_tokenizer.h"
43 #include "src/trace_processor/importers/proto/metadata_tracker.h"
44 #include "src/trace_processor/importers/systrace/systrace_trace_parser.h"
45 #include "src/trace_processor/iterator_impl.h"
46 #include "src/trace_processor/sqlite/span_join_operator_table.h"
47 #include "src/trace_processor/sqlite/sql_stats_table.h"
48 #include "src/trace_processor/sqlite/sqlite3_str_split.h"
49 #include "src/trace_processor/sqlite/sqlite_raw_table.h"
50 #include "src/trace_processor/sqlite/sqlite_table.h"
51 #include "src/trace_processor/sqlite/sqlite_utils.h"
52 #include "src/trace_processor/sqlite/stats_table.h"
53 #include "src/trace_processor/sqlite/window_operator_table.h"
54 #include "src/trace_processor/tp_metatrace.h"
55 #include "src/trace_processor/types/variadic.h"
56 #include "src/trace_processor/util/protozero_to_text.h"
57
58 #include "protos/perfetto/trace/perfetto/perfetto_metatrace.pbzero.h"
59 #include "protos/perfetto/trace/trace.pbzero.h"
60 #include "protos/perfetto/trace/trace_packet.pbzero.h"
61
62 #include "src/trace_processor/metrics/chrome/all_chrome_metrics.descriptor.h"
63 #include "src/trace_processor/metrics/metrics.descriptor.h"
64 #include "src/trace_processor/metrics/metrics.h"
65 #include "src/trace_processor/metrics/sql_metrics.h"
66
67 #if !PERFETTO_BUILDFLAG(PERFETTO_OS_WIN)
68 #include <cxxabi.h>
69 #endif
70
71 // In Android and Chromium tree builds, we don't have the percentile module.
72 // Just don't include it.
73 #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
74 // defined in sqlite_src/ext/misc/percentile.c
75 extern "C" int sqlite3_percentile_init(sqlite3* db,
76 char** error,
77 const sqlite3_api_routines* api);
78 #endif // PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
79
80 namespace perfetto {
81 namespace trace_processor {
82 namespace {
83
84 const char kAllTablesQuery[] =
85 "SELECT tbl_name, type FROM (SELECT * FROM sqlite_master UNION ALL SELECT "
86 "* FROM sqlite_temp_master)";
87
InitializeSqlite(sqlite3 * db)88 void InitializeSqlite(sqlite3* db) {
89 char* error = nullptr;
90 sqlite3_exec(db, "PRAGMA temp_store=2", 0, 0, &error);
91 if (error) {
92 PERFETTO_FATAL("Error setting pragma temp_store: %s", error);
93 }
94 sqlite3_str_split_init(db);
95 // In Android tree builds, we don't have the percentile module.
96 // Just don't include it.
97 #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
98 sqlite3_percentile_init(db, &error, nullptr);
99 if (error) {
100 PERFETTO_ELOG("Error initializing: %s", error);
101 sqlite3_free(error);
102 }
103 #endif
104 }
105
BuildBoundsTable(sqlite3 * db,std::pair<int64_t,int64_t> bounds)106 void BuildBoundsTable(sqlite3* db, std::pair<int64_t, int64_t> bounds) {
107 char* error = nullptr;
108 sqlite3_exec(db, "DELETE FROM trace_bounds", nullptr, nullptr, &error);
109 if (error) {
110 PERFETTO_ELOG("Error deleting from bounds table: %s", error);
111 sqlite3_free(error);
112 return;
113 }
114
115 char* insert_sql = sqlite3_mprintf("INSERT INTO trace_bounds VALUES(%" PRId64
116 ", %" PRId64 ")",
117 bounds.first, bounds.second);
118
119 sqlite3_exec(db, insert_sql, 0, 0, &error);
120 sqlite3_free(insert_sql);
121 if (error) {
122 PERFETTO_ELOG("Error inserting bounds table: %s", error);
123 sqlite3_free(error);
124 }
125 }
126
CreateBuiltinTables(sqlite3 * db)127 void CreateBuiltinTables(sqlite3* db) {
128 char* error = nullptr;
129 sqlite3_exec(db, "CREATE TABLE perfetto_tables(name STRING)", 0, 0, &error);
130 if (error) {
131 PERFETTO_ELOG("Error initializing: %s", error);
132 sqlite3_free(error);
133 }
134 sqlite3_exec(db,
135 "CREATE TABLE trace_bounds(start_ts BIG INT, end_ts BIG INT)", 0,
136 0, &error);
137 if (error) {
138 PERFETTO_ELOG("Error initializing: %s", error);
139 sqlite3_free(error);
140 }
141 // Ensure that the entries in power_profile are unique to prevent duplicates
142 // when the power_profile is augmented with additional profiles.
143 sqlite3_exec(db,
144 "CREATE TABLE power_profile("
145 "device STRING, cpu INT, cluster INT, freq INT, power DOUBLE,"
146 "UNIQUE(device, cpu, cluster, freq));",
147 0, 0, &error);
148 if (error) {
149 PERFETTO_ELOG("Error initializing: %s", error);
150 sqlite3_free(error);
151 }
152 sqlite3_exec(db, "CREATE TABLE trace_metrics(name STRING)", 0, 0, &error);
153 if (error) {
154 PERFETTO_ELOG("Error initializing: %s", error);
155 sqlite3_free(error);
156 }
157 // This is a table intended to be used for metric debugging/developing. Data
158 // in the table is shown specially in the UI, and users can insert rows into
159 // this table to draw more things.
160 sqlite3_exec(db,
161 "CREATE TABLE debug_slices (id BIG INT, name STRING, ts BIG INT,"
162 "dur BIG INT, depth BIG INT)",
163 0, 0, &error);
164 if (error) {
165 PERFETTO_ELOG("Error initializing: %s", error);
166 sqlite3_free(error);
167 }
168
169 // Initialize the bounds table with some data so even before parsing any data,
170 // we still have a valid table.
171 BuildBoundsTable(db, std::make_pair(0, 0));
172 }
173
CreateBuiltinViews(sqlite3 * db)174 void CreateBuiltinViews(sqlite3* db) {
175 char* error = nullptr;
176 sqlite3_exec(db,
177 "CREATE VIEW counter_definitions AS "
178 "SELECT "
179 " *, "
180 " id AS counter_id "
181 "FROM counter_track",
182 0, 0, &error);
183 if (error) {
184 PERFETTO_ELOG("Error initializing: %s", error);
185 sqlite3_free(error);
186 }
187
188 sqlite3_exec(db,
189 "CREATE VIEW counter_values AS "
190 "SELECT "
191 " *, "
192 " track_id as counter_id "
193 "FROM counter",
194 0, 0, &error);
195 if (error) {
196 PERFETTO_ELOG("Error initializing: %s", error);
197 sqlite3_free(error);
198 }
199
200 sqlite3_exec(db,
201 "CREATE VIEW counters AS "
202 "SELECT * "
203 "FROM counter_values v "
204 "INNER JOIN counter_track t "
205 "ON v.track_id = t.id "
206 "ORDER BY ts;",
207 0, 0, &error);
208 if (error) {
209 PERFETTO_ELOG("Error initializing: %s", error);
210 sqlite3_free(error);
211 }
212
213 sqlite3_exec(db,
214 "CREATE VIEW slice AS "
215 "SELECT "
216 " *, "
217 " category AS cat, "
218 " id AS slice_id "
219 "FROM internal_slice;",
220 0, 0, &error);
221 if (error) {
222 PERFETTO_ELOG("Error initializing: %s", error);
223 sqlite3_free(error);
224 }
225
226 sqlite3_exec(db,
227 "CREATE VIEW instants AS "
228 "SELECT "
229 "*, "
230 "0.0 as value "
231 "FROM instant;",
232 0, 0, &error);
233
234 if (error) {
235 PERFETTO_ELOG("Error initializing: %s", error);
236 sqlite3_free(error);
237 }
238
239 sqlite3_exec(db,
240 "CREATE VIEW sched AS "
241 "SELECT "
242 "*, "
243 "ts + dur as ts_end "
244 "FROM sched_slice;",
245 0, 0, &error);
246
247 if (error) {
248 PERFETTO_ELOG("Error initializing: %s", error);
249 sqlite3_free(error);
250 }
251
252 // Legacy view for "slice" table with a deprecated table name.
253 // TODO(eseckler): Remove this view when all users have switched to "slice".
254 sqlite3_exec(db,
255 "CREATE VIEW slices AS "
256 "SELECT * FROM slice;",
257 0, 0, &error);
258 if (error) {
259 PERFETTO_ELOG("Error initializing: %s", error);
260 sqlite3_free(error);
261 }
262
263 sqlite3_exec(db,
264 "CREATE VIEW thread AS "
265 "SELECT "
266 "id as utid, "
267 "* "
268 "FROM internal_thread;",
269 0, 0, &error);
270 if (error) {
271 PERFETTO_ELOG("Error initializing: %s", error);
272 sqlite3_free(error);
273 }
274
275 sqlite3_exec(db,
276 "CREATE VIEW process AS "
277 "SELECT "
278 "id as upid, "
279 "* "
280 "FROM internal_process;",
281 0, 0, &error);
282 if (error) {
283 PERFETTO_ELOG("Error initializing: %s", error);
284 sqlite3_free(error);
285 }
286 }
287
ExportJson(sqlite3_context * ctx,int,sqlite3_value ** argv)288 void ExportJson(sqlite3_context* ctx, int /*argc*/, sqlite3_value** argv) {
289 TraceStorage* storage = static_cast<TraceStorage*>(sqlite3_user_data(ctx));
290 FILE* output;
291 if (sqlite3_value_type(argv[0]) == SQLITE_INTEGER) {
292 // Assume input is an FD.
293 output = fdopen(sqlite3_value_int(argv[0]), "w");
294 if (!output) {
295 sqlite3_result_error(ctx, "Couldn't open output file from given FD", -1);
296 return;
297 }
298 } else {
299 const char* filename =
300 reinterpret_cast<const char*>(sqlite3_value_text(argv[0]));
301 output = fopen(filename, "w");
302 if (!output) {
303 sqlite3_result_error(ctx, "Couldn't open output file", -1);
304 return;
305 }
306 }
307
308 util::Status result = json::ExportJson(storage, output);
309 if (!result.ok()) {
310 sqlite3_result_error(ctx, result.message().c_str(), -1);
311 return;
312 }
313 }
314
CreateJsonExportFunction(TraceStorage * ts,sqlite3 * db)315 void CreateJsonExportFunction(TraceStorage* ts, sqlite3* db) {
316 auto ret = sqlite3_create_function_v2(db, "EXPORT_JSON", 1, SQLITE_UTF8, ts,
317 ExportJson, nullptr, nullptr,
318 sqlite_utils::kSqliteStatic);
319 if (ret) {
320 PERFETTO_ELOG("Error initializing EXPORT_JSON");
321 }
322 }
323
Hash(sqlite3_context * ctx,int argc,sqlite3_value ** argv)324 void Hash(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
325 base::Hash hash;
326 for (int i = 0; i < argc; ++i) {
327 sqlite3_value* value = argv[i];
328 switch (sqlite3_value_type(value)) {
329 case SQLITE_INTEGER:
330 hash.Update(sqlite3_value_int64(value));
331 break;
332 case SQLITE_TEXT: {
333 const char* ptr =
334 reinterpret_cast<const char*>(sqlite3_value_text(value));
335 hash.Update(ptr, strlen(ptr));
336 break;
337 }
338 default:
339 sqlite3_result_error(ctx, "Unsupported type of arg passed to HASH", -1);
340 return;
341 }
342 }
343 sqlite3_result_int64(ctx, static_cast<int64_t>(hash.digest()));
344 }
345
Demangle(sqlite3_context * ctx,int argc,sqlite3_value ** argv)346 void Demangle(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
347 if (argc != 1) {
348 sqlite3_result_error(ctx, "Unsupported number of arg passed to DEMANGLE",
349 -1);
350 return;
351 }
352 sqlite3_value* value = argv[0];
353 if (sqlite3_value_type(value) == SQLITE_NULL) {
354 sqlite3_result_null(ctx);
355 return;
356 }
357 if (sqlite3_value_type(value) != SQLITE_TEXT) {
358 sqlite3_result_error(ctx, "Unsupported type of arg passed to DEMANGLE", -1);
359 return;
360 }
361 const char* ptr = reinterpret_cast<const char*>(sqlite3_value_text(value));
362 #if !PERFETTO_BUILDFLAG(PERFETTO_OS_WIN)
363 int ignored = 0;
364 // This memory was allocated by malloc and will be passed to SQLite to free.
365 char* demangled_name = abi::__cxa_demangle(ptr, nullptr, nullptr, &ignored);
366 if (!demangled_name) {
367 sqlite3_result_null(ctx);
368 return;
369 }
370 sqlite3_result_text(ctx, demangled_name, -1, free);
371 #else
372 sqlite3_result_text(ctx, ptr, -1, sqlite_utils::kSqliteTransient);
373 #endif
374 }
375
LastNonNullStep(sqlite3_context * ctx,int argc,sqlite3_value ** argv)376 void LastNonNullStep(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
377 if (argc != 1) {
378 sqlite3_result_error(
379 ctx, "Unsupported number of args passed to LAST_NON_NULL", -1);
380 return;
381 }
382 sqlite3_value* value = argv[0];
383 if (sqlite3_value_type(value) == SQLITE_NULL) {
384 return;
385 }
386 sqlite3_value** ptr = reinterpret_cast<sqlite3_value**>(
387 sqlite3_aggregate_context(ctx, sizeof(sqlite3_value*)));
388 if (ptr) {
389 if (*ptr != nullptr) {
390 sqlite3_value_free(*ptr);
391 }
392 *ptr = sqlite3_value_dup(value);
393 }
394 }
395
LastNonNullInverse(sqlite3_context * ctx,int argc,sqlite3_value ** argv)396 void LastNonNullInverse(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
397 // Do nothing.
398 base::ignore_result(ctx);
399 base::ignore_result(argc);
400 base::ignore_result(argv);
401 }
402
LastNonNullValue(sqlite3_context * ctx)403 void LastNonNullValue(sqlite3_context* ctx) {
404 sqlite3_value** ptr =
405 reinterpret_cast<sqlite3_value**>(sqlite3_aggregate_context(ctx, 0));
406 if (!ptr || !*ptr) {
407 sqlite3_result_null(ctx);
408 } else {
409 sqlite3_result_value(ctx, *ptr);
410 }
411 }
412
LastNonNullFinal(sqlite3_context * ctx)413 void LastNonNullFinal(sqlite3_context* ctx) {
414 sqlite3_value** ptr =
415 reinterpret_cast<sqlite3_value**>(sqlite3_aggregate_context(ctx, 0));
416 if (!ptr || !*ptr) {
417 sqlite3_result_null(ctx);
418 } else {
419 sqlite3_result_value(ctx, *ptr);
420 sqlite3_value_free(*ptr);
421 }
422 }
423
CreateHashFunction(sqlite3 * db)424 void CreateHashFunction(sqlite3* db) {
425 auto ret = sqlite3_create_function_v2(
426 db, "HASH", -1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, &Hash,
427 nullptr, nullptr, nullptr);
428 if (ret) {
429 PERFETTO_ELOG("Error initializing HASH");
430 }
431 }
432
CreateDemangledNameFunction(sqlite3 * db)433 void CreateDemangledNameFunction(sqlite3* db) {
434 auto ret = sqlite3_create_function_v2(
435 db, "DEMANGLE", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, &Demangle,
436 nullptr, nullptr, nullptr);
437 if (ret != SQLITE_OK) {
438 PERFETTO_ELOG("Error initializing DEMANGLE: %s", sqlite3_errmsg(db));
439 }
440 }
441
CreateLastNonNullFunction(sqlite3 * db)442 void CreateLastNonNullFunction(sqlite3* db) {
443 auto ret = sqlite3_create_window_function(
444 db, "LAST_NON_NULL", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr,
445 &LastNonNullStep, &LastNonNullFinal, &LastNonNullValue,
446 &LastNonNullInverse, nullptr);
447 if (ret) {
448 PERFETTO_ELOG("Error initializing LAST_NON_NULL");
449 }
450 }
451
452 struct ValueAtMaxTsContext {
453 bool initialized;
454 int value_type;
455
456 int64_t max_ts;
457 int64_t int_value_at_max_ts;
458 double double_value_at_max_ts;
459 };
460
ValueAtMaxTsStep(sqlite3_context * ctx,int,sqlite3_value ** argv)461 void ValueAtMaxTsStep(sqlite3_context* ctx, int, sqlite3_value** argv) {
462 sqlite3_value* ts = argv[0];
463 sqlite3_value* value = argv[1];
464
465 // Note that sqlite3_aggregate_context zeros the memory for us so all the
466 // variables of the struct should be zero.
467 ValueAtMaxTsContext* fn_ctx = reinterpret_cast<ValueAtMaxTsContext*>(
468 sqlite3_aggregate_context(ctx, sizeof(ValueAtMaxTsContext)));
469
470 // For performance reasons, we only do the check for the type of ts and value
471 // on the first call of the function.
472 if (PERFETTO_UNLIKELY(!fn_ctx->initialized)) {
473 if (sqlite3_value_type(ts) != SQLITE_INTEGER) {
474 sqlite3_result_error(ctx, "VALUE_AT_MAX_TS: ts passed was not an integer",
475 -1);
476 return;
477 }
478
479 fn_ctx->value_type = sqlite3_value_type(value);
480 if (fn_ctx->value_type != SQLITE_INTEGER &&
481 fn_ctx->value_type != SQLITE_FLOAT) {
482 sqlite3_result_error(
483 ctx, "VALUE_AT_MAX_TS: value passed was not an integer or float", -1);
484 return;
485 }
486
487 fn_ctx->initialized = true;
488 }
489
490 // On dcheck builds however, we check every passed ts and value.
491 #if PERFETTO_DCHECK_IS_ON()
492 if (sqlite3_value_type(ts) != SQLITE_INTEGER) {
493 sqlite3_result_error(ctx, "VALUE_AT_MAX_TS: ts passed was not an integer",
494 -1);
495 return;
496 }
497 if (sqlite3_value_type(value) != fn_ctx->value_type) {
498 sqlite3_result_error(ctx, "VALUE_AT_MAX_TS: value type is inconsistent",
499 -1);
500 return;
501 }
502 #endif
503
504 int64_t ts_int = sqlite3_value_int64(ts);
505 if (PERFETTO_LIKELY(fn_ctx->max_ts < ts_int)) {
506 fn_ctx->max_ts = ts_int;
507
508 if (fn_ctx->value_type == SQLITE_INTEGER) {
509 fn_ctx->int_value_at_max_ts = sqlite3_value_int64(value);
510 } else {
511 fn_ctx->double_value_at_max_ts = sqlite3_value_double(value);
512 }
513 }
514 }
515
ValueAtMaxTsFinal(sqlite3_context * ctx)516 void ValueAtMaxTsFinal(sqlite3_context* ctx) {
517 ValueAtMaxTsContext* fn_ctx =
518 reinterpret_cast<ValueAtMaxTsContext*>(sqlite3_aggregate_context(ctx, 0));
519 if (!fn_ctx) {
520 sqlite3_result_null(ctx);
521 return;
522 }
523 if (fn_ctx->value_type == SQLITE_INTEGER) {
524 sqlite3_result_int64(ctx, fn_ctx->int_value_at_max_ts);
525 } else {
526 sqlite3_result_double(ctx, fn_ctx->double_value_at_max_ts);
527 }
528 }
529
CreateValueAtMaxTsFunction(sqlite3 * db)530 void CreateValueAtMaxTsFunction(sqlite3* db) {
531 auto ret = sqlite3_create_function_v2(
532 db, "VALUE_AT_MAX_TS", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr,
533 nullptr, &ValueAtMaxTsStep, &ValueAtMaxTsFinal, nullptr);
534 if (ret) {
535 PERFETTO_ELOG("Error initializing VALUE_AT_MAX_TS");
536 }
537 }
538
ExtractArg(sqlite3_context * ctx,int argc,sqlite3_value ** argv)539 void ExtractArg(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
540 if (argc != 2) {
541 sqlite3_result_error(ctx, "EXTRACT_ARG: 2 args required", -1);
542 return;
543 }
544 if (sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
545 sqlite3_result_error(ctx, "EXTRACT_ARG: 1st argument should be arg set id",
546 -1);
547 return;
548 }
549 if (sqlite3_value_type(argv[1]) != SQLITE_TEXT) {
550 sqlite3_result_error(ctx, "EXTRACT_ARG: 2nd argument should be key", -1);
551 return;
552 }
553
554 TraceStorage* storage = static_cast<TraceStorage*>(sqlite3_user_data(ctx));
555 uint32_t arg_set_id = static_cast<uint32_t>(sqlite3_value_int(argv[0]));
556 const char* key = reinterpret_cast<const char*>(sqlite3_value_text(argv[1]));
557
558 base::Optional<Variadic> opt_value;
559 util::Status status = storage->ExtractArg(arg_set_id, key, &opt_value);
560 if (!status.ok()) {
561 sqlite3_result_error(ctx, status.c_message(), -1);
562 return;
563 }
564
565 if (!opt_value) {
566 sqlite3_result_null(ctx);
567 return;
568 }
569
570 switch (opt_value->type) {
571 case Variadic::kInt:
572 sqlite3_result_int64(ctx, opt_value->int_value);
573 break;
574 case Variadic::kBool:
575 sqlite3_result_int64(ctx, opt_value->bool_value);
576 break;
577 case Variadic::kUint:
578 sqlite3_result_int64(ctx, static_cast<int64_t>(opt_value->uint_value));
579 break;
580 case Variadic::kPointer:
581 sqlite3_result_int64(ctx, static_cast<int64_t>(opt_value->pointer_value));
582 break;
583 case Variadic::kJson:
584 sqlite3_result_text(ctx, storage->GetString(opt_value->json_value).data(),
585 -1, nullptr);
586 break;
587 case Variadic::kString:
588 sqlite3_result_text(
589 ctx, storage->GetString(opt_value->string_value).data(), -1, nullptr);
590 break;
591 case Variadic::kReal:
592 sqlite3_result_double(ctx, opt_value->real_value);
593 break;
594 }
595 }
596
CreateExtractArgFunction(TraceStorage * ts,sqlite3 * db)597 void CreateExtractArgFunction(TraceStorage* ts, sqlite3* db) {
598 auto ret = sqlite3_create_function_v2(db, "EXTRACT_ARG", 2,
599 SQLITE_UTF8 | SQLITE_DETERMINISTIC, ts,
600 &ExtractArg, nullptr, nullptr, nullptr);
601 if (ret != SQLITE_OK) {
602 PERFETTO_FATAL("Error initializing EXTRACT_ARG: %s", sqlite3_errmsg(db));
603 }
604 }
605
CreateSourceGeqFunction(sqlite3 * db)606 void CreateSourceGeqFunction(sqlite3* db) {
607 auto fn = [](sqlite3_context* ctx, int, sqlite3_value**) {
608 sqlite3_result_error(
609 ctx, "SOURCE_GEQ should not be called from the global scope", -1);
610 };
611 auto ret = sqlite3_create_function_v2(db, "SOURCE_GEQ", -1,
612 SQLITE_UTF8 | SQLITE_DETERMINISTIC,
613 nullptr, fn, nullptr, nullptr, nullptr);
614 if (ret != SQLITE_OK) {
615 PERFETTO_FATAL("Error initializing SOURCE_GEQ: %s", sqlite3_errmsg(db));
616 }
617 }
618
SetupMetrics(TraceProcessor * tp,sqlite3 * db,std::vector<metrics::SqlMetricFile> * sql_metrics)619 void SetupMetrics(TraceProcessor* tp,
620 sqlite3* db,
621 std::vector<metrics::SqlMetricFile>* sql_metrics) {
622 tp->ExtendMetricsProto(kMetricsDescriptor.data(), kMetricsDescriptor.size());
623 tp->ExtendMetricsProto(kAllChromeMetricsDescriptor.data(),
624 kAllChromeMetricsDescriptor.size());
625
626 for (const auto& file_to_sql : metrics::sql_metrics::kFileToSql) {
627 tp->RegisterMetric(file_to_sql.path, file_to_sql.sql);
628 }
629
630 {
631 std::unique_ptr<metrics::RunMetricContext> ctx(
632 new metrics::RunMetricContext());
633 ctx->tp = tp;
634 ctx->metrics = sql_metrics;
635 auto ret = sqlite3_create_function_v2(
636 db, "RUN_METRIC", -1, SQLITE_UTF8, ctx.release(), metrics::RunMetric,
637 nullptr, nullptr,
638 [](void* ptr) { delete static_cast<metrics::RunMetricContext*>(ptr); });
639 if (ret)
640 PERFETTO_FATAL("Error initializing RUN_METRIC");
641 }
642
643 {
644 auto ret = sqlite3_create_function_v2(
645 db, "RepeatedField", 1, SQLITE_UTF8, nullptr, nullptr,
646 metrics::RepeatedFieldStep, metrics::RepeatedFieldFinal, nullptr);
647 if (ret)
648 PERFETTO_FATAL("Error initializing RepeatedField");
649 }
650
651 {
652 auto ret = sqlite3_create_function_v2(db, "NULL_IF_EMPTY", 1, SQLITE_UTF8,
653 nullptr, metrics::NullIfEmpty,
654 nullptr, nullptr, nullptr);
655 if (ret)
656 PERFETTO_FATAL("Error initializing NULL_IF_EMPTY");
657 }
658 }
659
EnsureSqliteInitialized()660 void EnsureSqliteInitialized() {
661 // sqlite3_initialize isn't actually thread-safe despite being documented
662 // as such; we need to make sure multiple TraceProcessorImpl instances don't
663 // call it concurrently and only gets called once per process, instead.
664 static bool init_once = [] { return sqlite3_initialize() == SQLITE_OK; }();
665 PERFETTO_CHECK(init_once);
666 }
667
InsertIntoTraceMetricsTable(sqlite3 * db,const std::string & metric_name)668 void InsertIntoTraceMetricsTable(sqlite3* db, const std::string& metric_name) {
669 char* insert_sql = sqlite3_mprintf(
670 "INSERT INTO trace_metrics(name) VALUES('%q')", metric_name.c_str());
671 char* insert_error = nullptr;
672 sqlite3_exec(db, insert_sql, nullptr, nullptr, &insert_error);
673 sqlite3_free(insert_sql);
674 if (insert_error) {
675 PERFETTO_ELOG("Error registering table: %s", insert_error);
676 sqlite3_free(insert_error);
677 }
678 }
679
680 } // namespace
681
TraceProcessorImpl(const Config & cfg)682 TraceProcessorImpl::TraceProcessorImpl(const Config& cfg)
683 : TraceProcessorStorageImpl(cfg) {
684 context_.fuchsia_trace_tokenizer.reset(new FuchsiaTraceTokenizer(&context_));
685 context_.fuchsia_trace_parser.reset(new FuchsiaTraceParser(&context_));
686
687 context_.systrace_trace_parser.reset(new SystraceTraceParser(&context_));
688
689 if (gzip::IsGzipSupported())
690 context_.gzip_trace_parser.reset(new GzipTraceParser(&context_));
691
692 if (json::IsJsonSupported()) {
693 context_.json_trace_tokenizer.reset(new JsonTraceTokenizer(&context_));
694 context_.json_trace_parser.reset(new JsonTraceParser(&context_));
695 }
696
697 RegisterAdditionalModules(&context_);
698
699 sqlite3* db = nullptr;
700 EnsureSqliteInitialized();
701 PERFETTO_CHECK(sqlite3_open(":memory:", &db) == SQLITE_OK);
702 InitializeSqlite(db);
703 CreateBuiltinTables(db);
704 CreateBuiltinViews(db);
705 db_.reset(std::move(db));
706
707 CreateJsonExportFunction(context_.storage.get(), db);
708 CreateHashFunction(db);
709 CreateDemangledNameFunction(db);
710 CreateLastNonNullFunction(db);
711 CreateExtractArgFunction(context_.storage.get(), db);
712 CreateSourceGeqFunction(db);
713 CreateValueAtMaxTsFunction(db);
714
715 SetupMetrics(this, *db_, &sql_metrics_);
716
717 // Setup the query cache.
718 query_cache_.reset(new QueryCache());
719
720 const TraceStorage* storage = context_.storage.get();
721
722 SqlStatsTable::RegisterTable(*db_, storage);
723 StatsTable::RegisterTable(*db_, storage);
724
725 // Operator tables.
726 SpanJoinOperatorTable::RegisterTable(*db_, storage);
727 WindowOperatorTable::RegisterTable(*db_, storage);
728
729 // New style tables but with some custom logic.
730 SqliteRawTable::RegisterTable(*db_, query_cache_.get(), &context_);
731
732 // Tables dynamically generated at query time.
733 RegisterDynamicTable(std::unique_ptr<ExperimentalFlamegraphGenerator>(
734 new ExperimentalFlamegraphGenerator(&context_)));
735 RegisterDynamicTable(std::unique_ptr<ExperimentalCounterDurGenerator>(
736 new ExperimentalCounterDurGenerator(storage->counter_table())));
737 RegisterDynamicTable(std::unique_ptr<DescribeSliceGenerator>(
738 new DescribeSliceGenerator(&context_)));
739 RegisterDynamicTable(std::unique_ptr<ExperimentalSliceLayoutGenerator>(
740 new ExperimentalSliceLayoutGenerator(
741 context_.storage.get()->mutable_string_pool(),
742 &storage->slice_table())));
743 RegisterDynamicTable(std::unique_ptr<AncestorSliceGenerator>(
744 new AncestorSliceGenerator(&context_)));
745 RegisterDynamicTable(std::unique_ptr<DescendantSliceGenerator>(
746 new DescendantSliceGenerator(&context_)));
747 RegisterDynamicTable(
748 std::unique_ptr<ConnectedFlowGenerator>(new ConnectedFlowGenerator(
749 ConnectedFlowGenerator::Direction::BOTH, &context_)));
750 RegisterDynamicTable(
751 std::unique_ptr<ConnectedFlowGenerator>(new ConnectedFlowGenerator(
752 ConnectedFlowGenerator::Direction::FOLLOWING, &context_)));
753 RegisterDynamicTable(
754 std::unique_ptr<ConnectedFlowGenerator>(new ConnectedFlowGenerator(
755 ConnectedFlowGenerator::Direction::PRECEDING, &context_)));
756 RegisterDynamicTable(std::unique_ptr<ExperimentalSchedUpidGenerator>(
757 new ExperimentalSchedUpidGenerator(storage->sched_slice_table(),
758 storage->thread_table())));
759 RegisterDynamicTable(std::unique_ptr<ThreadStateGenerator>(
760 new ThreadStateGenerator(&context_)));
761
762 // New style db-backed tables.
763 RegisterDbTable(storage->arg_table());
764 RegisterDbTable(storage->thread_table());
765 RegisterDbTable(storage->process_table());
766
767 RegisterDbTable(storage->slice_table());
768 RegisterDbTable(storage->flow_table());
769 RegisterDbTable(storage->sched_slice_table());
770 RegisterDbTable(storage->instant_table());
771 RegisterDbTable(storage->gpu_slice_table());
772
773 RegisterDbTable(storage->track_table());
774 RegisterDbTable(storage->thread_track_table());
775 RegisterDbTable(storage->process_track_table());
776 RegisterDbTable(storage->gpu_track_table());
777
778 RegisterDbTable(storage->counter_table());
779
780 RegisterDbTable(storage->counter_track_table());
781 RegisterDbTable(storage->process_counter_track_table());
782 RegisterDbTable(storage->thread_counter_track_table());
783 RegisterDbTable(storage->cpu_counter_track_table());
784 RegisterDbTable(storage->irq_counter_track_table());
785 RegisterDbTable(storage->softirq_counter_track_table());
786 RegisterDbTable(storage->gpu_counter_track_table());
787 RegisterDbTable(storage->gpu_counter_group_table());
788
789 RegisterDbTable(storage->heap_graph_object_table());
790 RegisterDbTable(storage->heap_graph_reference_table());
791 RegisterDbTable(storage->heap_graph_class_table());
792
793 RegisterDbTable(storage->symbol_table());
794 RegisterDbTable(storage->heap_profile_allocation_table());
795 RegisterDbTable(storage->cpu_profile_stack_sample_table());
796 RegisterDbTable(storage->perf_sample_table());
797 RegisterDbTable(storage->stack_profile_callsite_table());
798 RegisterDbTable(storage->stack_profile_mapping_table());
799 RegisterDbTable(storage->stack_profile_frame_table());
800 RegisterDbTable(storage->package_list_table());
801 RegisterDbTable(storage->profiler_smaps_table());
802
803 RegisterDbTable(storage->android_log_table());
804
805 RegisterDbTable(storage->vulkan_memory_allocations_table());
806
807 RegisterDbTable(storage->graphics_frame_slice_table());
808
809 RegisterDbTable(storage->metadata_table());
810 RegisterDbTable(storage->cpu_table());
811 RegisterDbTable(storage->cpu_freq_table());
812
813 RegisterDbTable(storage->memory_snapshot_table());
814 RegisterDbTable(storage->process_memory_snapshot_table());
815 RegisterDbTable(storage->memory_snapshot_node_table());
816 RegisterDbTable(storage->memory_snapshot_edge_table());
817 }
818
819 TraceProcessorImpl::~TraceProcessorImpl() = default;
820
Parse(std::unique_ptr<uint8_t[]> data,size_t size)821 util::Status TraceProcessorImpl::Parse(std::unique_ptr<uint8_t[]> data,
822 size_t size) {
823 bytes_parsed_ += size;
824 return TraceProcessorStorageImpl::Parse(std::move(data), size);
825 }
826
GetCurrentTraceName()827 std::string TraceProcessorImpl::GetCurrentTraceName() {
828 if (current_trace_name_.empty())
829 return "";
830 auto size = " (" + std::to_string(bytes_parsed_ / 1024 / 1024) + " MB)";
831 return current_trace_name_ + size;
832 }
833
SetCurrentTraceName(const std::string & name)834 void TraceProcessorImpl::SetCurrentTraceName(const std::string& name) {
835 current_trace_name_ = name;
836 }
837
NotifyEndOfFile()838 void TraceProcessorImpl::NotifyEndOfFile() {
839 if (current_trace_name_.empty())
840 current_trace_name_ = "Unnamed trace";
841
842 TraceProcessorStorageImpl::NotifyEndOfFile();
843
844 SchedEventTracker::GetOrCreate(&context_)->FlushPendingEvents();
845 context_.metadata_tracker->SetMetadata(
846 metadata::trace_size_bytes,
847 Variadic::Integer(static_cast<int64_t>(bytes_parsed_)));
848 BuildBoundsTable(*db_, context_.storage->GetTraceTimestampBoundsNs());
849
850 // Create a snapshot of all tables and views created so far. This is so later
851 // we can drop all extra tables created by the UI and reset to the original
852 // state (see RestoreInitialTables).
853 initial_tables_.clear();
854 auto it = ExecuteQuery(kAllTablesQuery);
855 while (it.Next()) {
856 auto value = it.Get(0);
857 PERFETTO_CHECK(value.type == SqlValue::Type::kString);
858 initial_tables_.push_back(value.string_value);
859 }
860 }
861
RestoreInitialTables()862 size_t TraceProcessorImpl::RestoreInitialTables() {
863 std::vector<std::pair<std::string, std::string>> deletion_list;
864 std::string msg = "Resetting DB to initial state, deleting table/views:";
865 for (auto it = ExecuteQuery(kAllTablesQuery); it.Next();) {
866 std::string name(it.Get(0).string_value);
867 std::string type(it.Get(1).string_value);
868 if (std::find(initial_tables_.begin(), initial_tables_.end(), name) ==
869 initial_tables_.end()) {
870 msg += " " + name;
871 deletion_list.push_back(std::make_pair(type, name));
872 }
873 }
874
875 PERFETTO_LOG("%s", msg.c_str());
876 for (const auto& tn : deletion_list) {
877 std::string query = "DROP " + tn.first + " " + tn.second;
878 auto it = ExecuteQuery(query);
879 while (it.Next()) {
880 }
881 // Index deletion can legitimately fail. If one creates an index "i" on a
882 // table "t" but issues the deletion in the order (t, i), the DROP index i
883 // will fail with "no such index" because deleting the table "t"
884 // automatically deletes all associated indexes.
885 if (!it.Status().ok() && tn.first != "index")
886 PERFETTO_FATAL("%s -> %s", query.c_str(), it.Status().c_message());
887 }
888 return deletion_list.size();
889 }
890
ExecuteQuery(const std::string & sql,int64_t time_queued)891 Iterator TraceProcessorImpl::ExecuteQuery(const std::string& sql,
892 int64_t time_queued) {
893 sqlite3_stmt* raw_stmt;
894 int err;
895 {
896 PERFETTO_TP_TRACE("QUERY_PREPARE");
897 err = sqlite3_prepare_v2(*db_, sql.c_str(), static_cast<int>(sql.size()),
898 &raw_stmt, nullptr);
899 }
900
901 util::Status status;
902 uint32_t col_count = 0;
903 if (err != SQLITE_OK) {
904 status = util::ErrStatus("%s", sqlite3_errmsg(*db_));
905 } else {
906 col_count = static_cast<uint32_t>(sqlite3_column_count(raw_stmt));
907 }
908
909 base::TimeNanos t_start = base::GetWallTimeNs();
910 uint32_t sql_stats_row =
911 context_.storage->mutable_sql_stats()->RecordQueryBegin(sql, time_queued,
912 t_start.count());
913
914 std::unique_ptr<IteratorImpl> impl(new IteratorImpl(
915 this, *db_, ScopedStmt(raw_stmt), col_count, status, sql_stats_row));
916 return Iterator(std::move(impl));
917 }
918
InterruptQuery()919 void TraceProcessorImpl::InterruptQuery() {
920 if (!db_)
921 return;
922 query_interrupted_.store(true);
923 sqlite3_interrupt(db_.get());
924 }
925
IsRootMetricField(const std::string & metric_name)926 bool TraceProcessorImpl::IsRootMetricField(const std::string& metric_name) {
927 base::Optional<uint32_t> desc_idx =
928 pool_.FindDescriptorIdx(".perfetto.protos.TraceMetrics");
929 if (!desc_idx.has_value())
930 return false;
931 base::Optional<uint32_t> field_idx =
932 pool_.descriptors()[*desc_idx].FindFieldIdxByName(metric_name);
933 return field_idx.has_value();
934 }
935
RegisterMetric(const std::string & path,const std::string & sql)936 util::Status TraceProcessorImpl::RegisterMetric(const std::string& path,
937 const std::string& sql) {
938 std::string stripped_sql;
939 for (base::StringSplitter sp(sql, '\n'); sp.Next();) {
940 if (strncmp(sp.cur_token(), "--", 2) != 0) {
941 stripped_sql.append(sp.cur_token());
942 stripped_sql.push_back('\n');
943 }
944 }
945
946 // Check if the metric with the given path already exists and if it does, just
947 // update the SQL associated with it.
948 auto it = std::find_if(
949 sql_metrics_.begin(), sql_metrics_.end(),
950 [&path](const metrics::SqlMetricFile& m) { return m.path == path; });
951 if (it != sql_metrics_.end()) {
952 it->sql = stripped_sql;
953 return util::OkStatus();
954 }
955
956 auto sep_idx = path.rfind("/");
957 std::string basename =
958 sep_idx == std::string::npos ? path : path.substr(sep_idx + 1);
959
960 auto sql_idx = basename.rfind(".sql");
961 if (sql_idx == std::string::npos) {
962 return util::ErrStatus("Unable to find .sql extension for metric");
963 }
964 auto no_ext_name = basename.substr(0, sql_idx);
965
966 metrics::SqlMetricFile metric;
967 metric.path = path;
968 metric.sql = stripped_sql;
969
970 if (IsRootMetricField(no_ext_name)) {
971 metric.proto_field_name = no_ext_name;
972 metric.output_table_name = no_ext_name + "_output";
973 InsertIntoTraceMetricsTable(*db_, no_ext_name);
974 }
975
976 sql_metrics_.emplace_back(metric);
977 return util::OkStatus();
978 }
979
ExtendMetricsProto(const uint8_t * data,size_t size)980 util::Status TraceProcessorImpl::ExtendMetricsProto(const uint8_t* data,
981 size_t size) {
982 util::Status status = pool_.AddFromFileDescriptorSet(data, size);
983 if (!status.ok())
984 return status;
985
986 for (const auto& desc : pool_.descriptors()) {
987 // Convert the full name (e.g. .perfetto.protos.TraceMetrics.SubMetric)
988 // into a function name of the form (TraceMetrics_SubMetric).
989 auto fn_name = desc.full_name().substr(desc.package_name().size() + 1);
990 std::replace(fn_name.begin(), fn_name.end(), '.', '_');
991
992 std::unique_ptr<metrics::BuildProtoContext> ctx(
993 new metrics::BuildProtoContext());
994 ctx->tp = this;
995 ctx->pool = &pool_;
996 ctx->desc = &desc;
997
998 auto ret = sqlite3_create_function_v2(
999 *db_, fn_name.c_str(), -1, SQLITE_UTF8, ctx.release(),
1000 metrics::BuildProto, nullptr, nullptr, [](void* ptr) {
1001 delete static_cast<metrics::BuildProtoContext*>(ptr);
1002 });
1003 if (ret != SQLITE_OK)
1004 return util::ErrStatus("%s", sqlite3_errmsg(*db_));
1005 }
1006 return util::OkStatus();
1007 }
1008
ComputeMetric(const std::vector<std::string> & metric_names,std::vector<uint8_t> * metrics_proto)1009 util::Status TraceProcessorImpl::ComputeMetric(
1010 const std::vector<std::string>& metric_names,
1011 std::vector<uint8_t>* metrics_proto) {
1012 auto opt_idx = pool_.FindDescriptorIdx(".perfetto.protos.TraceMetrics");
1013 if (!opt_idx.has_value())
1014 return util::Status("Root metrics proto descriptor not found");
1015
1016 const auto& root_descriptor = pool_.descriptors()[opt_idx.value()];
1017 return metrics::ComputeMetrics(this, metric_names, sql_metrics_,
1018 root_descriptor, metrics_proto);
1019 }
1020
ComputeMetricText(const std::vector<std::string> & metric_names,TraceProcessor::MetricResultFormat format,std::string * metrics_string)1021 util::Status TraceProcessorImpl::ComputeMetricText(
1022 const std::vector<std::string>& metric_names,
1023 TraceProcessor::MetricResultFormat format,
1024 std::string* metrics_string) {
1025 std::vector<uint8_t> metrics_proto;
1026 util::Status status = ComputeMetric(metric_names, &metrics_proto);
1027 if (!status.ok())
1028 return status;
1029 switch (format) {
1030 case TraceProcessor::MetricResultFormat::kProtoText:
1031 *metrics_string = protozero_to_text::ProtozeroToText(
1032 pool_, ".perfetto.protos.TraceMetrics",
1033 protozero::ConstBytes{metrics_proto.data(), metrics_proto.size()},
1034 protozero_to_text::kIncludeNewLines);
1035 break;
1036 case TraceProcessor::MetricResultFormat::kJson:
1037 // TODO(dproy): Implement this.
1038 PERFETTO_FATAL("Json formatted metrics not supported yet.");
1039 break;
1040 }
1041 return status;
1042 }
1043
GetMetricDescriptors()1044 std::vector<uint8_t> TraceProcessorImpl::GetMetricDescriptors() {
1045 return pool_.SerializeAsDescriptorSet();
1046 }
1047
EnableMetatrace()1048 void TraceProcessorImpl::EnableMetatrace() {
1049 metatrace::Enable();
1050 }
1051
DisableAndReadMetatrace(std::vector<uint8_t> * trace_proto)1052 util::Status TraceProcessorImpl::DisableAndReadMetatrace(
1053 std::vector<uint8_t>* trace_proto) {
1054 protozero::HeapBuffered<protos::pbzero::Trace> trace;
1055 metatrace::DisableAndReadBuffer([&trace](metatrace::Record* record) {
1056 auto packet = trace->add_packet();
1057 packet->set_timestamp(record->timestamp_ns);
1058 auto* evt = packet->set_perfetto_metatrace();
1059 evt->set_event_name(record->event_name);
1060 evt->set_event_duration_ns(record->duration_ns);
1061 evt->set_thread_id(1); // Not really important, just required for the ui.
1062
1063 if (record->args_buffer_size == 0)
1064 return;
1065
1066 base::StringSplitter s(record->args_buffer, record->args_buffer_size, '\0');
1067 for (; s.Next();) {
1068 auto* arg_proto = evt->add_args();
1069 arg_proto->set_key(s.cur_token());
1070
1071 bool has_next = s.Next();
1072 PERFETTO_CHECK(has_next);
1073 arg_proto->set_value(s.cur_token());
1074 }
1075 });
1076 *trace_proto = trace.SerializeAsArray();
1077 return util::OkStatus();
1078 }
1079
1080 } // namespace trace_processor
1081 } // namespace perfetto
1082