1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "src/trace_processor/trace_processor_impl.h"
18 
19 #include <inttypes.h>
20 #include <algorithm>
21 
22 #include "perfetto/base/logging.h"
23 #include "perfetto/base/time.h"
24 #include "perfetto/ext/base/string_splitter.h"
25 #include "perfetto/ext/base/string_utils.h"
26 #include "src/trace_processor/dynamic/ancestor_slice_generator.h"
27 #include "src/trace_processor/dynamic/connected_flow_generator.h"
28 #include "src/trace_processor/dynamic/descendant_slice_generator.h"
29 #include "src/trace_processor/dynamic/describe_slice_generator.h"
30 #include "src/trace_processor/dynamic/experimental_counter_dur_generator.h"
31 #include "src/trace_processor/dynamic/experimental_flamegraph_generator.h"
32 #include "src/trace_processor/dynamic/experimental_sched_upid_generator.h"
33 #include "src/trace_processor/dynamic/experimental_slice_layout_generator.h"
34 #include "src/trace_processor/dynamic/thread_state_generator.h"
35 #include "src/trace_processor/export_json.h"
36 #include "src/trace_processor/importers/additional_modules.h"
37 #include "src/trace_processor/importers/ftrace/sched_event_tracker.h"
38 #include "src/trace_processor/importers/fuchsia/fuchsia_trace_parser.h"
39 #include "src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.h"
40 #include "src/trace_processor/importers/gzip/gzip_trace_parser.h"
41 #include "src/trace_processor/importers/json/json_trace_parser.h"
42 #include "src/trace_processor/importers/json/json_trace_tokenizer.h"
43 #include "src/trace_processor/importers/proto/metadata_tracker.h"
44 #include "src/trace_processor/importers/systrace/systrace_trace_parser.h"
45 #include "src/trace_processor/iterator_impl.h"
46 #include "src/trace_processor/sqlite/span_join_operator_table.h"
47 #include "src/trace_processor/sqlite/sql_stats_table.h"
48 #include "src/trace_processor/sqlite/sqlite3_str_split.h"
49 #include "src/trace_processor/sqlite/sqlite_raw_table.h"
50 #include "src/trace_processor/sqlite/sqlite_table.h"
51 #include "src/trace_processor/sqlite/sqlite_utils.h"
52 #include "src/trace_processor/sqlite/stats_table.h"
53 #include "src/trace_processor/sqlite/window_operator_table.h"
54 #include "src/trace_processor/tp_metatrace.h"
55 #include "src/trace_processor/types/variadic.h"
56 #include "src/trace_processor/util/protozero_to_text.h"
57 
58 #include "protos/perfetto/trace/perfetto/perfetto_metatrace.pbzero.h"
59 #include "protos/perfetto/trace/trace.pbzero.h"
60 #include "protos/perfetto/trace/trace_packet.pbzero.h"
61 
62 #include "src/trace_processor/metrics/chrome/all_chrome_metrics.descriptor.h"
63 #include "src/trace_processor/metrics/metrics.descriptor.h"
64 #include "src/trace_processor/metrics/metrics.h"
65 #include "src/trace_processor/metrics/sql_metrics.h"
66 
67 #if !PERFETTO_BUILDFLAG(PERFETTO_OS_WIN)
68 #include <cxxabi.h>
69 #endif
70 
71 // In Android and Chromium tree builds, we don't have the percentile module.
72 // Just don't include it.
73 #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
74 // defined in sqlite_src/ext/misc/percentile.c
75 extern "C" int sqlite3_percentile_init(sqlite3* db,
76                                        char** error,
77                                        const sqlite3_api_routines* api);
78 #endif  // PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
79 
80 namespace perfetto {
81 namespace trace_processor {
82 namespace {
83 
84 const char kAllTablesQuery[] =
85     "SELECT tbl_name, type FROM (SELECT * FROM sqlite_master UNION ALL SELECT "
86     "* FROM sqlite_temp_master)";
87 
InitializeSqlite(sqlite3 * db)88 void InitializeSqlite(sqlite3* db) {
89   char* error = nullptr;
90   sqlite3_exec(db, "PRAGMA temp_store=2", 0, 0, &error);
91   if (error) {
92     PERFETTO_FATAL("Error setting pragma temp_store: %s", error);
93   }
94   sqlite3_str_split_init(db);
95 // In Android tree builds, we don't have the percentile module.
96 // Just don't include it.
97 #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
98   sqlite3_percentile_init(db, &error, nullptr);
99   if (error) {
100     PERFETTO_ELOG("Error initializing: %s", error);
101     sqlite3_free(error);
102   }
103 #endif
104 }
105 
BuildBoundsTable(sqlite3 * db,std::pair<int64_t,int64_t> bounds)106 void BuildBoundsTable(sqlite3* db, std::pair<int64_t, int64_t> bounds) {
107   char* error = nullptr;
108   sqlite3_exec(db, "DELETE FROM trace_bounds", nullptr, nullptr, &error);
109   if (error) {
110     PERFETTO_ELOG("Error deleting from bounds table: %s", error);
111     sqlite3_free(error);
112     return;
113   }
114 
115   char* insert_sql = sqlite3_mprintf("INSERT INTO trace_bounds VALUES(%" PRId64
116                                      ", %" PRId64 ")",
117                                      bounds.first, bounds.second);
118 
119   sqlite3_exec(db, insert_sql, 0, 0, &error);
120   sqlite3_free(insert_sql);
121   if (error) {
122     PERFETTO_ELOG("Error inserting bounds table: %s", error);
123     sqlite3_free(error);
124   }
125 }
126 
CreateBuiltinTables(sqlite3 * db)127 void CreateBuiltinTables(sqlite3* db) {
128   char* error = nullptr;
129   sqlite3_exec(db, "CREATE TABLE perfetto_tables(name STRING)", 0, 0, &error);
130   if (error) {
131     PERFETTO_ELOG("Error initializing: %s", error);
132     sqlite3_free(error);
133   }
134   sqlite3_exec(db,
135                "CREATE TABLE trace_bounds(start_ts BIG INT, end_ts BIG INT)", 0,
136                0, &error);
137   if (error) {
138     PERFETTO_ELOG("Error initializing: %s", error);
139     sqlite3_free(error);
140   }
141   // Ensure that the entries in power_profile are unique to prevent duplicates
142   // when the power_profile is augmented with additional profiles.
143   sqlite3_exec(db,
144                "CREATE TABLE power_profile("
145                "device STRING, cpu INT, cluster INT, freq INT, power DOUBLE,"
146                "UNIQUE(device, cpu, cluster, freq));",
147                0, 0, &error);
148   if (error) {
149     PERFETTO_ELOG("Error initializing: %s", error);
150     sqlite3_free(error);
151   }
152   sqlite3_exec(db, "CREATE TABLE trace_metrics(name STRING)", 0, 0, &error);
153   if (error) {
154     PERFETTO_ELOG("Error initializing: %s", error);
155     sqlite3_free(error);
156   }
157   // This is a table intended to be used for metric debugging/developing. Data
158   // in the table is shown specially in the UI, and users can insert rows into
159   // this table to draw more things.
160   sqlite3_exec(db,
161                "CREATE TABLE debug_slices (id BIG INT, name STRING, ts BIG INT,"
162                "dur BIG INT, depth BIG INT)",
163                0, 0, &error);
164   if (error) {
165     PERFETTO_ELOG("Error initializing: %s", error);
166     sqlite3_free(error);
167   }
168 
169   // Initialize the bounds table with some data so even before parsing any data,
170   // we still have a valid table.
171   BuildBoundsTable(db, std::make_pair(0, 0));
172 }
173 
CreateBuiltinViews(sqlite3 * db)174 void CreateBuiltinViews(sqlite3* db) {
175   char* error = nullptr;
176   sqlite3_exec(db,
177                "CREATE VIEW counter_definitions AS "
178                "SELECT "
179                "  *, "
180                "  id AS counter_id "
181                "FROM counter_track",
182                0, 0, &error);
183   if (error) {
184     PERFETTO_ELOG("Error initializing: %s", error);
185     sqlite3_free(error);
186   }
187 
188   sqlite3_exec(db,
189                "CREATE VIEW counter_values AS "
190                "SELECT "
191                "  *, "
192                "  track_id as counter_id "
193                "FROM counter",
194                0, 0, &error);
195   if (error) {
196     PERFETTO_ELOG("Error initializing: %s", error);
197     sqlite3_free(error);
198   }
199 
200   sqlite3_exec(db,
201                "CREATE VIEW counters AS "
202                "SELECT * "
203                "FROM counter_values v "
204                "INNER JOIN counter_track t "
205                "ON v.track_id = t.id "
206                "ORDER BY ts;",
207                0, 0, &error);
208   if (error) {
209     PERFETTO_ELOG("Error initializing: %s", error);
210     sqlite3_free(error);
211   }
212 
213   sqlite3_exec(db,
214                "CREATE VIEW slice AS "
215                "SELECT "
216                "  *, "
217                "  category AS cat, "
218                "  id AS slice_id "
219                "FROM internal_slice;",
220                0, 0, &error);
221   if (error) {
222     PERFETTO_ELOG("Error initializing: %s", error);
223     sqlite3_free(error);
224   }
225 
226   sqlite3_exec(db,
227                "CREATE VIEW instants AS "
228                "SELECT "
229                "*, "
230                "0.0 as value "
231                "FROM instant;",
232                0, 0, &error);
233 
234   if (error) {
235     PERFETTO_ELOG("Error initializing: %s", error);
236     sqlite3_free(error);
237   }
238 
239   sqlite3_exec(db,
240                "CREATE VIEW sched AS "
241                "SELECT "
242                "*, "
243                "ts + dur as ts_end "
244                "FROM sched_slice;",
245                0, 0, &error);
246 
247   if (error) {
248     PERFETTO_ELOG("Error initializing: %s", error);
249     sqlite3_free(error);
250   }
251 
252   // Legacy view for "slice" table with a deprecated table name.
253   // TODO(eseckler): Remove this view when all users have switched to "slice".
254   sqlite3_exec(db,
255                "CREATE VIEW slices AS "
256                "SELECT * FROM slice;",
257                0, 0, &error);
258   if (error) {
259     PERFETTO_ELOG("Error initializing: %s", error);
260     sqlite3_free(error);
261   }
262 
263   sqlite3_exec(db,
264                "CREATE VIEW thread AS "
265                "SELECT "
266                "id as utid, "
267                "* "
268                "FROM internal_thread;",
269                0, 0, &error);
270   if (error) {
271     PERFETTO_ELOG("Error initializing: %s", error);
272     sqlite3_free(error);
273   }
274 
275   sqlite3_exec(db,
276                "CREATE VIEW process AS "
277                "SELECT "
278                "id as upid, "
279                "* "
280                "FROM internal_process;",
281                0, 0, &error);
282   if (error) {
283     PERFETTO_ELOG("Error initializing: %s", error);
284     sqlite3_free(error);
285   }
286 }
287 
ExportJson(sqlite3_context * ctx,int,sqlite3_value ** argv)288 void ExportJson(sqlite3_context* ctx, int /*argc*/, sqlite3_value** argv) {
289   TraceStorage* storage = static_cast<TraceStorage*>(sqlite3_user_data(ctx));
290   FILE* output;
291   if (sqlite3_value_type(argv[0]) == SQLITE_INTEGER) {
292     // Assume input is an FD.
293     output = fdopen(sqlite3_value_int(argv[0]), "w");
294     if (!output) {
295       sqlite3_result_error(ctx, "Couldn't open output file from given FD", -1);
296       return;
297     }
298   } else {
299     const char* filename =
300         reinterpret_cast<const char*>(sqlite3_value_text(argv[0]));
301     output = fopen(filename, "w");
302     if (!output) {
303       sqlite3_result_error(ctx, "Couldn't open output file", -1);
304       return;
305     }
306   }
307 
308   util::Status result = json::ExportJson(storage, output);
309   if (!result.ok()) {
310     sqlite3_result_error(ctx, result.message().c_str(), -1);
311     return;
312   }
313 }
314 
CreateJsonExportFunction(TraceStorage * ts,sqlite3 * db)315 void CreateJsonExportFunction(TraceStorage* ts, sqlite3* db) {
316   auto ret = sqlite3_create_function_v2(db, "EXPORT_JSON", 1, SQLITE_UTF8, ts,
317                                         ExportJson, nullptr, nullptr,
318                                         sqlite_utils::kSqliteStatic);
319   if (ret) {
320     PERFETTO_ELOG("Error initializing EXPORT_JSON");
321   }
322 }
323 
Hash(sqlite3_context * ctx,int argc,sqlite3_value ** argv)324 void Hash(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
325   base::Hash hash;
326   for (int i = 0; i < argc; ++i) {
327     sqlite3_value* value = argv[i];
328     switch (sqlite3_value_type(value)) {
329       case SQLITE_INTEGER:
330         hash.Update(sqlite3_value_int64(value));
331         break;
332       case SQLITE_TEXT: {
333         const char* ptr =
334             reinterpret_cast<const char*>(sqlite3_value_text(value));
335         hash.Update(ptr, strlen(ptr));
336         break;
337       }
338       default:
339         sqlite3_result_error(ctx, "Unsupported type of arg passed to HASH", -1);
340         return;
341     }
342   }
343   sqlite3_result_int64(ctx, static_cast<int64_t>(hash.digest()));
344 }
345 
Demangle(sqlite3_context * ctx,int argc,sqlite3_value ** argv)346 void Demangle(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
347   if (argc != 1) {
348     sqlite3_result_error(ctx, "Unsupported number of arg passed to DEMANGLE",
349                          -1);
350     return;
351   }
352   sqlite3_value* value = argv[0];
353   if (sqlite3_value_type(value) == SQLITE_NULL) {
354     sqlite3_result_null(ctx);
355     return;
356   }
357   if (sqlite3_value_type(value) != SQLITE_TEXT) {
358     sqlite3_result_error(ctx, "Unsupported type of arg passed to DEMANGLE", -1);
359     return;
360   }
361   const char* ptr = reinterpret_cast<const char*>(sqlite3_value_text(value));
362 #if !PERFETTO_BUILDFLAG(PERFETTO_OS_WIN)
363   int ignored = 0;
364   // This memory was allocated by malloc and will be passed to SQLite to free.
365   char* demangled_name = abi::__cxa_demangle(ptr, nullptr, nullptr, &ignored);
366   if (!demangled_name) {
367     sqlite3_result_null(ctx);
368     return;
369   }
370   sqlite3_result_text(ctx, demangled_name, -1, free);
371 #else
372   sqlite3_result_text(ctx, ptr, -1, sqlite_utils::kSqliteTransient);
373 #endif
374 }
375 
LastNonNullStep(sqlite3_context * ctx,int argc,sqlite3_value ** argv)376 void LastNonNullStep(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
377   if (argc != 1) {
378     sqlite3_result_error(
379         ctx, "Unsupported number of args passed to LAST_NON_NULL", -1);
380     return;
381   }
382   sqlite3_value* value = argv[0];
383   if (sqlite3_value_type(value) == SQLITE_NULL) {
384     return;
385   }
386   sqlite3_value** ptr = reinterpret_cast<sqlite3_value**>(
387       sqlite3_aggregate_context(ctx, sizeof(sqlite3_value*)));
388   if (ptr) {
389     if (*ptr != nullptr) {
390       sqlite3_value_free(*ptr);
391     }
392     *ptr = sqlite3_value_dup(value);
393   }
394 }
395 
LastNonNullInverse(sqlite3_context * ctx,int argc,sqlite3_value ** argv)396 void LastNonNullInverse(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
397   // Do nothing.
398   base::ignore_result(ctx);
399   base::ignore_result(argc);
400   base::ignore_result(argv);
401 }
402 
LastNonNullValue(sqlite3_context * ctx)403 void LastNonNullValue(sqlite3_context* ctx) {
404   sqlite3_value** ptr =
405       reinterpret_cast<sqlite3_value**>(sqlite3_aggregate_context(ctx, 0));
406   if (!ptr || !*ptr) {
407     sqlite3_result_null(ctx);
408   } else {
409     sqlite3_result_value(ctx, *ptr);
410   }
411 }
412 
LastNonNullFinal(sqlite3_context * ctx)413 void LastNonNullFinal(sqlite3_context* ctx) {
414   sqlite3_value** ptr =
415       reinterpret_cast<sqlite3_value**>(sqlite3_aggregate_context(ctx, 0));
416   if (!ptr || !*ptr) {
417     sqlite3_result_null(ctx);
418   } else {
419     sqlite3_result_value(ctx, *ptr);
420     sqlite3_value_free(*ptr);
421   }
422 }
423 
CreateHashFunction(sqlite3 * db)424 void CreateHashFunction(sqlite3* db) {
425   auto ret = sqlite3_create_function_v2(
426       db, "HASH", -1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, &Hash,
427       nullptr, nullptr, nullptr);
428   if (ret) {
429     PERFETTO_ELOG("Error initializing HASH");
430   }
431 }
432 
CreateDemangledNameFunction(sqlite3 * db)433 void CreateDemangledNameFunction(sqlite3* db) {
434   auto ret = sqlite3_create_function_v2(
435       db, "DEMANGLE", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, &Demangle,
436       nullptr, nullptr, nullptr);
437   if (ret != SQLITE_OK) {
438     PERFETTO_ELOG("Error initializing DEMANGLE: %s", sqlite3_errmsg(db));
439   }
440 }
441 
CreateLastNonNullFunction(sqlite3 * db)442 void CreateLastNonNullFunction(sqlite3* db) {
443   auto ret = sqlite3_create_window_function(
444       db, "LAST_NON_NULL", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr,
445       &LastNonNullStep, &LastNonNullFinal, &LastNonNullValue,
446       &LastNonNullInverse, nullptr);
447   if (ret) {
448     PERFETTO_ELOG("Error initializing LAST_NON_NULL");
449   }
450 }
451 
452 struct ValueAtMaxTsContext {
453   bool initialized;
454   int value_type;
455 
456   int64_t max_ts;
457   int64_t int_value_at_max_ts;
458   double double_value_at_max_ts;
459 };
460 
ValueAtMaxTsStep(sqlite3_context * ctx,int,sqlite3_value ** argv)461 void ValueAtMaxTsStep(sqlite3_context* ctx, int, sqlite3_value** argv) {
462   sqlite3_value* ts = argv[0];
463   sqlite3_value* value = argv[1];
464 
465   // Note that sqlite3_aggregate_context zeros the memory for us so all the
466   // variables of the struct should be zero.
467   ValueAtMaxTsContext* fn_ctx = reinterpret_cast<ValueAtMaxTsContext*>(
468       sqlite3_aggregate_context(ctx, sizeof(ValueAtMaxTsContext)));
469 
470   // For performance reasons, we only do the check for the type of ts and value
471   // on the first call of the function.
472   if (PERFETTO_UNLIKELY(!fn_ctx->initialized)) {
473     if (sqlite3_value_type(ts) != SQLITE_INTEGER) {
474       sqlite3_result_error(ctx, "VALUE_AT_MAX_TS: ts passed was not an integer",
475                            -1);
476       return;
477     }
478 
479     fn_ctx->value_type = sqlite3_value_type(value);
480     if (fn_ctx->value_type != SQLITE_INTEGER &&
481         fn_ctx->value_type != SQLITE_FLOAT) {
482       sqlite3_result_error(
483           ctx, "VALUE_AT_MAX_TS: value passed was not an integer or float", -1);
484       return;
485     }
486 
487     fn_ctx->initialized = true;
488   }
489 
490   // On dcheck builds however, we check every passed ts and value.
491 #if PERFETTO_DCHECK_IS_ON()
492   if (sqlite3_value_type(ts) != SQLITE_INTEGER) {
493     sqlite3_result_error(ctx, "VALUE_AT_MAX_TS: ts passed was not an integer",
494                          -1);
495     return;
496   }
497   if (sqlite3_value_type(value) != fn_ctx->value_type) {
498     sqlite3_result_error(ctx, "VALUE_AT_MAX_TS: value type is inconsistent",
499                          -1);
500     return;
501   }
502 #endif
503 
504   int64_t ts_int = sqlite3_value_int64(ts);
505   if (PERFETTO_LIKELY(fn_ctx->max_ts < ts_int)) {
506     fn_ctx->max_ts = ts_int;
507 
508     if (fn_ctx->value_type == SQLITE_INTEGER) {
509       fn_ctx->int_value_at_max_ts = sqlite3_value_int64(value);
510     } else {
511       fn_ctx->double_value_at_max_ts = sqlite3_value_double(value);
512     }
513   }
514 }
515 
ValueAtMaxTsFinal(sqlite3_context * ctx)516 void ValueAtMaxTsFinal(sqlite3_context* ctx) {
517   ValueAtMaxTsContext* fn_ctx =
518       reinterpret_cast<ValueAtMaxTsContext*>(sqlite3_aggregate_context(ctx, 0));
519   if (!fn_ctx) {
520     sqlite3_result_null(ctx);
521     return;
522   }
523   if (fn_ctx->value_type == SQLITE_INTEGER) {
524     sqlite3_result_int64(ctx, fn_ctx->int_value_at_max_ts);
525   } else {
526     sqlite3_result_double(ctx, fn_ctx->double_value_at_max_ts);
527   }
528 }
529 
CreateValueAtMaxTsFunction(sqlite3 * db)530 void CreateValueAtMaxTsFunction(sqlite3* db) {
531   auto ret = sqlite3_create_function_v2(
532       db, "VALUE_AT_MAX_TS", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr,
533       nullptr, &ValueAtMaxTsStep, &ValueAtMaxTsFinal, nullptr);
534   if (ret) {
535     PERFETTO_ELOG("Error initializing VALUE_AT_MAX_TS");
536   }
537 }
538 
ExtractArg(sqlite3_context * ctx,int argc,sqlite3_value ** argv)539 void ExtractArg(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
540   if (argc != 2) {
541     sqlite3_result_error(ctx, "EXTRACT_ARG: 2 args required", -1);
542     return;
543   }
544   if (sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
545     sqlite3_result_error(ctx, "EXTRACT_ARG: 1st argument should be arg set id",
546                          -1);
547     return;
548   }
549   if (sqlite3_value_type(argv[1]) != SQLITE_TEXT) {
550     sqlite3_result_error(ctx, "EXTRACT_ARG: 2nd argument should be key", -1);
551     return;
552   }
553 
554   TraceStorage* storage = static_cast<TraceStorage*>(sqlite3_user_data(ctx));
555   uint32_t arg_set_id = static_cast<uint32_t>(sqlite3_value_int(argv[0]));
556   const char* key = reinterpret_cast<const char*>(sqlite3_value_text(argv[1]));
557 
558   base::Optional<Variadic> opt_value;
559   util::Status status = storage->ExtractArg(arg_set_id, key, &opt_value);
560   if (!status.ok()) {
561     sqlite3_result_error(ctx, status.c_message(), -1);
562     return;
563   }
564 
565   if (!opt_value) {
566     sqlite3_result_null(ctx);
567     return;
568   }
569 
570   switch (opt_value->type) {
571     case Variadic::kInt:
572       sqlite3_result_int64(ctx, opt_value->int_value);
573       break;
574     case Variadic::kBool:
575       sqlite3_result_int64(ctx, opt_value->bool_value);
576       break;
577     case Variadic::kUint:
578       sqlite3_result_int64(ctx, static_cast<int64_t>(opt_value->uint_value));
579       break;
580     case Variadic::kPointer:
581       sqlite3_result_int64(ctx, static_cast<int64_t>(opt_value->pointer_value));
582       break;
583     case Variadic::kJson:
584       sqlite3_result_text(ctx, storage->GetString(opt_value->json_value).data(),
585                           -1, nullptr);
586       break;
587     case Variadic::kString:
588       sqlite3_result_text(
589           ctx, storage->GetString(opt_value->string_value).data(), -1, nullptr);
590       break;
591     case Variadic::kReal:
592       sqlite3_result_double(ctx, opt_value->real_value);
593       break;
594   }
595 }
596 
CreateExtractArgFunction(TraceStorage * ts,sqlite3 * db)597 void CreateExtractArgFunction(TraceStorage* ts, sqlite3* db) {
598   auto ret = sqlite3_create_function_v2(db, "EXTRACT_ARG", 2,
599                                         SQLITE_UTF8 | SQLITE_DETERMINISTIC, ts,
600                                         &ExtractArg, nullptr, nullptr, nullptr);
601   if (ret != SQLITE_OK) {
602     PERFETTO_FATAL("Error initializing EXTRACT_ARG: %s", sqlite3_errmsg(db));
603   }
604 }
605 
CreateSourceGeqFunction(sqlite3 * db)606 void CreateSourceGeqFunction(sqlite3* db) {
607   auto fn = [](sqlite3_context* ctx, int, sqlite3_value**) {
608     sqlite3_result_error(
609         ctx, "SOURCE_GEQ should not be called from the global scope", -1);
610   };
611   auto ret = sqlite3_create_function_v2(db, "SOURCE_GEQ", -1,
612                                         SQLITE_UTF8 | SQLITE_DETERMINISTIC,
613                                         nullptr, fn, nullptr, nullptr, nullptr);
614   if (ret != SQLITE_OK) {
615     PERFETTO_FATAL("Error initializing SOURCE_GEQ: %s", sqlite3_errmsg(db));
616   }
617 }
618 
SetupMetrics(TraceProcessor * tp,sqlite3 * db,std::vector<metrics::SqlMetricFile> * sql_metrics)619 void SetupMetrics(TraceProcessor* tp,
620                   sqlite3* db,
621                   std::vector<metrics::SqlMetricFile>* sql_metrics) {
622   tp->ExtendMetricsProto(kMetricsDescriptor.data(), kMetricsDescriptor.size());
623   tp->ExtendMetricsProto(kAllChromeMetricsDescriptor.data(),
624                          kAllChromeMetricsDescriptor.size());
625 
626   for (const auto& file_to_sql : metrics::sql_metrics::kFileToSql) {
627     tp->RegisterMetric(file_to_sql.path, file_to_sql.sql);
628   }
629 
630   {
631     std::unique_ptr<metrics::RunMetricContext> ctx(
632         new metrics::RunMetricContext());
633     ctx->tp = tp;
634     ctx->metrics = sql_metrics;
635     auto ret = sqlite3_create_function_v2(
636         db, "RUN_METRIC", -1, SQLITE_UTF8, ctx.release(), metrics::RunMetric,
637         nullptr, nullptr,
638         [](void* ptr) { delete static_cast<metrics::RunMetricContext*>(ptr); });
639     if (ret)
640       PERFETTO_FATAL("Error initializing RUN_METRIC");
641   }
642 
643   {
644     auto ret = sqlite3_create_function_v2(
645         db, "RepeatedField", 1, SQLITE_UTF8, nullptr, nullptr,
646         metrics::RepeatedFieldStep, metrics::RepeatedFieldFinal, nullptr);
647     if (ret)
648       PERFETTO_FATAL("Error initializing RepeatedField");
649   }
650 
651   {
652     auto ret = sqlite3_create_function_v2(db, "NULL_IF_EMPTY", 1, SQLITE_UTF8,
653                                           nullptr, metrics::NullIfEmpty,
654                                           nullptr, nullptr, nullptr);
655     if (ret)
656       PERFETTO_FATAL("Error initializing NULL_IF_EMPTY");
657   }
658 }
659 
EnsureSqliteInitialized()660 void EnsureSqliteInitialized() {
661   // sqlite3_initialize isn't actually thread-safe despite being documented
662   // as such; we need to make sure multiple TraceProcessorImpl instances don't
663   // call it concurrently and only gets called once per process, instead.
664   static bool init_once = [] { return sqlite3_initialize() == SQLITE_OK; }();
665   PERFETTO_CHECK(init_once);
666 }
667 
InsertIntoTraceMetricsTable(sqlite3 * db,const std::string & metric_name)668 void InsertIntoTraceMetricsTable(sqlite3* db, const std::string& metric_name) {
669   char* insert_sql = sqlite3_mprintf(
670       "INSERT INTO trace_metrics(name) VALUES('%q')", metric_name.c_str());
671   char* insert_error = nullptr;
672   sqlite3_exec(db, insert_sql, nullptr, nullptr, &insert_error);
673   sqlite3_free(insert_sql);
674   if (insert_error) {
675     PERFETTO_ELOG("Error registering table: %s", insert_error);
676     sqlite3_free(insert_error);
677   }
678 }
679 
680 }  // namespace
681 
TraceProcessorImpl(const Config & cfg)682 TraceProcessorImpl::TraceProcessorImpl(const Config& cfg)
683     : TraceProcessorStorageImpl(cfg) {
684   context_.fuchsia_trace_tokenizer.reset(new FuchsiaTraceTokenizer(&context_));
685   context_.fuchsia_trace_parser.reset(new FuchsiaTraceParser(&context_));
686 
687   context_.systrace_trace_parser.reset(new SystraceTraceParser(&context_));
688 
689   if (gzip::IsGzipSupported())
690     context_.gzip_trace_parser.reset(new GzipTraceParser(&context_));
691 
692   if (json::IsJsonSupported()) {
693     context_.json_trace_tokenizer.reset(new JsonTraceTokenizer(&context_));
694     context_.json_trace_parser.reset(new JsonTraceParser(&context_));
695   }
696 
697   RegisterAdditionalModules(&context_);
698 
699   sqlite3* db = nullptr;
700   EnsureSqliteInitialized();
701   PERFETTO_CHECK(sqlite3_open(":memory:", &db) == SQLITE_OK);
702   InitializeSqlite(db);
703   CreateBuiltinTables(db);
704   CreateBuiltinViews(db);
705   db_.reset(std::move(db));
706 
707   CreateJsonExportFunction(context_.storage.get(), db);
708   CreateHashFunction(db);
709   CreateDemangledNameFunction(db);
710   CreateLastNonNullFunction(db);
711   CreateExtractArgFunction(context_.storage.get(), db);
712   CreateSourceGeqFunction(db);
713   CreateValueAtMaxTsFunction(db);
714 
715   SetupMetrics(this, *db_, &sql_metrics_);
716 
717   // Setup the query cache.
718   query_cache_.reset(new QueryCache());
719 
720   const TraceStorage* storage = context_.storage.get();
721 
722   SqlStatsTable::RegisterTable(*db_, storage);
723   StatsTable::RegisterTable(*db_, storage);
724 
725   // Operator tables.
726   SpanJoinOperatorTable::RegisterTable(*db_, storage);
727   WindowOperatorTable::RegisterTable(*db_, storage);
728 
729   // New style tables but with some custom logic.
730   SqliteRawTable::RegisterTable(*db_, query_cache_.get(), &context_);
731 
732   // Tables dynamically generated at query time.
733   RegisterDynamicTable(std::unique_ptr<ExperimentalFlamegraphGenerator>(
734       new ExperimentalFlamegraphGenerator(&context_)));
735   RegisterDynamicTable(std::unique_ptr<ExperimentalCounterDurGenerator>(
736       new ExperimentalCounterDurGenerator(storage->counter_table())));
737   RegisterDynamicTable(std::unique_ptr<DescribeSliceGenerator>(
738       new DescribeSliceGenerator(&context_)));
739   RegisterDynamicTable(std::unique_ptr<ExperimentalSliceLayoutGenerator>(
740       new ExperimentalSliceLayoutGenerator(
741           context_.storage.get()->mutable_string_pool(),
742           &storage->slice_table())));
743   RegisterDynamicTable(std::unique_ptr<AncestorSliceGenerator>(
744       new AncestorSliceGenerator(&context_)));
745   RegisterDynamicTable(std::unique_ptr<DescendantSliceGenerator>(
746       new DescendantSliceGenerator(&context_)));
747   RegisterDynamicTable(
748       std::unique_ptr<ConnectedFlowGenerator>(new ConnectedFlowGenerator(
749           ConnectedFlowGenerator::Direction::BOTH, &context_)));
750   RegisterDynamicTable(
751       std::unique_ptr<ConnectedFlowGenerator>(new ConnectedFlowGenerator(
752           ConnectedFlowGenerator::Direction::FOLLOWING, &context_)));
753   RegisterDynamicTable(
754       std::unique_ptr<ConnectedFlowGenerator>(new ConnectedFlowGenerator(
755           ConnectedFlowGenerator::Direction::PRECEDING, &context_)));
756   RegisterDynamicTable(std::unique_ptr<ExperimentalSchedUpidGenerator>(
757       new ExperimentalSchedUpidGenerator(storage->sched_slice_table(),
758                                          storage->thread_table())));
759   RegisterDynamicTable(std::unique_ptr<ThreadStateGenerator>(
760       new ThreadStateGenerator(&context_)));
761 
762   // New style db-backed tables.
763   RegisterDbTable(storage->arg_table());
764   RegisterDbTable(storage->thread_table());
765   RegisterDbTable(storage->process_table());
766 
767   RegisterDbTable(storage->slice_table());
768   RegisterDbTable(storage->flow_table());
769   RegisterDbTable(storage->sched_slice_table());
770   RegisterDbTable(storage->instant_table());
771   RegisterDbTable(storage->gpu_slice_table());
772 
773   RegisterDbTable(storage->track_table());
774   RegisterDbTable(storage->thread_track_table());
775   RegisterDbTable(storage->process_track_table());
776   RegisterDbTable(storage->gpu_track_table());
777 
778   RegisterDbTable(storage->counter_table());
779 
780   RegisterDbTable(storage->counter_track_table());
781   RegisterDbTable(storage->process_counter_track_table());
782   RegisterDbTable(storage->thread_counter_track_table());
783   RegisterDbTable(storage->cpu_counter_track_table());
784   RegisterDbTable(storage->irq_counter_track_table());
785   RegisterDbTable(storage->softirq_counter_track_table());
786   RegisterDbTable(storage->gpu_counter_track_table());
787   RegisterDbTable(storage->gpu_counter_group_table());
788 
789   RegisterDbTable(storage->heap_graph_object_table());
790   RegisterDbTable(storage->heap_graph_reference_table());
791   RegisterDbTable(storage->heap_graph_class_table());
792 
793   RegisterDbTable(storage->symbol_table());
794   RegisterDbTable(storage->heap_profile_allocation_table());
795   RegisterDbTable(storage->cpu_profile_stack_sample_table());
796   RegisterDbTable(storage->perf_sample_table());
797   RegisterDbTable(storage->stack_profile_callsite_table());
798   RegisterDbTable(storage->stack_profile_mapping_table());
799   RegisterDbTable(storage->stack_profile_frame_table());
800   RegisterDbTable(storage->package_list_table());
801   RegisterDbTable(storage->profiler_smaps_table());
802 
803   RegisterDbTable(storage->android_log_table());
804 
805   RegisterDbTable(storage->vulkan_memory_allocations_table());
806 
807   RegisterDbTable(storage->graphics_frame_slice_table());
808 
809   RegisterDbTable(storage->metadata_table());
810   RegisterDbTable(storage->cpu_table());
811   RegisterDbTable(storage->cpu_freq_table());
812 
813   RegisterDbTable(storage->memory_snapshot_table());
814   RegisterDbTable(storage->process_memory_snapshot_table());
815   RegisterDbTable(storage->memory_snapshot_node_table());
816   RegisterDbTable(storage->memory_snapshot_edge_table());
817 }
818 
819 TraceProcessorImpl::~TraceProcessorImpl() = default;
820 
Parse(std::unique_ptr<uint8_t[]> data,size_t size)821 util::Status TraceProcessorImpl::Parse(std::unique_ptr<uint8_t[]> data,
822                                        size_t size) {
823   bytes_parsed_ += size;
824   return TraceProcessorStorageImpl::Parse(std::move(data), size);
825 }
826 
GetCurrentTraceName()827 std::string TraceProcessorImpl::GetCurrentTraceName() {
828   if (current_trace_name_.empty())
829     return "";
830   auto size = " (" + std::to_string(bytes_parsed_ / 1024 / 1024) + " MB)";
831   return current_trace_name_ + size;
832 }
833 
SetCurrentTraceName(const std::string & name)834 void TraceProcessorImpl::SetCurrentTraceName(const std::string& name) {
835   current_trace_name_ = name;
836 }
837 
NotifyEndOfFile()838 void TraceProcessorImpl::NotifyEndOfFile() {
839   if (current_trace_name_.empty())
840     current_trace_name_ = "Unnamed trace";
841 
842   TraceProcessorStorageImpl::NotifyEndOfFile();
843 
844   SchedEventTracker::GetOrCreate(&context_)->FlushPendingEvents();
845   context_.metadata_tracker->SetMetadata(
846       metadata::trace_size_bytes,
847       Variadic::Integer(static_cast<int64_t>(bytes_parsed_)));
848   BuildBoundsTable(*db_, context_.storage->GetTraceTimestampBoundsNs());
849 
850   // Create a snapshot of all tables and views created so far. This is so later
851   // we can drop all extra tables created by the UI and reset to the original
852   // state (see RestoreInitialTables).
853   initial_tables_.clear();
854   auto it = ExecuteQuery(kAllTablesQuery);
855   while (it.Next()) {
856     auto value = it.Get(0);
857     PERFETTO_CHECK(value.type == SqlValue::Type::kString);
858     initial_tables_.push_back(value.string_value);
859   }
860 }
861 
RestoreInitialTables()862 size_t TraceProcessorImpl::RestoreInitialTables() {
863   std::vector<std::pair<std::string, std::string>> deletion_list;
864   std::string msg = "Resetting DB to initial state, deleting table/views:";
865   for (auto it = ExecuteQuery(kAllTablesQuery); it.Next();) {
866     std::string name(it.Get(0).string_value);
867     std::string type(it.Get(1).string_value);
868     if (std::find(initial_tables_.begin(), initial_tables_.end(), name) ==
869         initial_tables_.end()) {
870       msg += " " + name;
871       deletion_list.push_back(std::make_pair(type, name));
872     }
873   }
874 
875   PERFETTO_LOG("%s", msg.c_str());
876   for (const auto& tn : deletion_list) {
877     std::string query = "DROP " + tn.first + " " + tn.second;
878     auto it = ExecuteQuery(query);
879     while (it.Next()) {
880     }
881     // Index deletion can legitimately fail. If one creates an index "i" on a
882     // table "t" but issues the deletion in the order (t, i), the DROP index i
883     // will fail with "no such index" because deleting the table "t"
884     // automatically deletes all associated indexes.
885     if (!it.Status().ok() && tn.first != "index")
886       PERFETTO_FATAL("%s -> %s", query.c_str(), it.Status().c_message());
887   }
888   return deletion_list.size();
889 }
890 
ExecuteQuery(const std::string & sql,int64_t time_queued)891 Iterator TraceProcessorImpl::ExecuteQuery(const std::string& sql,
892                                           int64_t time_queued) {
893   sqlite3_stmt* raw_stmt;
894   int err;
895   {
896     PERFETTO_TP_TRACE("QUERY_PREPARE");
897     err = sqlite3_prepare_v2(*db_, sql.c_str(), static_cast<int>(sql.size()),
898                              &raw_stmt, nullptr);
899   }
900 
901   util::Status status;
902   uint32_t col_count = 0;
903   if (err != SQLITE_OK) {
904     status = util::ErrStatus("%s", sqlite3_errmsg(*db_));
905   } else {
906     col_count = static_cast<uint32_t>(sqlite3_column_count(raw_stmt));
907   }
908 
909   base::TimeNanos t_start = base::GetWallTimeNs();
910   uint32_t sql_stats_row =
911       context_.storage->mutable_sql_stats()->RecordQueryBegin(sql, time_queued,
912                                                               t_start.count());
913 
914   std::unique_ptr<IteratorImpl> impl(new IteratorImpl(
915       this, *db_, ScopedStmt(raw_stmt), col_count, status, sql_stats_row));
916   return Iterator(std::move(impl));
917 }
918 
InterruptQuery()919 void TraceProcessorImpl::InterruptQuery() {
920   if (!db_)
921     return;
922   query_interrupted_.store(true);
923   sqlite3_interrupt(db_.get());
924 }
925 
IsRootMetricField(const std::string & metric_name)926 bool TraceProcessorImpl::IsRootMetricField(const std::string& metric_name) {
927   base::Optional<uint32_t> desc_idx =
928       pool_.FindDescriptorIdx(".perfetto.protos.TraceMetrics");
929   if (!desc_idx.has_value())
930     return false;
931   base::Optional<uint32_t> field_idx =
932       pool_.descriptors()[*desc_idx].FindFieldIdxByName(metric_name);
933   return field_idx.has_value();
934 }
935 
RegisterMetric(const std::string & path,const std::string & sql)936 util::Status TraceProcessorImpl::RegisterMetric(const std::string& path,
937                                                 const std::string& sql) {
938   std::string stripped_sql;
939   for (base::StringSplitter sp(sql, '\n'); sp.Next();) {
940     if (strncmp(sp.cur_token(), "--", 2) != 0) {
941       stripped_sql.append(sp.cur_token());
942       stripped_sql.push_back('\n');
943     }
944   }
945 
946   // Check if the metric with the given path already exists and if it does, just
947   // update the SQL associated with it.
948   auto it = std::find_if(
949       sql_metrics_.begin(), sql_metrics_.end(),
950       [&path](const metrics::SqlMetricFile& m) { return m.path == path; });
951   if (it != sql_metrics_.end()) {
952     it->sql = stripped_sql;
953     return util::OkStatus();
954   }
955 
956   auto sep_idx = path.rfind("/");
957   std::string basename =
958       sep_idx == std::string::npos ? path : path.substr(sep_idx + 1);
959 
960   auto sql_idx = basename.rfind(".sql");
961   if (sql_idx == std::string::npos) {
962     return util::ErrStatus("Unable to find .sql extension for metric");
963   }
964   auto no_ext_name = basename.substr(0, sql_idx);
965 
966   metrics::SqlMetricFile metric;
967   metric.path = path;
968   metric.sql = stripped_sql;
969 
970   if (IsRootMetricField(no_ext_name)) {
971     metric.proto_field_name = no_ext_name;
972     metric.output_table_name = no_ext_name + "_output";
973     InsertIntoTraceMetricsTable(*db_, no_ext_name);
974   }
975 
976   sql_metrics_.emplace_back(metric);
977   return util::OkStatus();
978 }
979 
ExtendMetricsProto(const uint8_t * data,size_t size)980 util::Status TraceProcessorImpl::ExtendMetricsProto(const uint8_t* data,
981                                                     size_t size) {
982   util::Status status = pool_.AddFromFileDescriptorSet(data, size);
983   if (!status.ok())
984     return status;
985 
986   for (const auto& desc : pool_.descriptors()) {
987     // Convert the full name (e.g. .perfetto.protos.TraceMetrics.SubMetric)
988     // into a function name of the form (TraceMetrics_SubMetric).
989     auto fn_name = desc.full_name().substr(desc.package_name().size() + 1);
990     std::replace(fn_name.begin(), fn_name.end(), '.', '_');
991 
992     std::unique_ptr<metrics::BuildProtoContext> ctx(
993         new metrics::BuildProtoContext());
994     ctx->tp = this;
995     ctx->pool = &pool_;
996     ctx->desc = &desc;
997 
998     auto ret = sqlite3_create_function_v2(
999         *db_, fn_name.c_str(), -1, SQLITE_UTF8, ctx.release(),
1000         metrics::BuildProto, nullptr, nullptr, [](void* ptr) {
1001           delete static_cast<metrics::BuildProtoContext*>(ptr);
1002         });
1003     if (ret != SQLITE_OK)
1004       return util::ErrStatus("%s", sqlite3_errmsg(*db_));
1005   }
1006   return util::OkStatus();
1007 }
1008 
ComputeMetric(const std::vector<std::string> & metric_names,std::vector<uint8_t> * metrics_proto)1009 util::Status TraceProcessorImpl::ComputeMetric(
1010     const std::vector<std::string>& metric_names,
1011     std::vector<uint8_t>* metrics_proto) {
1012   auto opt_idx = pool_.FindDescriptorIdx(".perfetto.protos.TraceMetrics");
1013   if (!opt_idx.has_value())
1014     return util::Status("Root metrics proto descriptor not found");
1015 
1016   const auto& root_descriptor = pool_.descriptors()[opt_idx.value()];
1017   return metrics::ComputeMetrics(this, metric_names, sql_metrics_,
1018                                  root_descriptor, metrics_proto);
1019 }
1020 
ComputeMetricText(const std::vector<std::string> & metric_names,TraceProcessor::MetricResultFormat format,std::string * metrics_string)1021 util::Status TraceProcessorImpl::ComputeMetricText(
1022     const std::vector<std::string>& metric_names,
1023     TraceProcessor::MetricResultFormat format,
1024     std::string* metrics_string) {
1025   std::vector<uint8_t> metrics_proto;
1026   util::Status status = ComputeMetric(metric_names, &metrics_proto);
1027   if (!status.ok())
1028     return status;
1029   switch (format) {
1030     case TraceProcessor::MetricResultFormat::kProtoText:
1031       *metrics_string = protozero_to_text::ProtozeroToText(
1032           pool_, ".perfetto.protos.TraceMetrics",
1033           protozero::ConstBytes{metrics_proto.data(), metrics_proto.size()},
1034           protozero_to_text::kIncludeNewLines);
1035       break;
1036     case TraceProcessor::MetricResultFormat::kJson:
1037       // TODO(dproy): Implement this.
1038       PERFETTO_FATAL("Json formatted metrics not supported yet.");
1039       break;
1040   }
1041   return status;
1042 }
1043 
GetMetricDescriptors()1044 std::vector<uint8_t> TraceProcessorImpl::GetMetricDescriptors() {
1045   return pool_.SerializeAsDescriptorSet();
1046 }
1047 
EnableMetatrace()1048 void TraceProcessorImpl::EnableMetatrace() {
1049   metatrace::Enable();
1050 }
1051 
DisableAndReadMetatrace(std::vector<uint8_t> * trace_proto)1052 util::Status TraceProcessorImpl::DisableAndReadMetatrace(
1053     std::vector<uint8_t>* trace_proto) {
1054   protozero::HeapBuffered<protos::pbzero::Trace> trace;
1055   metatrace::DisableAndReadBuffer([&trace](metatrace::Record* record) {
1056     auto packet = trace->add_packet();
1057     packet->set_timestamp(record->timestamp_ns);
1058     auto* evt = packet->set_perfetto_metatrace();
1059     evt->set_event_name(record->event_name);
1060     evt->set_event_duration_ns(record->duration_ns);
1061     evt->set_thread_id(1);  // Not really important, just required for the ui.
1062 
1063     if (record->args_buffer_size == 0)
1064       return;
1065 
1066     base::StringSplitter s(record->args_buffer, record->args_buffer_size, '\0');
1067     for (; s.Next();) {
1068       auto* arg_proto = evt->add_args();
1069       arg_proto->set_key(s.cur_token());
1070 
1071       bool has_next = s.Next();
1072       PERFETTO_CHECK(has_next);
1073       arg_proto->set_value(s.cur_token());
1074     }
1075   });
1076   *trace_proto = trace.SerializeAsArray();
1077   return util::OkStatus();
1078 }
1079 
1080 }  // namespace trace_processor
1081 }  // namespace perfetto
1082