1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4from telemetry.timeline import chrome_trace_category_filter
5from telemetry.timeline import tracing_config
6from telemetry.web_perf import story_test
7
8# TimelineBasedMeasurement considers all instrumentation as producing a single
9# timeline. But, depending on the amount of instrumentation that is enabled,
10# overhead increases. The user of the measurement must therefore chose between
11# a few levels of instrumentation.
12LOW_OVERHEAD_LEVEL = 'low-overhead'
13DEFAULT_OVERHEAD_LEVEL = 'default-overhead'
14DEBUG_OVERHEAD_LEVEL = 'debug-overhead'
15
16ALL_OVERHEAD_LEVELS = [
17    LOW_OVERHEAD_LEVEL,
18    DEFAULT_OVERHEAD_LEVEL,
19    DEBUG_OVERHEAD_LEVEL,
20]
21
22
23class Options(object):
24  """A class to be used to configure TimelineBasedMeasurement.
25
26  This is created and returned by
27  Benchmark.CreateCoreTimelineBasedMeasurementOptions.
28
29  """
30
31  def __init__(self, overhead_level=LOW_OVERHEAD_LEVEL):
32    """As the amount of instrumentation increases, so does the overhead.
33    The user of the measurement chooses the overhead level that is appropriate,
34    and the tracing is filtered accordingly.
35
36    overhead_level: Can either be a custom ChromeTraceCategoryFilter object or
37        one of LOW_OVERHEAD_LEVEL, DEFAULT_OVERHEAD_LEVEL or
38        DEBUG_OVERHEAD_LEVEL.
39    """
40    self._config = tracing_config.TracingConfig()
41    self._config.enable_chrome_trace = True
42    self._config.enable_platform_display_trace = False
43
44    if isinstance(overhead_level,
45                  chrome_trace_category_filter.ChromeTraceCategoryFilter):
46      self._config.chrome_trace_config.SetCategoryFilter(overhead_level)
47    elif overhead_level in ALL_OVERHEAD_LEVELS:
48      if overhead_level == LOW_OVERHEAD_LEVEL:
49        self._config.chrome_trace_config.SetLowOverheadFilter()
50      elif overhead_level == DEFAULT_OVERHEAD_LEVEL:
51        self._config.chrome_trace_config.SetDefaultOverheadFilter()
52      else:
53        self._config.chrome_trace_config.SetDebugOverheadFilter()
54    else:
55      raise Exception("Overhead level must be a ChromeTraceCategoryFilter "
56                      "object or valid overhead level string. Given overhead "
57                      "level: %s" % overhead_level)
58
59    self._timeline_based_metrics = None
60
61
62  def ExtendTraceCategoryFilter(self, filters):
63    for category_filter in filters:
64      self.AddTraceCategoryFilter(category_filter)
65
66  def AddTraceCategoryFilter(self, category_filter):
67    self._config.chrome_trace_config.category_filter.AddFilter(category_filter)
68
69  @property
70  def category_filter(self):
71    return self._config.chrome_trace_config.category_filter
72
73  @property
74  def config(self):
75    return self._config
76
77  def ExtendTimelineBasedMetric(self, metrics):
78    for metric in metrics:
79      self.AddTimelineBasedMetric(metric)
80
81  def AddTimelineBasedMetric(self, metric):
82    assert isinstance(metric, basestring)
83    if self._timeline_based_metrics is None:
84      self._timeline_based_metrics = []
85    self._timeline_based_metrics.append(metric)
86
87  def SetTimelineBasedMetrics(self, metrics):
88    """Sets the Timeline Based Metrics (TBM) to run.
89
90    TBMv2 metrics are assumed to live in catapult //tracing/tracing/metrics;
91    for a metric defined e.g. in 'sample_metric.html' you should pass
92    'tbmv2:sampleMetric' or just 'sampleMetric' (note camel cased names).
93
94    TBMv3 metrics live in chromium //tools/perf/core/tbmv3/metrics, for a
95    metric defined e.g. in a 'dummy_metric.sql' file you should pass the
96    name 'tbmv3:dummy_metric'.
97
98    Args:
99      metrics: A list of strings with metric names as described above.
100    """
101    assert isinstance(metrics, list)
102    for metric in metrics:
103      assert isinstance(metric, basestring)
104    self._timeline_based_metrics = metrics
105
106  def GetTimelineBasedMetrics(self):
107    return self._timeline_based_metrics or []
108
109
110class TimelineBasedMeasurement(story_test.StoryTest):
111  """Collects multiple metrics based on their interaction records.
112
113  A timeline based measurement shifts the burden of what metrics to collect onto
114  the story under test. Instead of the measurement
115  having a fixed set of values it collects, the story being tested
116  issues (via javascript) an Interaction record into the user timing API that
117  describing what is happening at that time, as well as a standardized set
118  of flags describing the semantics of the work being done. The
119  TimelineBasedMeasurement object collects a trace that includes both these
120  interaction records, and a user-chosen amount of performance data using
121  Telemetry's various timeline-producing APIs, tracing especially.
122
123  It then passes the recorded timeline to different TimelineBasedMetrics based
124  on those flags. As an example, this allows a single story run to produce
125  load timing data, smoothness data, critical jank information and overall cpu
126  usage information.
127
128  Args:
129      options: an instance of timeline_based_measurement.Options.
130  """
131  def __init__(self, options):
132    self._tbm_options = options
133
134  def WillRunStory(self, platform):
135    """Configure and start tracing."""
136    if self._tbm_options.config.enable_chrome_trace:
137      # Always enable 'blink.console' and 'v8.console' categories for:
138      # 1) Backward compat of chrome clock sync (https://crbug.com/646925).
139      # 2) Allows users to add trace event through javascript.
140      # 3) For the console error metric (https://crbug.com/880432).
141      # Note that these categories are extremely low-overhead, so this doesn't
142      # affect the tracing overhead budget much.
143      chrome_config = self._tbm_options.config.chrome_trace_config
144      chrome_config.category_filter.AddIncludedCategory('blink.console')
145      chrome_config.category_filter.AddIncludedCategory('v8.console')
146    platform.tracing_controller.StartTracing(self._tbm_options.config)
147
148  def Measure(self, platform, results):
149    """Collect all possible metrics and added them to results."""
150    platform.tracing_controller.RecordBenchmarkMetadata(results)
151    traces = platform.tracing_controller.StopTracing()
152    tbm_metrics = self._tbm_options.GetTimelineBasedMetrics()
153    tbm_metrics = (
154        self._tbm_options.GetTimelineBasedMetrics() +
155        results.current_story.GetExtraTracingMetrics())
156    assert tbm_metrics, (
157        'Please specify required metrics using SetTimelineBasedMetrics')
158    results.AddTraces(traces, tbm_metrics=tbm_metrics)
159    traces.CleanUpTraceData()
160
161  def DidRunStory(self, platform, results):
162    """Clean up after running the story."""
163    if platform.tracing_controller.is_tracing_running:
164      traces = platform.tracing_controller.StopTracing()
165      results.AddTraces(traces)
166      traces.CleanUpTraceData()
167
168  @property
169  def tbm_options(self):
170    return self._tbm_options
171