1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4from telemetry.timeline import chrome_trace_category_filter 5from telemetry.timeline import tracing_config 6from telemetry.web_perf import story_test 7 8# TimelineBasedMeasurement considers all instrumentation as producing a single 9# timeline. But, depending on the amount of instrumentation that is enabled, 10# overhead increases. The user of the measurement must therefore chose between 11# a few levels of instrumentation. 12LOW_OVERHEAD_LEVEL = 'low-overhead' 13DEFAULT_OVERHEAD_LEVEL = 'default-overhead' 14DEBUG_OVERHEAD_LEVEL = 'debug-overhead' 15 16ALL_OVERHEAD_LEVELS = [ 17 LOW_OVERHEAD_LEVEL, 18 DEFAULT_OVERHEAD_LEVEL, 19 DEBUG_OVERHEAD_LEVEL, 20] 21 22 23class Options(object): 24 """A class to be used to configure TimelineBasedMeasurement. 25 26 This is created and returned by 27 Benchmark.CreateCoreTimelineBasedMeasurementOptions. 28 29 """ 30 31 def __init__(self, overhead_level=LOW_OVERHEAD_LEVEL): 32 """As the amount of instrumentation increases, so does the overhead. 33 The user of the measurement chooses the overhead level that is appropriate, 34 and the tracing is filtered accordingly. 35 36 overhead_level: Can either be a custom ChromeTraceCategoryFilter object or 37 one of LOW_OVERHEAD_LEVEL, DEFAULT_OVERHEAD_LEVEL or 38 DEBUG_OVERHEAD_LEVEL. 39 """ 40 self._config = tracing_config.TracingConfig() 41 self._config.enable_chrome_trace = True 42 self._config.enable_platform_display_trace = False 43 44 if isinstance(overhead_level, 45 chrome_trace_category_filter.ChromeTraceCategoryFilter): 46 self._config.chrome_trace_config.SetCategoryFilter(overhead_level) 47 elif overhead_level in ALL_OVERHEAD_LEVELS: 48 if overhead_level == LOW_OVERHEAD_LEVEL: 49 self._config.chrome_trace_config.SetLowOverheadFilter() 50 elif overhead_level == DEFAULT_OVERHEAD_LEVEL: 51 self._config.chrome_trace_config.SetDefaultOverheadFilter() 52 else: 53 self._config.chrome_trace_config.SetDebugOverheadFilter() 54 else: 55 raise Exception("Overhead level must be a ChromeTraceCategoryFilter " 56 "object or valid overhead level string. Given overhead " 57 "level: %s" % overhead_level) 58 59 self._timeline_based_metrics = None 60 61 62 def ExtendTraceCategoryFilter(self, filters): 63 for category_filter in filters: 64 self.AddTraceCategoryFilter(category_filter) 65 66 def AddTraceCategoryFilter(self, category_filter): 67 self._config.chrome_trace_config.category_filter.AddFilter(category_filter) 68 69 @property 70 def category_filter(self): 71 return self._config.chrome_trace_config.category_filter 72 73 @property 74 def config(self): 75 return self._config 76 77 def ExtendTimelineBasedMetric(self, metrics): 78 for metric in metrics: 79 self.AddTimelineBasedMetric(metric) 80 81 def AddTimelineBasedMetric(self, metric): 82 assert isinstance(metric, basestring) 83 if self._timeline_based_metrics is None: 84 self._timeline_based_metrics = [] 85 self._timeline_based_metrics.append(metric) 86 87 def SetTimelineBasedMetrics(self, metrics): 88 """Sets the Timeline Based Metrics (TBM) to run. 89 90 TBMv2 metrics are assumed to live in catapult //tracing/tracing/metrics; 91 for a metric defined e.g. in 'sample_metric.html' you should pass 92 'tbmv2:sampleMetric' or just 'sampleMetric' (note camel cased names). 93 94 TBMv3 metrics live in chromium //tools/perf/core/tbmv3/metrics, for a 95 metric defined e.g. in a 'dummy_metric.sql' file you should pass the 96 name 'tbmv3:dummy_metric'. 97 98 Args: 99 metrics: A list of strings with metric names as described above. 100 """ 101 assert isinstance(metrics, list) 102 for metric in metrics: 103 assert isinstance(metric, basestring) 104 self._timeline_based_metrics = metrics 105 106 def GetTimelineBasedMetrics(self): 107 return self._timeline_based_metrics or [] 108 109 110class TimelineBasedMeasurement(story_test.StoryTest): 111 """Collects multiple metrics based on their interaction records. 112 113 A timeline based measurement shifts the burden of what metrics to collect onto 114 the story under test. Instead of the measurement 115 having a fixed set of values it collects, the story being tested 116 issues (via javascript) an Interaction record into the user timing API that 117 describing what is happening at that time, as well as a standardized set 118 of flags describing the semantics of the work being done. The 119 TimelineBasedMeasurement object collects a trace that includes both these 120 interaction records, and a user-chosen amount of performance data using 121 Telemetry's various timeline-producing APIs, tracing especially. 122 123 It then passes the recorded timeline to different TimelineBasedMetrics based 124 on those flags. As an example, this allows a single story run to produce 125 load timing data, smoothness data, critical jank information and overall cpu 126 usage information. 127 128 Args: 129 options: an instance of timeline_based_measurement.Options. 130 """ 131 def __init__(self, options): 132 self._tbm_options = options 133 134 def WillRunStory(self, platform): 135 """Configure and start tracing.""" 136 if self._tbm_options.config.enable_chrome_trace: 137 # Always enable 'blink.console' and 'v8.console' categories for: 138 # 1) Backward compat of chrome clock sync (https://crbug.com/646925). 139 # 2) Allows users to add trace event through javascript. 140 # 3) For the console error metric (https://crbug.com/880432). 141 # Note that these categories are extremely low-overhead, so this doesn't 142 # affect the tracing overhead budget much. 143 chrome_config = self._tbm_options.config.chrome_trace_config 144 chrome_config.category_filter.AddIncludedCategory('blink.console') 145 chrome_config.category_filter.AddIncludedCategory('v8.console') 146 platform.tracing_controller.StartTracing(self._tbm_options.config) 147 148 def Measure(self, platform, results): 149 """Collect all possible metrics and added them to results.""" 150 platform.tracing_controller.RecordBenchmarkMetadata(results) 151 traces = platform.tracing_controller.StopTracing() 152 tbm_metrics = self._tbm_options.GetTimelineBasedMetrics() 153 tbm_metrics = ( 154 self._tbm_options.GetTimelineBasedMetrics() + 155 results.current_story.GetExtraTracingMetrics()) 156 assert tbm_metrics, ( 157 'Please specify required metrics using SetTimelineBasedMetrics') 158 results.AddTraces(traces, tbm_metrics=tbm_metrics) 159 traces.CleanUpTraceData() 160 161 def DidRunStory(self, platform, results): 162 """Clean up after running the story.""" 163 if platform.tracing_controller.is_tracing_running: 164 traces = platform.tracing_controller.StopTracing() 165 results.AddTraces(traces) 166 traces.CleanUpTraceData() 167 168 @property 169 def tbm_options(self): 170 return self._tbm_options 171