1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import print_function
6
7import collections
8import os
9import re
10
11from core import path_util
12from core import perf_benchmark
13
14from telemetry import benchmark
15from telemetry import page as page_module
16from telemetry.core import memory_cache_http_server
17from telemetry.page import legacy_page_test
18from telemetry.page import shared_page_state
19from telemetry import story
20from telemetry.timeline import bounds
21from telemetry.timeline import model as model_module
22from telemetry.timeline import tracing_config
23
24
25BLINK_PERF_BASE_DIR = os.path.join(path_util.GetChromiumSrcDir(),
26                                   'third_party', 'blink', 'perf_tests')
27SKIPPED_FILE = os.path.join(BLINK_PERF_BASE_DIR, 'Skipped')
28
29EventBoundary = collections.namedtuple('EventBoundary',
30                                       ['type', 'wall_time', 'thread_time'])
31
32MergedEvent = collections.namedtuple('MergedEvent',
33                                     ['bounds', 'thread_or_wall_duration'])
34
35
36class _BlinkPerfPage(page_module.Page):
37  def RunPageInteractions(self, action_runner):
38    action_runner.ExecuteJavaScript('testRunner.scheduleTestRun()')
39    action_runner.WaitForJavaScriptCondition('testRunner.isDone', timeout=600)
40
41def StoryNameFromUrl(url, prefix):
42  filename = url[len(prefix):].strip('/')
43  baseName, extension = filename.split('.')
44  if extension.find('?') != -1:
45    query = extension.split('?')[1]
46    baseName += "_" + query # So that queried page-names don't collide
47  return "{b}.{e}".format(b=baseName, e=extension)
48
49def CreateStorySetFromPath(path, skipped_file,
50                           shared_page_state_class=(
51                               shared_page_state.SharedPageState),
52                           append_query=None,
53                           extra_tags=None):
54  assert os.path.exists(path)
55
56  page_urls = []
57  serving_dirs = set()
58
59  def _AddPage(path):
60    if not path.endswith('.html'):
61      return
62    if '../' in open(path, 'r').read():
63      # If the page looks like it references its parent dir, include it.
64      serving_dirs.add(os.path.dirname(os.path.dirname(path)))
65    page_url = 'file://' + path.replace('\\', '/')
66    if append_query:
67      page_url += '?' + append_query
68    page_urls.append(page_url)
69
70
71  def _AddDir(dir_path, skipped):
72    for candidate_path in os.listdir(dir_path):
73      if candidate_path == 'resources':
74        continue
75      candidate_path = os.path.join(dir_path, candidate_path)
76      if candidate_path.startswith(skipped):
77        continue
78      if os.path.isdir(candidate_path):
79        _AddDir(candidate_path, skipped)
80      else:
81        _AddPage(candidate_path)
82
83  if os.path.isdir(path):
84    skipped = []
85    if os.path.exists(skipped_file):
86      for line in open(skipped_file, 'r').readlines():
87        line = line.strip()
88        if line and not line.startswith('#'):
89          skipped_path = os.path.join(os.path.dirname(skipped_file), line)
90          skipped.append(skipped_path.replace('/', os.sep))
91    _AddDir(path, tuple(skipped))
92  else:
93    _AddPage(path)
94  ps = story.StorySet(base_dir=os.getcwd() + os.sep,
95                      serving_dirs=serving_dirs)
96
97  all_urls = [p.rstrip('/') for p in page_urls]
98  common_prefix = os.path.dirname(os.path.commonprefix(all_urls))
99  for url in sorted(page_urls):
100    name = StoryNameFromUrl(url, common_prefix)
101    ps.AddStory(_BlinkPerfPage(
102        url, ps, ps.base_dir,
103        shared_page_state_class=shared_page_state_class,
104        name=name,
105        tags=extra_tags))
106  return ps
107
108def _CreateMergedEventsBoundaries(events, max_start_time):
109  """ Merge events with the given |event_name| and return a list of MergedEvent
110  objects. All events that are overlapping are megred together. Same as a union
111  operation.
112
113  Note: When merging multiple events, we approximate the thread_duration:
114  duration = (last.thread_start + last.thread_duration) - first.thread_start
115
116  Args:
117    events: a list of TimelineEvents
118    max_start_time: the maximum time that a TimelineEvent's start value can be.
119    Events past this this time will be ignored.
120
121  Returns:
122    a sorted list of MergedEvent objects which contain a Bounds object of the
123    wall time boundary and a thread_or_wall_duration which contains the thread
124    duration if possible, and otherwise the wall duration.
125  """
126  event_boundaries = []  # Contains EventBoundary objects.
127  merged_event_boundaries = []     # Contains MergedEvent objects.
128
129  # Deconstruct our trace events into boundaries, sort, and then create
130  # MergedEvents.
131  # This is O(N*log(N)), although this can be done in O(N) with fancy
132  # datastructures.
133  # Note: When merging multiple events, we approximate the thread_duration:
134  # dur = (last.thread_start + last.thread_duration) - first.thread_start
135  for event in events:
136    # Handle events where thread duration is None (async events).
137    thread_start = None
138    thread_end = None
139    if event.thread_start and event.thread_duration:
140      thread_start = event.thread_start
141      thread_end = event.thread_start + event.thread_duration
142    event_boundaries.append(
143        EventBoundary("start", event.start, thread_start))
144    event_boundaries.append(
145        EventBoundary("end", event.end, thread_end))
146  event_boundaries.sort(key=lambda e: e[1])
147
148  # Merge all trace events that overlap.
149  event_counter = 0
150  curr_bounds = None
151  curr_thread_start = None
152  for event_boundary in event_boundaries:
153    if event_boundary.type == "start":
154      event_counter += 1
155    else:
156      event_counter -= 1
157    # Initialization
158    if curr_bounds is None:
159      assert event_boundary.type == "start"
160      # Exit early if we reach the max time.
161      if event_boundary.wall_time > max_start_time:
162        return merged_event_boundaries
163      curr_bounds = bounds.Bounds()
164      curr_bounds.AddValue(event_boundary.wall_time)
165      curr_thread_start = event_boundary.thread_time
166      continue
167    # Create a the final bounds and thread duration when our event grouping
168    # is over.
169    if event_counter == 0:
170      curr_bounds.AddValue(event_boundary.wall_time)
171      thread_or_wall_duration = curr_bounds.bounds
172      if curr_thread_start and event_boundary.thread_time:
173        thread_or_wall_duration = event_boundary.thread_time - curr_thread_start
174      merged_event_boundaries.append(
175          MergedEvent(curr_bounds, thread_or_wall_duration))
176      curr_bounds = None
177  return merged_event_boundaries
178
179def _ComputeTraceEventsThreadTimeForBlinkPerf(
180    model, renderer_thread, trace_events_to_measure):
181  """ Compute the CPU duration for each of |trace_events_to_measure| during
182  blink_perf test.
183
184  Args:
185    renderer_thread: the renderer thread which run blink_perf test.
186    trace_events_to_measure: a list of string names of trace events to measure
187    CPU duration for.
188
189  Returns:
190    a dictionary in which each key is a trace event' name (from
191    |trace_events_to_measure| list), and value is a list of numbers that
192    represents to total cpu time of that trace events in each blink_perf test.
193  """
194  trace_cpu_time_metrics = {}
195
196  # Collect the bounds of "blink_perf.runTest" events.
197  test_runs_bounds = []
198  for event in renderer_thread.async_slices:
199    if event.name == "blink_perf.runTest":
200      test_runs_bounds.append(bounds.Bounds.CreateFromEvent(event))
201  test_runs_bounds.sort(key=lambda b: b.min)
202
203  for t in trace_events_to_measure:
204    trace_cpu_time_metrics[t] = [0.0] * len(test_runs_bounds)
205
206  # Handle case where there are no tests.
207  if not test_runs_bounds:
208    return trace_cpu_time_metrics
209
210  for event_name in trace_events_to_measure:
211    merged_event_boundaries = _CreateMergedEventsBoundaries(
212        model.IterAllEventsOfName(event_name), test_runs_bounds[-1].max)
213
214    curr_test_runs_bound_index = 0
215    for b in merged_event_boundaries:
216      if b.bounds.bounds == 0:
217        continue
218      # Fast forward (if needed) to the first relevant test.
219      while (curr_test_runs_bound_index < len(test_runs_bounds) and
220             b.bounds.min > test_runs_bounds[curr_test_runs_bound_index].max):
221        curr_test_runs_bound_index += 1
222      if curr_test_runs_bound_index >= len(test_runs_bounds):
223        break
224      # Add metrics for all intersecting tests, as there may be multiple
225      # tests that intersect with the event bounds.
226      start_index = curr_test_runs_bound_index
227      while (curr_test_runs_bound_index < len(test_runs_bounds) and
228             b.bounds.Intersects(
229                 test_runs_bounds[curr_test_runs_bound_index])):
230        intersect_wall_time = bounds.Bounds.GetOverlapBetweenBounds(
231            test_runs_bounds[curr_test_runs_bound_index], b.bounds)
232        intersect_cpu_or_wall_time = (
233            intersect_wall_time * b.thread_or_wall_duration / b.bounds.bounds)
234        trace_cpu_time_metrics[event_name][curr_test_runs_bound_index] += (
235            intersect_cpu_or_wall_time)
236        curr_test_runs_bound_index += 1
237      # Rewind to the last intersecting test as it might intersect with the
238      # next event.
239      curr_test_runs_bound_index = max(start_index,
240                                       curr_test_runs_bound_index - 1)
241  return trace_cpu_time_metrics
242
243
244class _BlinkPerfMeasurement(legacy_page_test.LegacyPageTest):
245  """Runs a blink performance test and reports the results."""
246
247  def __init__(self):
248    super(_BlinkPerfMeasurement, self).__init__()
249    with open(os.path.join(os.path.dirname(__file__),
250                           'blink_perf.js'), 'r') as f:
251      self._blink_perf_js = f.read()
252    self._is_tracing = False
253    self._extra_chrome_categories = None
254    self._enable_systrace = None
255
256  def WillNavigateToPage(self, page, tab):
257    del tab  # unused
258    page.script_to_evaluate_on_commit = self._blink_perf_js
259
260  def DidNavigateToPage(self, page, tab):
261    tab.WaitForJavaScriptCondition('testRunner.isWaitingForTelemetry')
262    self._StartTracingIfNeeded(tab)
263
264  def CustomizeBrowserOptions(self, options):
265    options.AppendExtraBrowserArgs([
266        '--js-flags=--expose_gc',
267        '--enable-experimental-web-platform-features',
268        '--autoplay-policy=no-user-gesture-required'
269    ])
270
271  def SetOptions(self, options):
272    super(_BlinkPerfMeasurement, self).SetOptions(options)
273    browser_type = options.browser_options.browser_type
274    if browser_type and 'content-shell' in browser_type:
275      options.AppendExtraBrowserArgs('--expose-internals-for-testing')
276    if options.extra_chrome_categories:
277      self._extra_chrome_categories = options.extra_chrome_categories
278    if options.enable_systrace:
279      self._enable_systrace = True
280
281  def _StartTracingIfNeeded(self, tab):
282    tracing_categories = tab.EvaluateJavaScript('testRunner.tracingCategories')
283    if (not tracing_categories and not self._extra_chrome_categories and
284        not self._enable_systrace):
285      return
286
287    self._is_tracing = True
288    config = tracing_config.TracingConfig()
289    config.enable_chrome_trace = True
290    config.chrome_trace_config.category_filter.AddFilterString(
291        'blink.console')  # This is always required for js land trace event
292    if tracing_categories:
293      config.chrome_trace_config.category_filter.AddFilterString(
294          tracing_categories)
295    if self._extra_chrome_categories:
296      config.chrome_trace_config.category_filter.AddFilterString(
297          self._extra_chrome_categories)
298    if self._enable_systrace:
299      config.chrome_trace_config.SetEnableSystrace()
300    tab.browser.platform.tracing_controller.StartTracing(config)
301
302
303  def PrintAndCollectTraceEventMetrics(self, trace_cpu_time_metrics, results):
304    unit = 'ms'
305    print()
306    for trace_event_name, cpu_times in trace_cpu_time_metrics.iteritems():
307      print('CPU times of trace event "%s":' % trace_event_name)
308      cpu_times_string = ', '.join(['{0:.10f}'.format(t) for t in cpu_times])
309      print('values %s %s' % (cpu_times_string, unit))
310      avg = 0.0
311      if cpu_times:
312        avg = sum(cpu_times)/len(cpu_times)
313      print('avg', '{0:.10f}'.format(avg), unit)
314      results.AddMeasurement(trace_event_name, unit, cpu_times)
315      print()
316    print('\n')
317
318  def ValidateAndMeasurePage(self, page, tab, results):
319    trace_cpu_time_metrics = {}
320    if self._is_tracing:
321      trace_data = tab.browser.platform.tracing_controller.StopTracing()
322      results.AddTraces(trace_data)
323      self._is_tracing = False
324
325      trace_events_to_measure = tab.EvaluateJavaScript(
326          'window.testRunner.traceEventsToMeasure')
327      if trace_events_to_measure:
328        model = model_module.TimelineModel(trace_data)
329        renderer_thread = model.GetFirstRendererThread(tab.id)
330        trace_cpu_time_metrics = _ComputeTraceEventsThreadTimeForBlinkPerf(
331            model, renderer_thread, trace_events_to_measure)
332
333    log = tab.EvaluateJavaScript('document.getElementById("log").innerHTML')
334
335    for line in log.splitlines():
336      if line.startswith("FATAL: "):
337        print(line)
338        continue
339      if not line.startswith('values '):
340        continue
341      parts = line.split()
342      values = [float(v.replace(',', '')) for v in parts[1:-1]]
343      units = parts[-1]
344      metric = page.name.split('.')[0].replace('/', '_')
345      if values:
346        results.AddMeasurement(metric, units, values)
347      else:
348        raise legacy_page_test.MeasurementFailure('Empty test results')
349
350      break
351
352    print(log)
353
354    self.PrintAndCollectTraceEventMetrics(trace_cpu_time_metrics, results)
355
356
357class _BlinkPerfBenchmark(perf_benchmark.PerfBenchmark):
358
359  test = _BlinkPerfMeasurement
360  TAGS = []
361
362  def CreateStorySet(self, options):
363    path = os.path.join(BLINK_PERF_BASE_DIR, self.SUBDIR)
364    return CreateStorySetFromPath(path, SKIPPED_FILE,
365                                  extra_tags=self.TAGS)
366
367
368@benchmark.Info(emails=['dmazzoni@chromium.org'],
369                component='Blink>Accessibility',
370                documentation_url='https://bit.ly/blink-perf-benchmarks')
371class BlinkPerfAccessibility(_BlinkPerfBenchmark):
372  SUBDIR = 'accessibility'
373  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
374
375  @classmethod
376  def Name(cls):
377    return 'blink_perf.accessibility'
378
379  def SetExtraBrowserOptions(self, options):
380    options.AppendExtraBrowserArgs([
381        '--force-renderer-accessibility',
382    ])
383
384
385@benchmark.Info(
386    component='Blink>Bindings',
387    emails=['jbroman@chromium.org', 'yukishiino@chromium.org',
388            'haraken@chromium.org'],
389    documentation_url='https://bit.ly/blink-perf-benchmarks')
390class BlinkPerfBindings(_BlinkPerfBenchmark):
391  SUBDIR = 'bindings'
392  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
393
394  @classmethod
395  def Name(cls):
396    return 'blink_perf.bindings'
397
398
399class ServiceWorkerRequestHandler(
400    memory_cache_http_server.MemoryCacheDynamicHTTPRequestHandler):
401  """This handler returns dynamic responses for service worker perf tests.
402  """
403  _SIZE_1K = 1024
404  _SIZE_10K = 10240
405  _SIZE_1M = 1048576
406  _FILE_NAME_PATTERN_1K =\
407      re.compile('.*/service_worker/resources/data/1K_[0-9]+\\.txt')
408
409  def ResponseFromHandler(self, path):
410    # normalize the path by replacing backslashes with slashes.
411    normpath = path.replace('\\', '/')
412    if normpath.endswith('/service_worker/resources/data/10K.txt'):
413      return self.MakeResponse('c' * self._SIZE_10K, 'text/plain', False)
414    elif normpath.endswith('/service_worker/resources/data/1M.txt'):
415      return self.MakeResponse('c' * self._SIZE_1M, 'text/plain', False)
416    elif self._FILE_NAME_PATTERN_1K.match(normpath):
417      return self.MakeResponse('c' * self._SIZE_1K, 'text/plain', False)
418    return None
419
420
421@benchmark.Info(
422    component='Blink>ServiceWorker',
423    emails=[
424        'shimazu@chromium.org', 'falken@chromium.org', 'ting.shao@intel.com'
425    ],
426    documentation_url='https://bit.ly/blink-perf-benchmarks')
427class BlinkPerfServiceWorker(_BlinkPerfBenchmark):
428  SUBDIR = 'service_worker'
429
430  @classmethod
431  def Name(cls):
432    return 'UNSCHEDULED_blink_perf.service_worker'
433
434  def CreateStorySet(self, options):
435    story_set = super(BlinkPerfServiceWorker, self).CreateStorySet(options)
436    story_set.SetRequestHandlerClass(ServiceWorkerRequestHandler)
437    return story_set
438
439
440@benchmark.Info(emails=['futhark@chromium.org', 'andruud@chromium.org'],
441                documentation_url='https://bit.ly/blink-perf-benchmarks',
442                component='Blink>CSS')
443class BlinkPerfCSS(_BlinkPerfBenchmark):
444  SUBDIR = 'css'
445  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
446
447  @classmethod
448  def Name(cls):
449    return 'blink_perf.css'
450
451@benchmark.Info(emails=['masonfreed@chromium.org'],
452                component='Blink>DOM',
453                documentation_url='https://bit.ly/blink-perf-benchmarks')
454class BlinkPerfDOM(_BlinkPerfBenchmark):
455  SUBDIR = 'dom'
456  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
457
458  @classmethod
459  def Name(cls):
460    return 'blink_perf.dom'
461
462
463@benchmark.Info(emails=['masonfreed@chromium.org'],
464                component='Blink>DOM',
465                documentation_url='https://bit.ly/blink-perf-benchmarks')
466class BlinkPerfEvents(_BlinkPerfBenchmark):
467  SUBDIR = 'events'
468  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
469
470  @classmethod
471  def Name(cls):
472    return 'blink_perf.events'
473
474  # TODO(yoichio): Migrate EventsDispatching tests to V1 and remove this flags
475  # crbug.com/937716.
476  def SetExtraBrowserOptions(self, options):
477    options.AppendExtraBrowserArgs(['--enable-blink-features=ShadowDOMV0'])
478
479
480@benchmark.Info(emails=['cblume@chromium.org'],
481                component='Internals>Images>Codecs',
482                documentation_url='https://bit.ly/blink-perf-benchmarks')
483class BlinkPerfImageDecoder(_BlinkPerfBenchmark):
484  SUBDIR = 'image_decoder'
485  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
486
487  @classmethod
488  def Name(cls):
489    return 'blink_perf.image_decoder'
490
491  def SetExtraBrowserOptions(self, options):
492    options.AppendExtraBrowserArgs([
493        '--enable-blink-features=JSImageDecode',
494    ])
495
496
497@benchmark.Info(
498    emails=['ikilpatrick@chromium.org', 'kojii@chromium.org'],
499    component='Blink>Layout',
500    documentation_url='https://bit.ly/blink-perf-benchmarks')
501class BlinkPerfLayout(_BlinkPerfBenchmark):
502  SUBDIR = 'layout'
503  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
504
505  @classmethod
506  def Name(cls):
507    return 'blink_perf.layout'
508
509
510@benchmark.Info(emails=['dmurph@chromium.org'],
511                component='Blink>Storage',
512                documentation_url='https://bit.ly/blink-perf-benchmarks')
513class BlinkPerfOWPStorage(_BlinkPerfBenchmark):
514  SUBDIR = 'owp_storage'
515  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
516
517  @classmethod
518  def Name(cls):
519    return 'blink_perf.owp_storage'
520
521  # This ensures that all blobs >= 20MB will be transported by files.
522  def SetExtraBrowserOptions(self, options):
523    options.AppendExtraBrowserArgs([
524        '--blob-transport-by-file-trigger=307300',
525        '--blob-transport-min-file-size=2048',
526        '--blob-transport-max-file-size=10240',
527        '--blob-transport-shared-memory-max-size=30720'
528    ])
529
530
531@benchmark.Info(emails=['pdr@chromium.org', 'wangxianzhu@chromium.org'],
532                component='Blink>Paint',
533                documentation_url='https://bit.ly/blink-perf-benchmarks')
534class BlinkPerfPaint(_BlinkPerfBenchmark):
535  SUBDIR = 'paint'
536  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
537
538  @classmethod
539  def Name(cls):
540    return 'blink_perf.paint'
541
542
543@benchmark.Info(component='Blink>Bindings',
544                emails=['jbroman@chromium.org',
545                         'yukishiino@chromium.org',
546                         'haraken@chromium.org'],
547                documentation_url='https://bit.ly/blink-perf-benchmarks')
548class BlinkPerfParser(_BlinkPerfBenchmark):
549  SUBDIR = 'parser'
550  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
551
552  @classmethod
553  def Name(cls):
554    return 'blink_perf.parser'
555
556
557@benchmark.Info(emails=['fs@opera.com', 'pdr@chromium.org'],
558                component='Blink>SVG',
559                documentation_url='https://bit.ly/blink-perf-benchmarks')
560class BlinkPerfSVG(_BlinkPerfBenchmark):
561  SUBDIR = 'svg'
562  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
563
564  @classmethod
565  def Name(cls):
566    return 'blink_perf.svg'
567
568
569@benchmark.Info(emails=['masonfreed@chromium.org'],
570                component='Blink>DOM>ShadowDOM',
571                documentation_url='https://bit.ly/blink-perf-benchmarks')
572class BlinkPerfShadowDOM(_BlinkPerfBenchmark):
573  SUBDIR = 'shadow_dom'
574  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
575
576  @classmethod
577  def Name(cls):
578    return 'blink_perf.shadow_dom'
579
580  # TODO(yoichio): Migrate shadow-style-share tests to V1 and remove this flags
581  # crbug.com/937716.
582  def SetExtraBrowserOptions(self, options):
583    options.AppendExtraBrowserArgs(['--enable-blink-features=ShadowDOMV0'])
584
585@benchmark.Info(emails=['vmpstr@chromium.org'],
586                component='Blink>Paint',
587                documentation_url='https://bit.ly/blink-perf-benchmarks')
588class BlinkPerfDisplayLocking(_BlinkPerfBenchmark):
589  SUBDIR = 'display_locking'
590  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
591
592  @classmethod
593  def Name(cls):
594    return 'blink_perf.display_locking'
595
596  def SetExtraBrowserOptions(self, options):
597    options.AppendExtraBrowserArgs(
598      ['--enable-blink-features=DisplayLocking,CSSContentSize'])
599
600@benchmark.Info(emails=['hongchan@chromium.org', 'rtoy@chromium.org'],
601                component='Blink>WebAudio',
602                documentation_url='https://bit.ly/blink-perf-benchmarks')
603class BlinkPerfWebAudio(_BlinkPerfBenchmark):
604  SUBDIR = 'webaudio'
605  TAGS = _BlinkPerfBenchmark.TAGS + ['all']
606
607  @classmethod
608  def Name(cls):
609    return 'blink_perf.webaudio'
610