1# Copyright 2019 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from datetime import date
6import logging
7import os
8import re
9import shutil
10import sys
11import tempfile
12
13from gpu_tests import color_profile_manager
14from gpu_tests import common_browser_args as cba
15from gpu_tests import gpu_integration_test
16from gpu_tests import path_util
17from gpu_tests.skia_gold import gpu_skia_gold_properties
18from gpu_tests.skia_gold import gpu_skia_gold_session_manager
19
20from py_utils import cloud_storage
21
22from telemetry.util import image_util
23
24GPU_RELATIVE_PATH = "content/test/data/gpu/"
25GPU_DATA_DIR = os.path.join(path_util.GetChromiumSrcDir(), GPU_RELATIVE_PATH)
26TEST_DATA_DIRS = [
27    GPU_DATA_DIR,
28    os.path.join(path_util.GetChromiumSrcDir(), 'media/test/data'),
29]
30
31SKIA_GOLD_CORPUS = 'chrome-gpu'
32
33
34class _ImageParameters(object):
35  def __init__(self):
36    # Parameters for cloud storage reference images.
37    self.vendor_id = None
38    self.device_id = None
39    self.vendor_string = None
40    self.device_string = None
41    self.msaa = False
42    self.model_name = None
43    self.driver_version = None
44    self.driver_vendor = None
45
46
47class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
48  """Base class for all tests that upload results to Skia Gold."""
49  # The command line options (which are passed to subclasses'
50  # GenerateGpuTests) *must* be configured here, via a call to
51  # SetParsedCommandLineOptions. If they are not, an error will be
52  # raised when running the tests.
53  _parsed_command_line_options = None
54
55  _error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests'
56
57  # This information is class-scoped, so that it can be shared across
58  # invocations of tests; but it's zapped every time the browser is
59  # restarted with different command line arguments.
60  _image_parameters = None
61
62  _skia_gold_temp_dir = None
63  _skia_gold_session_manager = None
64  _skia_gold_properties = None
65
66  @classmethod
67  def SetParsedCommandLineOptions(cls, options):
68    cls._parsed_command_line_options = options
69
70  @classmethod
71  def GetParsedCommandLineOptions(cls):
72    return cls._parsed_command_line_options
73
74  @classmethod
75  def SetUpProcess(cls):
76    options = cls.GetParsedCommandLineOptions()
77    color_profile_manager.ForceUntilExitSRGB(
78        options.dont_restore_color_profile_after_test)
79    super(SkiaGoldIntegrationTestBase, cls).SetUpProcess()
80    cls.CustomizeBrowserArgs([])
81    cls.StartBrowser()
82    cls.SetStaticServerDirs(TEST_DATA_DIRS)
83    cls._skia_gold_temp_dir = tempfile.mkdtemp()
84
85  @classmethod
86  def GetSkiaGoldProperties(cls):
87    if not cls._skia_gold_properties:
88      cls._skia_gold_properties =\
89          gpu_skia_gold_properties.GpuSkiaGoldProperties(
90              cls.GetParsedCommandLineOptions())
91    return cls._skia_gold_properties
92
93  @classmethod
94  def GetSkiaGoldSessionManager(cls):
95    if not cls._skia_gold_session_manager:
96      cls._skia_gold_session_manager =\
97          gpu_skia_gold_session_manager.GpuSkiaGoldSessionManager(
98              cls._skia_gold_temp_dir, cls.GetSkiaGoldProperties())
99    return cls._skia_gold_session_manager
100
101  @classmethod
102  def GenerateBrowserArgs(cls, additional_args):
103    """Adds default arguments to |additional_args|.
104
105    See the parent class' method documentation for additional information.
106    """
107    default_args = super(SkiaGoldIntegrationTestBase,
108                         cls).GenerateBrowserArgs(additional_args)
109    default_args.extend([cba.ENABLE_GPU_BENCHMARKING, cba.TEST_TYPE_GPU])
110    force_color_profile_arg = [
111        arg for arg in default_args if arg.startswith('--force-color-profile=')
112    ]
113    if not force_color_profile_arg:
114      default_args.extend([
115          cba.FORCE_COLOR_PROFILE_SRGB,
116          cba.ENSURE_FORCED_COLOR_PROFILE,
117      ])
118    return default_args
119
120  @classmethod
121  def StopBrowser(cls):
122    super(SkiaGoldIntegrationTestBase, cls).StopBrowser()
123    cls.ResetGpuInfo()
124
125  @classmethod
126  def TearDownProcess(cls):
127    super(SkiaGoldIntegrationTestBase, cls).TearDownProcess()
128    shutil.rmtree(cls._skia_gold_temp_dir)
129
130  @classmethod
131  def AddCommandlineArgs(cls, parser):
132    super(SkiaGoldIntegrationTestBase, cls).AddCommandlineArgs(parser)
133    parser.add_option(
134        '--git-revision', help='Chrome revision being tested.', default=None)
135    parser.add_option(
136        '--test-machine-name',
137        help='Name of the test machine. Specifying this argument causes this '
138        'script to upload failure images and diffs to cloud storage directly, '
139        'instead of relying on the archive_gpu_pixel_test_results.py script.',
140        default='')
141    parser.add_option(
142        '--dont-restore-color-profile-after-test',
143        dest='dont_restore_color_profile_after_test',
144        action='store_true',
145        default=False,
146        help='(Mainly on Mac) don\'t restore the system\'s original color '
147        'profile after the test completes; leave the system using the sRGB '
148        'color profile. See http://crbug.com/784456.')
149    parser.add_option(
150        '--gerrit-issue',
151        help='For Skia Gold integration. Gerrit issue ID.',
152        default='')
153    parser.add_option(
154        '--gerrit-patchset',
155        help='For Skia Gold integration. Gerrit patch set number.',
156        default='')
157    parser.add_option(
158        '--buildbucket-id',
159        help='For Skia Gold integration. Buildbucket build ID.',
160        default='')
161    parser.add_option(
162        '--no-skia-gold-failure',
163        action='store_true',
164        default=False,
165        help='For Skia Gold integration. Always report that the test passed '
166        'even if the Skia Gold image comparison reported a failure, but '
167        'otherwise perform the same steps as usual.')
168    # Telemetry is *still* using optparse instead of argparse, so we can't have
169    # these two options in a mutually exclusive group.
170    parser.add_option(
171        '--local-pixel-tests',
172        action='store_true',
173        default=None,
174        help='Specifies to run the test harness in local run mode or not. When '
175        'run in local mode, uploading to Gold is disabled and links to '
176        'help with local debugging are output. Running in local mode also '
177        'implies --no-luci-auth. If both this and --no-local-pixel-tests are '
178        'left unset, the test harness will attempt to detect whether it is '
179        'running on a workstation or not and set this option accordingly.')
180    parser.add_option(
181        '--no-local-pixel-tests',
182        action='store_false',
183        dest='local_pixel_tests',
184        help='Specifies to run the test harness in non-local (bot) mode. When '
185        'run in this mode, data is actually uploaded to Gold and triage links '
186        'arge generated. If both this and --local-pixel-tests are left unset, '
187        'the test harness will attempt to detect whether it is running on a '
188        'workstation or not and set this option accordingly.')
189    parser.add_option(
190        '--no-luci-auth',
191        action='store_true',
192        default=False,
193        help='Don\'t use the service account provided by LUCI for '
194        'authentication for Skia Gold, instead relying on gsutil to be '
195        'pre-authenticated. Meant for testing locally instead of on the bots.')
196    parser.add_option(
197        '--bypass-skia-gold-functionality',
198        action='store_true',
199        default=False,
200        help='Bypass all interaction with Skia Gold, effectively disabling the '
201        'image comparison portion of any tests that use Gold. Only meant to '
202        'be used in case a Gold outage occurs and cannot be fixed quickly.')
203
204  @classmethod
205  def ResetGpuInfo(cls):
206    cls._image_parameters = None
207
208  @classmethod
209  def GetImageParameters(cls, page):
210    if not cls._image_parameters:
211      cls._ComputeGpuInfo(page)
212    return cls._image_parameters
213
214  @classmethod
215  def _ComputeGpuInfo(cls, page):
216    if cls._image_parameters:
217      return
218    browser = cls.browser
219    system_info = browser.GetSystemInfo()
220    if not system_info:
221      raise Exception('System info must be supported by the browser')
222    if not system_info.gpu:
223      raise Exception('GPU information was absent')
224    device = system_info.gpu.devices[0]
225    cls._image_parameters = _ImageParameters()
226    params = cls._image_parameters
227    if device.vendor_id and device.device_id:
228      params.vendor_id = device.vendor_id
229      params.device_id = device.device_id
230    elif device.vendor_string and device.device_string:
231      params.vendor_string = device.vendor_string
232      params.device_string = device.device_string
233    elif page.gpu_process_disabled:
234      # Match the vendor and device IDs that the browser advertises
235      # when the software renderer is active.
236      params.vendor_id = 65535
237      params.device_id = 65535
238    else:
239      raise Exception('GPU device information was incomplete')
240    # TODO(senorblanco): This should probably be checking
241    # for the presence of the extensions in system_info.gpu_aux_attributes
242    # in order to check for MSAA, rather than sniffing the blocklist.
243    params.msaa = not (('disable_chromium_framebuffer_multisample' in
244                        system_info.gpu.driver_bug_workarounds) or
245                       ('disable_multisample_render_to_texture' in system_info.
246                        gpu.driver_bug_workarounds))
247    params.model_name = system_info.model_name
248    params.driver_version = device.driver_version
249    params.driver_vendor = device.driver_vendor
250
251  @classmethod
252  def _UploadBitmapToCloudStorage(cls, bucket, name, bitmap, public=False):
253    # This sequence of steps works on all platforms to write a temporary
254    # PNG to disk, following the pattern in bitmap_unittest.py. The key to
255    # avoiding PermissionErrors seems to be to not actually try to write to
256    # the temporary file object, but to re-open its name for all operations.
257    temp_file = tempfile.NamedTemporaryFile(suffix='.png').name
258    image_util.WritePngFile(bitmap, temp_file)
259    cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public)
260
261  # Not used consistently, but potentially useful for debugging issues on the
262  # bots, so kept around for future use.
263  @classmethod
264  def _UploadGoldErrorImageToCloudStorage(cls, image_name, screenshot):
265    revision = cls.GetSkiaGoldProperties().git_revision
266    machine_name = re.sub(r'\W+', '_',
267                          cls.GetParsedCommandLineOptions().test_machine_name)
268    base_bucket = '%s/gold_failures' % (cls._error_image_cloud_storage_bucket)
269    image_name_with_revision_and_machine = '%s_%s_%s.png' % (
270        image_name, machine_name, revision)
271    cls._UploadBitmapToCloudStorage(
272        base_bucket,
273        image_name_with_revision_and_machine,
274        screenshot,
275        public=True)
276
277  @staticmethod
278  def _UrlToImageName(url):
279    image_name = re.sub(r'^(http|https|file)://(/*)', '', url)
280    image_name = re.sub(r'\.\./', '', image_name)
281    image_name = re.sub(r'(\.|/|-)', '_', image_name)
282    return image_name
283
284  def GetGoldJsonKeys(self, page):
285    """Get all the JSON metadata that will be passed to golctl."""
286    img_params = self.GetImageParameters(page)
287    # The frequently changing last part of the ANGLE driver version (revision of
288    # some sort?) messes a bit with inexact matching since each revision will
289    # be treated as a separate trace, so strip it off.
290    _StripAngleRevisionFromDriver(img_params)
291    # All values need to be strings, otherwise goldctl fails.
292    gpu_keys = {
293        'vendor_id':
294        _ToHexOrNone(img_params.vendor_id),
295        'device_id':
296        _ToHexOrNone(img_params.device_id),
297        'vendor_string':
298        _ToNonEmptyStrOrNone(img_params.vendor_string),
299        'device_string':
300        _ToNonEmptyStrOrNone(img_params.device_string),
301        'msaa':
302        str(img_params.msaa),
303        'model_name':
304        _ToNonEmptyStrOrNone(img_params.model_name),
305        'os':
306        _ToNonEmptyStrOrNone(self.browser.platform.GetOSName()),
307        'os_version':
308        _ToNonEmptyStrOrNone(self.browser.platform.GetOSVersionName()),
309        'os_version_detail_string':
310        _ToNonEmptyStrOrNone(self.browser.platform.GetOSVersionDetailString()),
311        'driver_version':
312        _ToNonEmptyStrOrNone(img_params.driver_version),
313        'driver_vendor':
314        _ToNonEmptyStrOrNone(img_params.driver_vendor),
315        'combined_hardware_identifier':
316        _GetCombinedHardwareIdentifier(img_params),
317    }
318    # If we have a grace period active, then the test is potentially flaky.
319    # Include a pair that will cause Gold to ignore any untriaged images, which
320    # will prevent it from automatically commenting on unrelated CLs that happen
321    # to produce a new image.
322    if _GracePeriodActive(page):
323      gpu_keys['ignore'] = '1'
324    return gpu_keys
325
326  def _UploadTestResultToSkiaGold(self, image_name, screenshot, page):
327    """Compares the given image using Skia Gold and uploads the result.
328
329    No uploading is done if the test is being run in local run mode. Compares
330    the given screenshot to baselines provided by Gold, raising an Exception if
331    a match is not found.
332
333    Args:
334      image_name: the name of the image being checked.
335      screenshot: the image being checked as a Telemetry Bitmap.
336      page: the GPU PixelTestPage object for the test.
337    """
338    # Write screenshot to PNG file on local disk.
339    png_temp_file = tempfile.NamedTemporaryFile(
340        suffix='.png', dir=self._skia_gold_temp_dir).name
341    image_util.WritePngFile(screenshot, png_temp_file)
342
343    gpu_keys = self.GetGoldJsonKeys(page)
344    gold_session = self.GetSkiaGoldSessionManager().GetSkiaGoldSession(
345        gpu_keys, corpus=SKIA_GOLD_CORPUS)
346    gold_properties = self.GetSkiaGoldProperties()
347    use_luci = not (gold_properties.local_pixel_tests
348                    or gold_properties.no_luci_auth)
349
350    inexact_matching_args = page.matching_algorithm.GetCmdline()
351
352    status, error = gold_session.RunComparison(
353        name=image_name,
354        png_file=png_temp_file,
355        inexact_matching_args=inexact_matching_args,
356        use_luci=use_luci)
357    if not status:
358      return
359
360    status_codes =\
361        self.GetSkiaGoldSessionManager().GetSessionClass().StatusCodes
362    if status == status_codes.AUTH_FAILURE:
363      logging.error('Gold authentication failed with output %s', error)
364    elif status == status_codes.INIT_FAILURE:
365      logging.error('Gold initialization failed with output %s', error)
366    elif status == status_codes.COMPARISON_FAILURE_REMOTE:
367      # We currently don't have an internal instance + public mirror like the
368      # general Chrome Gold instance, so just report the "internal" link, which
369      # points to the correct instance.
370      _, triage_link = gold_session.GetTriageLinks(image_name)
371      if not triage_link:
372        logging.error('Failed to get triage link for %s, raw output: %s',
373                      image_name, error)
374        logging.error('Reason for no triage link: %s',
375                      gold_session.GetTriageLinkOmissionReason(image_name))
376      elif gold_properties.IsTryjobRun():
377        self.artifacts.CreateLink('triage_link_for_entire_cl', triage_link)
378      else:
379        self.artifacts.CreateLink('gold_triage_link', triage_link)
380    elif status == status_codes.COMPARISON_FAILURE_LOCAL:
381      logging.error('Local comparison failed. Local diff files:')
382      _OutputLocalDiffFiles(gold_session, image_name)
383    elif status == status_codes.LOCAL_DIFF_FAILURE:
384      logging.error(
385          'Local comparison failed and an error occurred during diff '
386          'generation: %s', error)
387      # There might be some files, so try outputting them.
388      logging.error('Local diff files:')
389      _OutputLocalDiffFiles(gold_session, image_name)
390    else:
391      logging.error(
392          'Given unhandled SkiaGoldSession StatusCode %s with error %s', status,
393          error)
394    if self._ShouldReportGoldFailure(page):
395      raise Exception('goldctl command failed, see above for details')
396
397  def _ShouldReportGoldFailure(self, page):
398    """Determines if a Gold failure should actually be surfaced.
399
400    Args:
401      page: The GPU PixelTestPage object for the test.
402
403    Returns:
404      True if the failure should be surfaced, i.e. the test should fail,
405      otherwise False.
406    """
407    parsed_options = self.GetParsedCommandLineOptions()
408    # Don't surface if we're explicitly told not to.
409    if parsed_options.no_skia_gold_failure:
410      return False
411    # Don't surface if the test was recently added and we're still within its
412    # grace period.
413    if _GracePeriodActive(page):
414      return False
415    return True
416
417  @classmethod
418  def GenerateGpuTests(cls, options):
419    del options
420    return []
421
422  def RunActualGpuTest(self, options):
423    raise NotImplementedError(
424        'RunActualGpuTest must be overridden in a subclass')
425
426
427def _ToHex(num):
428  return hex(int(num))
429
430
431def _ToHexOrNone(num):
432  return 'None' if num == None else _ToHex(num)
433
434
435def _ToNonEmptyStrOrNone(val):
436  return 'None' if val == '' else str(val)
437
438
439def _GracePeriodActive(page):
440  """Returns whether a grace period is currently active for a test.
441
442  Args:
443    page: The GPU PixelTestPage object for the test in question.
444
445  Returns:
446    True if a grace period is defined for |page| and has not yet expired.
447    Otherwise, False.
448  """
449  return page.grace_period_end and date.today() <= page.grace_period_end
450
451
452def _StripAngleRevisionFromDriver(img_params):
453  """Strips the revision off the end of an ANGLE driver version.
454
455  E.g. 2.1.0.b50541b2d6c4 -> 2.1.0
456
457  Modifies the string in place. No-ops if the driver vendor is not ANGLE.
458
459  Args:
460    img_params: An _ImageParameters instance to modify.
461  """
462  if 'ANGLE' not in img_params.driver_vendor or not img_params.driver_version:
463    return
464  # Assume that we're never going to have portions of the driver we care about
465  # that are longer than 8 characters.
466  driver_parts = img_params.driver_version.split('.')
467  kept_parts = []
468  for part in driver_parts:
469    if len(part) > 8:
470      break
471    kept_parts.append(part)
472  img_params.driver_version = '.'.join(kept_parts)
473
474
475def _GetCombinedHardwareIdentifier(img_params):
476  """Combine all relevant hardware identifiers into a single key.
477
478  This makes Gold forwarding more precise by allowing us to forward explicit
479  configurations instead of individual components.
480  """
481  vendor_id = _ToHexOrNone(img_params.vendor_id)
482  device_id = _ToHexOrNone(img_params.device_id)
483  device_string = _ToNonEmptyStrOrNone(img_params.device_string)
484  combined_hw_identifiers = ('vendor_id:{vendor_id}, '
485                             'device_id:{device_id}, '
486                             'device_string:{device_string}')
487  combined_hw_identifiers = combined_hw_identifiers.format(
488      vendor_id=vendor_id, device_id=device_id, device_string=device_string)
489  return combined_hw_identifiers
490
491
492def _OutputLocalDiffFiles(gold_session, image_name):
493  """Logs the local diff image files from the given SkiaGoldSession.
494
495  Args:
496    gold_session: A skia_gold_session.SkiaGoldSession instance to pull files
497        from.
498    image_name: A string containing the name of the image/test that was
499        compared.
500  """
501  given_file = gold_session.GetGivenImageLink(image_name)
502  closest_file = gold_session.GetClosestImageLink(image_name)
503  diff_file = gold_session.GetDiffImageLink(image_name)
504  failure_message = 'Unable to retrieve link'
505  logging.error('Generated image: %s', given_file or failure_message)
506  logging.error('Closest image: %s', closest_file or failure_message)
507  logging.error('Diff image: %s', diff_file or failure_message)
508
509
510def load_tests(loader, tests, pattern):
511  del loader, tests, pattern  # Unused.
512  return gpu_integration_test.LoadAllTestsInModule(sys.modules[__name__])
513