1# Copyright 2019 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from datetime import date 6import logging 7import os 8import re 9import shutil 10import sys 11import tempfile 12 13from gpu_tests import color_profile_manager 14from gpu_tests import common_browser_args as cba 15from gpu_tests import gpu_integration_test 16from gpu_tests import path_util 17from gpu_tests.skia_gold import gpu_skia_gold_properties 18from gpu_tests.skia_gold import gpu_skia_gold_session_manager 19 20from py_utils import cloud_storage 21 22from telemetry.util import image_util 23 24GPU_RELATIVE_PATH = "content/test/data/gpu/" 25GPU_DATA_DIR = os.path.join(path_util.GetChromiumSrcDir(), GPU_RELATIVE_PATH) 26TEST_DATA_DIRS = [ 27 GPU_DATA_DIR, 28 os.path.join(path_util.GetChromiumSrcDir(), 'media/test/data'), 29] 30 31SKIA_GOLD_CORPUS = 'chrome-gpu' 32 33 34class _ImageParameters(object): 35 def __init__(self): 36 # Parameters for cloud storage reference images. 37 self.vendor_id = None 38 self.device_id = None 39 self.vendor_string = None 40 self.device_string = None 41 self.msaa = False 42 self.model_name = None 43 self.driver_version = None 44 self.driver_vendor = None 45 46 47class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest): 48 """Base class for all tests that upload results to Skia Gold.""" 49 # The command line options (which are passed to subclasses' 50 # GenerateGpuTests) *must* be configured here, via a call to 51 # SetParsedCommandLineOptions. If they are not, an error will be 52 # raised when running the tests. 53 _parsed_command_line_options = None 54 55 _error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests' 56 57 # This information is class-scoped, so that it can be shared across 58 # invocations of tests; but it's zapped every time the browser is 59 # restarted with different command line arguments. 60 _image_parameters = None 61 62 _skia_gold_temp_dir = None 63 _skia_gold_session_manager = None 64 _skia_gold_properties = None 65 66 @classmethod 67 def SetParsedCommandLineOptions(cls, options): 68 cls._parsed_command_line_options = options 69 70 @classmethod 71 def GetParsedCommandLineOptions(cls): 72 return cls._parsed_command_line_options 73 74 @classmethod 75 def SetUpProcess(cls): 76 options = cls.GetParsedCommandLineOptions() 77 color_profile_manager.ForceUntilExitSRGB( 78 options.dont_restore_color_profile_after_test) 79 super(SkiaGoldIntegrationTestBase, cls).SetUpProcess() 80 cls.CustomizeBrowserArgs([]) 81 cls.StartBrowser() 82 cls.SetStaticServerDirs(TEST_DATA_DIRS) 83 cls._skia_gold_temp_dir = tempfile.mkdtemp() 84 85 @classmethod 86 def GetSkiaGoldProperties(cls): 87 if not cls._skia_gold_properties: 88 cls._skia_gold_properties =\ 89 gpu_skia_gold_properties.GpuSkiaGoldProperties( 90 cls.GetParsedCommandLineOptions()) 91 return cls._skia_gold_properties 92 93 @classmethod 94 def GetSkiaGoldSessionManager(cls): 95 if not cls._skia_gold_session_manager: 96 cls._skia_gold_session_manager =\ 97 gpu_skia_gold_session_manager.GpuSkiaGoldSessionManager( 98 cls._skia_gold_temp_dir, cls.GetSkiaGoldProperties()) 99 return cls._skia_gold_session_manager 100 101 @classmethod 102 def GenerateBrowserArgs(cls, additional_args): 103 """Adds default arguments to |additional_args|. 104 105 See the parent class' method documentation for additional information. 106 """ 107 default_args = super(SkiaGoldIntegrationTestBase, 108 cls).GenerateBrowserArgs(additional_args) 109 default_args.extend([cba.ENABLE_GPU_BENCHMARKING, cba.TEST_TYPE_GPU]) 110 force_color_profile_arg = [ 111 arg for arg in default_args if arg.startswith('--force-color-profile=') 112 ] 113 if not force_color_profile_arg: 114 default_args.extend([ 115 cba.FORCE_COLOR_PROFILE_SRGB, 116 cba.ENSURE_FORCED_COLOR_PROFILE, 117 ]) 118 return default_args 119 120 @classmethod 121 def StopBrowser(cls): 122 super(SkiaGoldIntegrationTestBase, cls).StopBrowser() 123 cls.ResetGpuInfo() 124 125 @classmethod 126 def TearDownProcess(cls): 127 super(SkiaGoldIntegrationTestBase, cls).TearDownProcess() 128 shutil.rmtree(cls._skia_gold_temp_dir) 129 130 @classmethod 131 def AddCommandlineArgs(cls, parser): 132 super(SkiaGoldIntegrationTestBase, cls).AddCommandlineArgs(parser) 133 parser.add_option( 134 '--git-revision', help='Chrome revision being tested.', default=None) 135 parser.add_option( 136 '--test-machine-name', 137 help='Name of the test machine. Specifying this argument causes this ' 138 'script to upload failure images and diffs to cloud storage directly, ' 139 'instead of relying on the archive_gpu_pixel_test_results.py script.', 140 default='') 141 parser.add_option( 142 '--dont-restore-color-profile-after-test', 143 dest='dont_restore_color_profile_after_test', 144 action='store_true', 145 default=False, 146 help='(Mainly on Mac) don\'t restore the system\'s original color ' 147 'profile after the test completes; leave the system using the sRGB ' 148 'color profile. See http://crbug.com/784456.') 149 parser.add_option( 150 '--gerrit-issue', 151 help='For Skia Gold integration. Gerrit issue ID.', 152 default='') 153 parser.add_option( 154 '--gerrit-patchset', 155 help='For Skia Gold integration. Gerrit patch set number.', 156 default='') 157 parser.add_option( 158 '--buildbucket-id', 159 help='For Skia Gold integration. Buildbucket build ID.', 160 default='') 161 parser.add_option( 162 '--no-skia-gold-failure', 163 action='store_true', 164 default=False, 165 help='For Skia Gold integration. Always report that the test passed ' 166 'even if the Skia Gold image comparison reported a failure, but ' 167 'otherwise perform the same steps as usual.') 168 # Telemetry is *still* using optparse instead of argparse, so we can't have 169 # these two options in a mutually exclusive group. 170 parser.add_option( 171 '--local-pixel-tests', 172 action='store_true', 173 default=None, 174 help='Specifies to run the test harness in local run mode or not. When ' 175 'run in local mode, uploading to Gold is disabled and links to ' 176 'help with local debugging are output. Running in local mode also ' 177 'implies --no-luci-auth. If both this and --no-local-pixel-tests are ' 178 'left unset, the test harness will attempt to detect whether it is ' 179 'running on a workstation or not and set this option accordingly.') 180 parser.add_option( 181 '--no-local-pixel-tests', 182 action='store_false', 183 dest='local_pixel_tests', 184 help='Specifies to run the test harness in non-local (bot) mode. When ' 185 'run in this mode, data is actually uploaded to Gold and triage links ' 186 'arge generated. If both this and --local-pixel-tests are left unset, ' 187 'the test harness will attempt to detect whether it is running on a ' 188 'workstation or not and set this option accordingly.') 189 parser.add_option( 190 '--no-luci-auth', 191 action='store_true', 192 default=False, 193 help='Don\'t use the service account provided by LUCI for ' 194 'authentication for Skia Gold, instead relying on gsutil to be ' 195 'pre-authenticated. Meant for testing locally instead of on the bots.') 196 parser.add_option( 197 '--bypass-skia-gold-functionality', 198 action='store_true', 199 default=False, 200 help='Bypass all interaction with Skia Gold, effectively disabling the ' 201 'image comparison portion of any tests that use Gold. Only meant to ' 202 'be used in case a Gold outage occurs and cannot be fixed quickly.') 203 204 @classmethod 205 def ResetGpuInfo(cls): 206 cls._image_parameters = None 207 208 @classmethod 209 def GetImageParameters(cls, page): 210 if not cls._image_parameters: 211 cls._ComputeGpuInfo(page) 212 return cls._image_parameters 213 214 @classmethod 215 def _ComputeGpuInfo(cls, page): 216 if cls._image_parameters: 217 return 218 browser = cls.browser 219 system_info = browser.GetSystemInfo() 220 if not system_info: 221 raise Exception('System info must be supported by the browser') 222 if not system_info.gpu: 223 raise Exception('GPU information was absent') 224 device = system_info.gpu.devices[0] 225 cls._image_parameters = _ImageParameters() 226 params = cls._image_parameters 227 if device.vendor_id and device.device_id: 228 params.vendor_id = device.vendor_id 229 params.device_id = device.device_id 230 elif device.vendor_string and device.device_string: 231 params.vendor_string = device.vendor_string 232 params.device_string = device.device_string 233 elif page.gpu_process_disabled: 234 # Match the vendor and device IDs that the browser advertises 235 # when the software renderer is active. 236 params.vendor_id = 65535 237 params.device_id = 65535 238 else: 239 raise Exception('GPU device information was incomplete') 240 # TODO(senorblanco): This should probably be checking 241 # for the presence of the extensions in system_info.gpu_aux_attributes 242 # in order to check for MSAA, rather than sniffing the blocklist. 243 params.msaa = not (('disable_chromium_framebuffer_multisample' in 244 system_info.gpu.driver_bug_workarounds) or 245 ('disable_multisample_render_to_texture' in system_info. 246 gpu.driver_bug_workarounds)) 247 params.model_name = system_info.model_name 248 params.driver_version = device.driver_version 249 params.driver_vendor = device.driver_vendor 250 251 @classmethod 252 def _UploadBitmapToCloudStorage(cls, bucket, name, bitmap, public=False): 253 # This sequence of steps works on all platforms to write a temporary 254 # PNG to disk, following the pattern in bitmap_unittest.py. The key to 255 # avoiding PermissionErrors seems to be to not actually try to write to 256 # the temporary file object, but to re-open its name for all operations. 257 temp_file = tempfile.NamedTemporaryFile(suffix='.png').name 258 image_util.WritePngFile(bitmap, temp_file) 259 cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public) 260 261 # Not used consistently, but potentially useful for debugging issues on the 262 # bots, so kept around for future use. 263 @classmethod 264 def _UploadGoldErrorImageToCloudStorage(cls, image_name, screenshot): 265 revision = cls.GetSkiaGoldProperties().git_revision 266 machine_name = re.sub(r'\W+', '_', 267 cls.GetParsedCommandLineOptions().test_machine_name) 268 base_bucket = '%s/gold_failures' % (cls._error_image_cloud_storage_bucket) 269 image_name_with_revision_and_machine = '%s_%s_%s.png' % ( 270 image_name, machine_name, revision) 271 cls._UploadBitmapToCloudStorage( 272 base_bucket, 273 image_name_with_revision_and_machine, 274 screenshot, 275 public=True) 276 277 @staticmethod 278 def _UrlToImageName(url): 279 image_name = re.sub(r'^(http|https|file)://(/*)', '', url) 280 image_name = re.sub(r'\.\./', '', image_name) 281 image_name = re.sub(r'(\.|/|-)', '_', image_name) 282 return image_name 283 284 def GetGoldJsonKeys(self, page): 285 """Get all the JSON metadata that will be passed to golctl.""" 286 img_params = self.GetImageParameters(page) 287 # The frequently changing last part of the ANGLE driver version (revision of 288 # some sort?) messes a bit with inexact matching since each revision will 289 # be treated as a separate trace, so strip it off. 290 _StripAngleRevisionFromDriver(img_params) 291 # All values need to be strings, otherwise goldctl fails. 292 gpu_keys = { 293 'vendor_id': 294 _ToHexOrNone(img_params.vendor_id), 295 'device_id': 296 _ToHexOrNone(img_params.device_id), 297 'vendor_string': 298 _ToNonEmptyStrOrNone(img_params.vendor_string), 299 'device_string': 300 _ToNonEmptyStrOrNone(img_params.device_string), 301 'msaa': 302 str(img_params.msaa), 303 'model_name': 304 _ToNonEmptyStrOrNone(img_params.model_name), 305 'os': 306 _ToNonEmptyStrOrNone(self.browser.platform.GetOSName()), 307 'os_version': 308 _ToNonEmptyStrOrNone(self.browser.platform.GetOSVersionName()), 309 'os_version_detail_string': 310 _ToNonEmptyStrOrNone(self.browser.platform.GetOSVersionDetailString()), 311 'driver_version': 312 _ToNonEmptyStrOrNone(img_params.driver_version), 313 'driver_vendor': 314 _ToNonEmptyStrOrNone(img_params.driver_vendor), 315 'combined_hardware_identifier': 316 _GetCombinedHardwareIdentifier(img_params), 317 } 318 # If we have a grace period active, then the test is potentially flaky. 319 # Include a pair that will cause Gold to ignore any untriaged images, which 320 # will prevent it from automatically commenting on unrelated CLs that happen 321 # to produce a new image. 322 if _GracePeriodActive(page): 323 gpu_keys['ignore'] = '1' 324 return gpu_keys 325 326 def _UploadTestResultToSkiaGold(self, image_name, screenshot, page): 327 """Compares the given image using Skia Gold and uploads the result. 328 329 No uploading is done if the test is being run in local run mode. Compares 330 the given screenshot to baselines provided by Gold, raising an Exception if 331 a match is not found. 332 333 Args: 334 image_name: the name of the image being checked. 335 screenshot: the image being checked as a Telemetry Bitmap. 336 page: the GPU PixelTestPage object for the test. 337 """ 338 # Write screenshot to PNG file on local disk. 339 png_temp_file = tempfile.NamedTemporaryFile( 340 suffix='.png', dir=self._skia_gold_temp_dir).name 341 image_util.WritePngFile(screenshot, png_temp_file) 342 343 gpu_keys = self.GetGoldJsonKeys(page) 344 gold_session = self.GetSkiaGoldSessionManager().GetSkiaGoldSession( 345 gpu_keys, corpus=SKIA_GOLD_CORPUS) 346 gold_properties = self.GetSkiaGoldProperties() 347 use_luci = not (gold_properties.local_pixel_tests 348 or gold_properties.no_luci_auth) 349 350 inexact_matching_args = page.matching_algorithm.GetCmdline() 351 352 status, error = gold_session.RunComparison( 353 name=image_name, 354 png_file=png_temp_file, 355 inexact_matching_args=inexact_matching_args, 356 use_luci=use_luci) 357 if not status: 358 return 359 360 status_codes =\ 361 self.GetSkiaGoldSessionManager().GetSessionClass().StatusCodes 362 if status == status_codes.AUTH_FAILURE: 363 logging.error('Gold authentication failed with output %s', error) 364 elif status == status_codes.INIT_FAILURE: 365 logging.error('Gold initialization failed with output %s', error) 366 elif status == status_codes.COMPARISON_FAILURE_REMOTE: 367 # We currently don't have an internal instance + public mirror like the 368 # general Chrome Gold instance, so just report the "internal" link, which 369 # points to the correct instance. 370 _, triage_link = gold_session.GetTriageLinks(image_name) 371 if not triage_link: 372 logging.error('Failed to get triage link for %s, raw output: %s', 373 image_name, error) 374 logging.error('Reason for no triage link: %s', 375 gold_session.GetTriageLinkOmissionReason(image_name)) 376 elif gold_properties.IsTryjobRun(): 377 self.artifacts.CreateLink('triage_link_for_entire_cl', triage_link) 378 else: 379 self.artifacts.CreateLink('gold_triage_link', triage_link) 380 elif status == status_codes.COMPARISON_FAILURE_LOCAL: 381 logging.error('Local comparison failed. Local diff files:') 382 _OutputLocalDiffFiles(gold_session, image_name) 383 elif status == status_codes.LOCAL_DIFF_FAILURE: 384 logging.error( 385 'Local comparison failed and an error occurred during diff ' 386 'generation: %s', error) 387 # There might be some files, so try outputting them. 388 logging.error('Local diff files:') 389 _OutputLocalDiffFiles(gold_session, image_name) 390 else: 391 logging.error( 392 'Given unhandled SkiaGoldSession StatusCode %s with error %s', status, 393 error) 394 if self._ShouldReportGoldFailure(page): 395 raise Exception('goldctl command failed, see above for details') 396 397 def _ShouldReportGoldFailure(self, page): 398 """Determines if a Gold failure should actually be surfaced. 399 400 Args: 401 page: The GPU PixelTestPage object for the test. 402 403 Returns: 404 True if the failure should be surfaced, i.e. the test should fail, 405 otherwise False. 406 """ 407 parsed_options = self.GetParsedCommandLineOptions() 408 # Don't surface if we're explicitly told not to. 409 if parsed_options.no_skia_gold_failure: 410 return False 411 # Don't surface if the test was recently added and we're still within its 412 # grace period. 413 if _GracePeriodActive(page): 414 return False 415 return True 416 417 @classmethod 418 def GenerateGpuTests(cls, options): 419 del options 420 return [] 421 422 def RunActualGpuTest(self, options): 423 raise NotImplementedError( 424 'RunActualGpuTest must be overridden in a subclass') 425 426 427def _ToHex(num): 428 return hex(int(num)) 429 430 431def _ToHexOrNone(num): 432 return 'None' if num == None else _ToHex(num) 433 434 435def _ToNonEmptyStrOrNone(val): 436 return 'None' if val == '' else str(val) 437 438 439def _GracePeriodActive(page): 440 """Returns whether a grace period is currently active for a test. 441 442 Args: 443 page: The GPU PixelTestPage object for the test in question. 444 445 Returns: 446 True if a grace period is defined for |page| and has not yet expired. 447 Otherwise, False. 448 """ 449 return page.grace_period_end and date.today() <= page.grace_period_end 450 451 452def _StripAngleRevisionFromDriver(img_params): 453 """Strips the revision off the end of an ANGLE driver version. 454 455 E.g. 2.1.0.b50541b2d6c4 -> 2.1.0 456 457 Modifies the string in place. No-ops if the driver vendor is not ANGLE. 458 459 Args: 460 img_params: An _ImageParameters instance to modify. 461 """ 462 if 'ANGLE' not in img_params.driver_vendor or not img_params.driver_version: 463 return 464 # Assume that we're never going to have portions of the driver we care about 465 # that are longer than 8 characters. 466 driver_parts = img_params.driver_version.split('.') 467 kept_parts = [] 468 for part in driver_parts: 469 if len(part) > 8: 470 break 471 kept_parts.append(part) 472 img_params.driver_version = '.'.join(kept_parts) 473 474 475def _GetCombinedHardwareIdentifier(img_params): 476 """Combine all relevant hardware identifiers into a single key. 477 478 This makes Gold forwarding more precise by allowing us to forward explicit 479 configurations instead of individual components. 480 """ 481 vendor_id = _ToHexOrNone(img_params.vendor_id) 482 device_id = _ToHexOrNone(img_params.device_id) 483 device_string = _ToNonEmptyStrOrNone(img_params.device_string) 484 combined_hw_identifiers = ('vendor_id:{vendor_id}, ' 485 'device_id:{device_id}, ' 486 'device_string:{device_string}') 487 combined_hw_identifiers = combined_hw_identifiers.format( 488 vendor_id=vendor_id, device_id=device_id, device_string=device_string) 489 return combined_hw_identifiers 490 491 492def _OutputLocalDiffFiles(gold_session, image_name): 493 """Logs the local diff image files from the given SkiaGoldSession. 494 495 Args: 496 gold_session: A skia_gold_session.SkiaGoldSession instance to pull files 497 from. 498 image_name: A string containing the name of the image/test that was 499 compared. 500 """ 501 given_file = gold_session.GetGivenImageLink(image_name) 502 closest_file = gold_session.GetClosestImageLink(image_name) 503 diff_file = gold_session.GetDiffImageLink(image_name) 504 failure_message = 'Unable to retrieve link' 505 logging.error('Generated image: %s', given_file or failure_message) 506 logging.error('Closest image: %s', closest_file or failure_message) 507 logging.error('Diff image: %s', diff_file or failure_message) 508 509 510def load_tests(loader, tests, pattern): 511 del loader, tests, pattern # Unused. 512 return gpu_integration_test.LoadAllTestsInModule(sys.modules[__name__]) 513