1#!/usr/bin/env vpython3 2# 3# Copyright 2020 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6"""This script facilitates running tests for lacros on Linux. 7 8 In order to run lacros tests on Linux, please first follow bit.ly/3juQVNJ 9 to setup build directory with the lacros-chrome-on-linux build configuration, 10 and corresponding test targets are built successfully. 11 12 * Example usages: 13 14 ./build/lacros/test_runner.py test out/lacros/url_unittests 15 ./build/lacros/test_runner.py test out/lacros/browser_tests 16 17 The commands above run url_unittests and browser_tests respecitively, and more 18 specifically, url_unitests is executed directly while browser_tests is 19 executed with the latest version of prebuilt ash-chrome, and the behavior is 20 controlled by |_TARGETS_REQUIRE_ASH_CHROME|, and it's worth noting that the 21 list is maintained manually, so if you see something is wrong, please upload a 22 CL to fix it. 23 24 ./build/lacros/test_runner.py test out/lacros/browser_tests \\ 25 --gtest_filter=BrowserTest.Title 26 27 The above command only runs 'BrowserTest.Title', and any argument accepted by 28 the underlying test binary can be specified in the command. 29 30 ./build/lacros/test_runner.py test out/lacros/browser_tests \\ 31 --ash-chrome-version=793554 32 33 The above command runs tests with a given version of ash-chrome, which is 34 useful to reproduce test failures, the version corresponds to the commit 35 position of commits on the master branch, and a list of prebuilt versions can 36 be found at: gs://ash-chromium-on-linux-prebuilts/x86_64. 37 38 ./testing/xvfb.py ./build/lacros/test_runner.py test out/lacros/browser_tests 39 40 The above command starts ash-chrome with xvfb instead of an X11 window, and 41 it's useful when running tests without a display attached, such as sshing. 42 43 For version skew testing when passing --ash-chrome-path-override, the runner 44 will try to find the ash major version and Lacros major version. If ash is 45 newer(major version larger), the runner will not run any tests and just 46 returns success. 47""" 48 49import argparse 50import json 51import os 52import logging 53import re 54import shutil 55import signal 56import subprocess 57import sys 58import tempfile 59import time 60import zipfile 61 62_SRC_ROOT = os.path.abspath( 63 os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) 64sys.path.append(os.path.join(_SRC_ROOT, 'third_party', 'depot_tools')) 65 66# Base GS URL to store prebuilt ash-chrome. 67_GS_URL_BASE = 'gs://ash-chromium-on-linux-prebuilts/x86_64' 68 69# Latest file version. 70_GS_URL_LATEST_FILE = _GS_URL_BASE + '/latest/ash-chromium.txt' 71 72# GS path to the zipped ash-chrome build with any given version. 73_GS_ASH_CHROME_PATH = 'ash-chromium.zip' 74 75# Directory to cache downloaded ash-chrome versions to avoid re-downloading. 76_PREBUILT_ASH_CHROME_DIR = os.path.join(os.path.dirname(__file__), 77 'prebuilt_ash_chrome') 78 79# Number of seconds to wait for ash-chrome to start. 80ASH_CHROME_TIMEOUT_SECONDS = ( 81 300 if os.environ.get('ASH_WRAPPER', None) else 10) 82 83# List of targets that require ash-chrome as a Wayland server in order to run. 84_TARGETS_REQUIRE_ASH_CHROME = [ 85 'app_shell_unittests', 86 'aura_unittests', 87 'browser_tests', 88 'components_unittests', 89 'compositor_unittests', 90 'content_unittests', 91 'dbus_unittests', 92 'extensions_unittests', 93 'media_unittests', 94 'message_center_unittests', 95 'snapshot_unittests', 96 'sync_integration_tests', 97 'unit_tests', 98 'views_unittests', 99 'wm_unittests', 100 101 # regex patterns. 102 '.*_browsertests', 103 '.*interactive_ui_tests' 104] 105 106# List of targets that require ash-chrome to support crosapi mojo APIs. 107_TARGETS_REQUIRE_MOJO_CROSAPI = [ 108 # TODO(jamescook): Add 'browser_tests' after multiple crosapi connections 109 # are allowed. For now we only enable crosapi in targets that run tests 110 # serially. 111 'interactive_ui_tests', 112 'lacros_chrome_browsertests', 113 'lacros_chrome_browsertests_run_in_series' 114] 115 116 117def _GetAshChromeDirPath(version): 118 """Returns a path to the dir storing the downloaded version of ash-chrome.""" 119 return os.path.join(_PREBUILT_ASH_CHROME_DIR, version) 120 121 122def _remove_unused_ash_chrome_versions(version_to_skip): 123 """Removes unused ash-chrome versions to save disk space. 124 125 Currently, when an ash-chrome zip is downloaded and unpacked, the atime/mtime 126 of the dir and the files are NOW instead of the time when they were built, but 127 there is no garanteen it will always be the behavior in the future, so avoid 128 removing the current version just in case. 129 130 Args: 131 version_to_skip (str): the version to skip removing regardless of its age. 132 """ 133 days = 7 134 expiration_duration = 60 * 60 * 24 * days 135 136 for f in os.listdir(_PREBUILT_ASH_CHROME_DIR): 137 if f == version_to_skip: 138 continue 139 140 p = os.path.join(_PREBUILT_ASH_CHROME_DIR, f) 141 if os.path.isfile(p): 142 # The prebuilt ash-chrome dir is NOT supposed to contain any files, remove 143 # them to keep the directory clean. 144 os.remove(p) 145 continue 146 chrome_path = os.path.join(p, 'test_ash_chrome') 147 if not os.path.exists(chrome_path): 148 chrome_path = p 149 age = time.time() - os.path.getatime(chrome_path) 150 if age > expiration_duration: 151 logging.info( 152 'Removing ash-chrome: "%s" as it hasn\'t been used in the ' 153 'past %d days', p, days) 154 shutil.rmtree(p) 155 156def _GsutilCopyWithRetry(gs_path, local_name, retry_times=3): 157 """Gsutil copy with retry. 158 159 Args: 160 gs_path: The gs path for remote location. 161 local_name: The local file name. 162 retry_times: The total try times if the gsutil call fails. 163 164 Raises: 165 RuntimeError: If failed to download the specified version, for example, 166 if the version is not present on gcs. 167 """ 168 import download_from_google_storage 169 gsutil = download_from_google_storage.Gsutil( 170 download_from_google_storage.GSUTIL_DEFAULT_PATH) 171 exit_code = 1 172 retry = 0 173 while exit_code and retry < retry_times: 174 retry += 1 175 exit_code = gsutil.call('cp', gs_path, local_name) 176 if exit_code: 177 raise RuntimeError('Failed to download: "%s"' % gs_path) 178 179 180def _DownloadAshChromeIfNecessary(version): 181 """Download a given version of ash-chrome if not already exists. 182 183 Args: 184 version: A string representing the version, such as "793554". 185 186 Raises: 187 RuntimeError: If failed to download the specified version, for example, 188 if the version is not present on gcs. 189 """ 190 191 def IsAshChromeDirValid(ash_chrome_dir): 192 # This function assumes that once 'chrome' is present, other dependencies 193 # will be present as well, it's not always true, for example, if the test 194 # runner process gets killed in the middle of unzipping (~2 seconds), but 195 # it's unlikely for the assumption to break in practice. 196 return os.path.isdir(ash_chrome_dir) and os.path.isfile( 197 os.path.join(ash_chrome_dir, 'test_ash_chrome')) 198 199 ash_chrome_dir = _GetAshChromeDirPath(version) 200 if IsAshChromeDirValid(ash_chrome_dir): 201 return 202 203 shutil.rmtree(ash_chrome_dir, ignore_errors=True) 204 os.makedirs(ash_chrome_dir) 205 with tempfile.NamedTemporaryFile() as tmp: 206 logging.info('Ash-chrome version: %s', version) 207 gs_path = _GS_URL_BASE + '/' + version + '/' + _GS_ASH_CHROME_PATH 208 _GsutilCopyWithRetry(gs_path, tmp.name) 209 210 # https://bugs.python.org/issue15795. ZipFile doesn't preserve permissions. 211 # And in order to workaround the issue, this function is created and used 212 # instead of ZipFile.extractall(). 213 # The solution is copied from: 214 # https://stackoverflow.com/questions/42326428/zipfile-in-python-file-permission 215 def ExtractFile(zf, info, extract_dir): 216 zf.extract(info.filename, path=extract_dir) 217 perm = info.external_attr >> 16 218 os.chmod(os.path.join(extract_dir, info.filename), perm) 219 220 with zipfile.ZipFile(tmp.name, 'r') as zf: 221 # Extra all files instead of just 'chrome' binary because 'chrome' needs 222 # other resources and libraries to run. 223 for info in zf.infolist(): 224 ExtractFile(zf, info, ash_chrome_dir) 225 226 _remove_unused_ash_chrome_versions(version) 227 228 229def _GetLatestVersionOfAshChrome(): 230 """Returns the latest version of uploaded ash-chrome.""" 231 with tempfile.NamedTemporaryFile() as tmp: 232 _GsutilCopyWithRetry(_GS_URL_LATEST_FILE, tmp.name) 233 with open(tmp.name, 'r') as f: 234 return f.read().strip() 235 236 237def _WaitForAshChromeToStart(tmp_xdg_dir, lacros_mojo_socket_file, 238 enable_mojo_crosapi): 239 """Waits for Ash-Chrome to be up and running and returns a boolean indicator. 240 241 Determine whether ash-chrome is up and running by checking whether two files 242 (lock file + socket) have been created in the |XDG_RUNTIME_DIR| and the lacros 243 mojo socket file has been created if enabling the mojo "crosapi" interface. 244 TODO(crbug.com/1107966): Figure out a more reliable hook to determine the 245 status of ash-chrome, likely through mojo connection. 246 247 Args: 248 tmp_xdg_dir (str): Path to the XDG_RUNTIME_DIR. 249 lacros_mojo_socket_file (str): Path to the lacros mojo socket file. 250 enable_mojo_crosapi (bool): Whether to bootstrap the crosapi mojo interface 251 between ash and the lacros test binary. 252 253 Returns: 254 A boolean indicating whether Ash-chrome is up and running. 255 """ 256 257 def IsAshChromeReady(tmp_xdg_dir, lacros_mojo_socket_file, 258 enable_mojo_crosapi): 259 return (len(os.listdir(tmp_xdg_dir)) >= 2 260 and (not enable_mojo_crosapi 261 or os.path.exists(lacros_mojo_socket_file))) 262 263 time_counter = 0 264 while not IsAshChromeReady(tmp_xdg_dir, lacros_mojo_socket_file, 265 enable_mojo_crosapi): 266 time.sleep(0.5) 267 time_counter += 0.5 268 if time_counter > ASH_CHROME_TIMEOUT_SECONDS: 269 break 270 271 return IsAshChromeReady(tmp_xdg_dir, lacros_mojo_socket_file, 272 enable_mojo_crosapi) 273 274 275def _ExtractAshMajorVersion(file_path): 276 """Extract major version from file_path. 277 278 File path like this: 279 ../../lacros_version_skew_tests_v94.0.4588.0/test_ash_chrome 280 281 Returns: 282 int representing the major version. Or 0 if it can't extract 283 major version. 284 """ 285 m = re.search( 286 'lacros_version_skew_tests_v(?P<version>[0-9]+).[0-9]+.[0-9]+.[0-9]+/', 287 file_path) 288 if (m and 'version' in m.groupdict().keys()): 289 return int(m.group('version')) 290 logging.warning('Can not find the ash version in %s.' % file_path) 291 # Returns ash major version as 0, so we can still run tests. 292 # This is likely happen because user is running in local environments. 293 return 0 294 295 296def _FindLacrosMajorVersionFromMetadata(): 297 # This handles the logic on bots. When running on bots, 298 # we don't copy source files to test machines. So we build a 299 # metadata.json file which contains version information. 300 if not os.path.exists('metadata.json'): 301 logging.error('Can not determine current version.') 302 # Returns 0 so it can't run any tests. 303 return 0 304 version = '' 305 with open('metadata.json', 'r') as file: 306 content = json.load(file) 307 version = content['content']['version'] 308 return int(version[:version.find('.')]) 309 310 311def _FindLacrosMajorVersion(): 312 """Returns the major version in the current checkout. 313 314 It would try to read src/chrome/VERSION. If it's not available, 315 then try to read metadata.json. 316 317 Returns: 318 int representing the major version. Or 0 if it fails to 319 determine the version. 320 """ 321 version_file = os.path.abspath( 322 os.path.join(os.path.abspath(os.path.dirname(__file__)), 323 '../../chrome/VERSION')) 324 # This is mostly happens for local development where 325 # src/chrome/VERSION exists. 326 if os.path.exists(version_file): 327 lines = open(version_file, 'r').readlines() 328 return int(lines[0][lines[0].find('=') + 1:-1]) 329 return _FindLacrosMajorVersionFromMetadata() 330 331 332def _ParseSummaryOutput(forward_args): 333 """Find the summary output file path. 334 335 Args: 336 forward_args (list): Args to be forwarded to the test command. 337 338 Returns: 339 None if not found, or str representing the output file path. 340 """ 341 logging.warning(forward_args) 342 for arg in forward_args: 343 if arg.startswith('--test-launcher-summary-output='): 344 return arg[len('--test-launcher-summary-output='):] 345 return None 346 347 348def _RunTestWithAshChrome(args, forward_args): 349 """Runs tests with ash-chrome. 350 351 Args: 352 args (dict): Args for this script. 353 forward_args (list): Args to be forwarded to the test command. 354 """ 355 if args.ash_chrome_path_override: 356 ash_chrome_file = args.ash_chrome_path_override 357 ash_major_version = _ExtractAshMajorVersion(ash_chrome_file) 358 lacros_major_version = _FindLacrosMajorVersion() 359 if ash_major_version > lacros_major_version: 360 logging.warning('''Not running any tests, because we do not \ 361support version skew testing for Lacros M%s against ash M%s''' % 362 (lacros_major_version, ash_major_version)) 363 # Create an empty output.json file so result adapter can read 364 # the file. Or else result adapter will report no file found 365 # and result infra failure. 366 output_json = _ParseSummaryOutput(forward_args) 367 if output_json: 368 with open(output_json, 'w') as f: 369 f.write("""{"all_tests":[],"disabled_tests":[],"global_tags":[], 370"per_iteration_data":[],"test_locations":{}}""") 371 # Although we don't run any tests, this is considered as success. 372 return 0 373 if not os.path.exists(ash_chrome_file): 374 logging.error("""Can not find ash chrome at %s. Did you download \ 375the ash from CIPD? If you don't plan to build your own ash, you need \ 376to download first. Example commandlines: 377 $ cipd auth-login 378 $ echo "chromium/testing/linux-ash-chromium/x86_64/ash.zip \ 379version:92.0.4515.130" > /tmp/ensure-file.txt 380 $ cipd ensure -ensure-file /tmp/ensure-file.txt \ 381-root lacros_version_skew_tests_v92.0.4515.130 382 Then you can use --ash-chrome-path-override=\ 383lacros_version_skew_tests_v92.0.4515.130/test_ash_chrome 384""" % ash_chrome_file) 385 return 1 386 elif args.ash_chrome_path: 387 ash_chrome_file = args.ash_chrome_path 388 else: 389 ash_chrome_version = (args.ash_chrome_version 390 or _GetLatestVersionOfAshChrome()) 391 _DownloadAshChromeIfNecessary(ash_chrome_version) 392 logging.info('Ash-chrome version: %s', ash_chrome_version) 393 394 ash_chrome_file = os.path.join(_GetAshChromeDirPath(ash_chrome_version), 395 'test_ash_chrome') 396 try: 397 # Starts Ash-Chrome. 398 tmp_xdg_dir_name = tempfile.mkdtemp() 399 tmp_ash_data_dir_name = tempfile.mkdtemp() 400 401 # Please refer to below file for how mojo connection is set up in testing. 402 # //chrome/browser/ash/crosapi/test_mojo_connection_manager.h 403 lacros_mojo_socket_file = '%s/lacros.sock' % tmp_ash_data_dir_name 404 lacros_mojo_socket_arg = ('--lacros-mojo-socket-for-testing=%s' % 405 lacros_mojo_socket_file) 406 enable_mojo_crosapi = any(t == os.path.basename(args.command) 407 for t in _TARGETS_REQUIRE_MOJO_CROSAPI) 408 409 ash_process = None 410 ash_env = os.environ.copy() 411 ash_env['XDG_RUNTIME_DIR'] = tmp_xdg_dir_name 412 ash_cmd = [ 413 ash_chrome_file, 414 '--user-data-dir=%s' % tmp_ash_data_dir_name, 415 '--enable-wayland-server', 416 '--no-startup-window', 417 ] 418 if enable_mojo_crosapi: 419 ash_cmd.append(lacros_mojo_socket_arg) 420 421 # Users can specify a wrapper for the ash binary to do things like 422 # attaching debuggers. For example, this will open a new terminal window 423 # and run GDB. 424 # $ export ASH_WRAPPER="gnome-terminal -- gdb --ex=r --args" 425 ash_wrapper = os.environ.get('ASH_WRAPPER', None) 426 if ash_wrapper: 427 logging.info('Running ash with "ASH_WRAPPER": %s', ash_wrapper) 428 ash_cmd = list(ash_wrapper.split()) + ash_cmd 429 430 ash_process_has_started = False 431 total_tries = 3 432 num_tries = 0 433 while not ash_process_has_started and num_tries < total_tries: 434 num_tries += 1 435 ash_process = subprocess.Popen(ash_cmd, env=ash_env) 436 ash_process_has_started = _WaitForAshChromeToStart( 437 tmp_xdg_dir_name, lacros_mojo_socket_file, enable_mojo_crosapi) 438 if ash_process_has_started: 439 break 440 441 logging.warning('Starting ash-chrome timed out after %ds', 442 ASH_CHROME_TIMEOUT_SECONDS) 443 logging.warning('Printing the output of "ps aux" for debugging:') 444 subprocess.call(['ps', 'aux']) 445 if ash_process and ash_process.poll() is None: 446 ash_process.kill() 447 448 if not ash_process_has_started: 449 raise RuntimeError('Timed out waiting for ash-chrome to start') 450 451 # Starts tests. 452 if enable_mojo_crosapi: 453 forward_args.append(lacros_mojo_socket_arg) 454 455 test_env = os.environ.copy() 456 test_env['EGL_PLATFORM'] = 'surfaceless' 457 test_env['XDG_RUNTIME_DIR'] = tmp_xdg_dir_name 458 test_process = subprocess.Popen([args.command] + forward_args, env=test_env) 459 return test_process.wait() 460 461 finally: 462 if ash_process and ash_process.poll() is None: 463 ash_process.terminate() 464 # Allow process to do cleanup and exit gracefully before killing. 465 time.sleep(0.5) 466 ash_process.kill() 467 468 shutil.rmtree(tmp_xdg_dir_name, ignore_errors=True) 469 shutil.rmtree(tmp_ash_data_dir_name, ignore_errors=True) 470 471 472def _RunTestDirectly(args, forward_args): 473 """Runs tests by invoking the test command directly. 474 475 args (dict): Args for this script. 476 forward_args (list): Args to be forwarded to the test command. 477 """ 478 try: 479 p = None 480 p = subprocess.Popen([args.command] + forward_args) 481 return p.wait() 482 finally: 483 if p and p.poll() is None: 484 p.terminate() 485 time.sleep(0.5) 486 p.kill() 487 488 489def _HandleSignal(sig, _): 490 """Handles received signals to make sure spawned test process are killed. 491 492 sig (int): An integer representing the received signal, for example SIGTERM. 493 """ 494 logging.warning('Received signal: %d, killing spawned processes', sig) 495 496 # Don't do any cleanup here, instead, leave it to the finally blocks. 497 # Assumption is based on https://docs.python.org/3/library/sys.html#sys.exit: 498 # cleanup actions specified by finally clauses of try statements are honored. 499 500 # https://tldp.org/LDP/abs/html/exitcodes.html: 501 # Exit code 128+n -> Fatal error signal "n". 502 sys.exit(128 + sig) 503 504 505def _RunTest(args, forward_args): 506 """Runs tests with given args. 507 508 args (dict): Args for this script. 509 forward_args (list): Args to be forwarded to the test command. 510 511 Raises: 512 RuntimeError: If the given test binary doesn't exist or the test runner 513 doesn't know how to run it. 514 """ 515 516 if not os.path.isfile(args.command): 517 raise RuntimeError('Specified test command: "%s" doesn\'t exist' % 518 args.command) 519 520 # |_TARGETS_REQUIRE_ASH_CHROME| may not always be accurate as it is updated 521 # with a best effort only, therefore, allow the invoker to override the 522 # behavior with a specified ash-chrome version, which makes sure that 523 # automated CI/CQ builders would always work correctly. 524 requires_ash_chrome = any( 525 re.match(t, os.path.basename(args.command)) 526 for t in _TARGETS_REQUIRE_ASH_CHROME) 527 if not requires_ash_chrome and not args.ash_chrome_version: 528 return _RunTestDirectly(args, forward_args) 529 530 return _RunTestWithAshChrome(args, forward_args) 531 532 533def Main(): 534 for sig in (signal.SIGTERM, signal.SIGINT): 535 signal.signal(sig, _HandleSignal) 536 537 logging.basicConfig(level=logging.INFO) 538 arg_parser = argparse.ArgumentParser() 539 arg_parser.usage = __doc__ 540 541 subparsers = arg_parser.add_subparsers() 542 543 test_parser = subparsers.add_parser('test', help='Run tests') 544 test_parser.set_defaults(func=_RunTest) 545 546 test_parser.add_argument( 547 'command', 548 help='A single command to invoke the tests, for example: ' 549 '"./url_unittests". Any argument unknown to this test runner script will ' 550 'be forwarded to the command, for example: "--gtest_filter=Suite.Test"') 551 552 version_group = test_parser.add_mutually_exclusive_group() 553 version_group.add_argument( 554 '--ash-chrome-version', 555 type=str, 556 help='Version of an prebuilt ash-chrome to use for testing, for example: ' 557 '"793554", and the version corresponds to the commit position of commits ' 558 'on the main branch. If not specified, will use the latest version ' 559 'available') 560 version_group.add_argument( 561 '--ash-chrome-path', 562 type=str, 563 help='Path to an locally built ash-chrome to use for testing. ' 564 'In general you should build //chrome/test:test_ash_chrome.') 565 566 # This is for version skew testing. The current CI/CQ builder builds 567 # an ash chrome and pass it using --ash-chrome-path. In order to use the same 568 # builder for version skew testing, we use a new argument to override 569 # the ash chrome. 570 test_parser.add_argument( 571 '--ash-chrome-path-override', 572 type=str, 573 help='The same as --ash-chrome-path. But this will override ' 574 '--ash-chrome-path or --ash-chrome-version if any of these ' 575 'arguments exist.') 576 args = arg_parser.parse_known_args() 577 return args[0].func(args[0], args[1]) 578 579 580if __name__ == '__main__': 581 sys.exit(Main()) 582