1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import contextlib 6import logging 7import os 8 9import py_utils 10from py_utils import binary_manager 11from py_utils import cloud_storage 12from py_utils import dependency_util 13import dependency_manager 14from dependency_manager import base_config 15 16from devil import devil_env 17 18 19from telemetry.core import exceptions 20from telemetry.core import util 21 22 23TELEMETRY_PROJECT_CONFIG = os.path.join( 24 util.GetTelemetryDir(), 'telemetry', 'binary_dependencies.json') 25 26 27CHROME_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'common', 'py_utils', 28 'py_utils', 'chrome_binaries.json') 29 30 31SUPPORTED_DEP_PLATFORMS = ( 32 'linux_aarch64', 'linux_x86_64', 'linux_armv7l', 'linux_mips', 33 'mac_x86_64', 34 'win_x86', 'win_AMD64', 35 'android_arm64-v8a', 'android_armeabi-v7a', 'android_arm', 'android_x64', 36 'android_x86' 37) 38 39PLATFORMS_TO_DOWNLOAD_FOLDER_MAP = { 40 'linux_aarch64': 'bin/linux/aarch64', 41 'linux_x86_64': 'bin/linux/x86_64', 42 'linux_armv7l': 'bin/linux/armv7l', 43 'linux_mips': 'bin/linux/mips', 44 'mac_x86_64': 'bin/mac/x86_64', 45 'win_x86': 'bin/win/x86', 46 'win_AMD64': 'bin/win/AMD64', 47 'android_arm64-v8a': 'bin/android/arm64-v8a', 48 'android_armeabi-v7a': 'bin/android/armeabi-v7a', 49 'android_arm': 'bin/android/arm', 50 'android_x64': 'bin/android/x64', 51 'android_x86': 'bin/android/x86', 52} 53 54NoPathFoundError = dependency_manager.NoPathFoundError 55CloudStorageError = dependency_manager.CloudStorageError 56 57 58_binary_manager = None 59_installed_helpers = set() 60 61 62TELEMETRY_BINARY_BASE_CS_FOLDER = 'binary_dependencies' 63TELEMETRY_BINARY_CS_BUCKET = cloud_storage.PUBLIC_BUCKET 64 65def NeedsInit(): 66 return not _binary_manager 67 68 69def InitDependencyManager(client_configs): 70 if GetBinaryManager(): 71 raise exceptions.InitializationError( 72 'Trying to re-initialize the binary manager with config %s' 73 % client_configs) 74 configs = [] 75 if client_configs: 76 configs += client_configs 77 configs += [TELEMETRY_PROJECT_CONFIG, CHROME_BINARY_CONFIG] 78 SetBinaryManager(binary_manager.BinaryManager(configs)) 79 80 devil_env.config.Initialize() 81 82 83@contextlib.contextmanager 84def TemporarilyReplaceBinaryManager(manager): 85 old_manager = GetBinaryManager() 86 try: 87 SetBinaryManager(manager) 88 yield 89 finally: 90 SetBinaryManager(old_manager) 91 92 93def GetBinaryManager(): 94 return _binary_manager 95 96 97def SetBinaryManager(manager): 98 global _binary_manager # pylint: disable=global-statement 99 _binary_manager = manager 100 101 102def _IsChromeOSLocalMode(os_name): 103 """Determines if we're running telemetry on a Chrome OS device. 104 105 Used to differentiate local mode (telemetry running on the CrOS DUT) from 106 remote mode (running telemetry on another platform that communicates with 107 the CrOS DUT over SSH). 108 """ 109 return os_name == 'chromeos' and py_utils.GetHostOsName() == 'chromeos' 110 111 112def FetchPath(binary_name, os_name, arch, os_version=None): 113 """ Return a path to the appropriate executable for <binary_name>, downloading 114 from cloud storage if needed, or None if it cannot be found. 115 """ 116 if GetBinaryManager() is None: 117 raise exceptions.InitializationError( 118 'Called FetchPath with uninitialized binary manager.') 119 return GetBinaryManager().FetchPath( 120 binary_name, 'linux' if _IsChromeOSLocalMode(os_name) else os_name, 121 arch, os_version) 122 123 124def LocalPath(binary_name, os_name, arch, os_version=None): 125 """ Return a local path to the given binary name, or None if an executable 126 cannot be found. Will not download the executable. 127 """ 128 if GetBinaryManager() is None: 129 raise exceptions.InitializationError( 130 'Called LocalPath with uninitialized binary manager.') 131 return GetBinaryManager().LocalPath(binary_name, os_name, arch, os_version) 132 133 134def FetchBinaryDependencies( 135 platform, client_configs, fetch_reference_chrome_binary): 136 """ Fetch all binary dependenencies for the given |platform|. 137 138 Note: we don't fetch browser binaries by default because the size of the 139 binary is about 2Gb, and it requires cloud storage permission to 140 chrome-telemetry bucket. 141 142 Args: 143 platform: an instance of telemetry.core.platform 144 client_configs: A list of paths (string) to dependencies json files. 145 fetch_reference_chrome_binary: whether to fetch reference chrome binary for 146 the given platform. 147 """ 148 configs = [ 149 dependency_manager.BaseConfig(TELEMETRY_PROJECT_CONFIG), 150 ] 151 dep_manager = dependency_manager.DependencyManager(configs) 152 os_name = platform.GetOSName() 153 # If we're running directly on a Chrome OS device, fetch the binaries for 154 # linux instead, which should be compatible with CrOS. Otherwise, if we're 155 # running remotely on CrOS, fetch the binaries for the host platform like 156 # we do with android below. 157 if _IsChromeOSLocalMode(os_name): 158 os_name = 'linux' 159 target_platform = '%s_%s' % (os_name, platform.GetArchName()) 160 dep_manager.PrefetchPaths(target_platform) 161 162 host_platform = None 163 fetch_devil_deps = False 164 if os_name in ('android', 'chromeos'): 165 host_platform = '%s_%s' % ( 166 py_utils.GetHostOsName(), py_utils.GetHostArchName()) 167 dep_manager.PrefetchPaths(host_platform) 168 if os_name == 'android': 169 if host_platform == 'linux_x86_64': 170 fetch_devil_deps = True 171 else: 172 logging.error('Devil only supports 64 bit linux as a host platform. ' 173 'Android tests may fail.') 174 175 if fetch_reference_chrome_binary: 176 _FetchReferenceBrowserBinary(platform) 177 178 # For now, handle client config separately because the BUILD.gn & .isolate of 179 # telemetry tests in chromium src failed to include the files specified in its 180 # client config. 181 # (https://github.com/catapult-project/catapult/issues/2192) 182 # For now this is ok because the client configs usually don't include cloud 183 # storage infos. 184 # TODO(crbug.com/1111556): remove the logic of swallowing exception once the 185 # issue is fixed on Chromium side. 186 if client_configs: 187 manager = dependency_manager.DependencyManager( 188 list(dependency_manager.BaseConfig(c) for c in client_configs)) 189 try: 190 manager.PrefetchPaths(target_platform) 191 if host_platform is not None: 192 manager.PrefetchPaths(host_platform) 193 194 except dependency_manager.NoPathFoundError as e: 195 logging.error('Error when trying to prefetch paths for %s: %s', 196 target_platform, e.message) 197 198 if fetch_devil_deps: 199 devil_env.config.Initialize() 200 devil_env.config.PrefetchPaths(arch=platform.GetArchName()) 201 devil_env.config.PrefetchPaths() 202 203 204def ReinstallAndroidHelperIfNeeded(binary_name, install_path, device): 205 """ Install a binary helper to a specific location. 206 207 Args: 208 binary_name: (str) The name of the binary from binary_dependencies.json 209 install_path: (str) The path to install the binary at 210 device: (device_utils.DeviceUtils) a device to install the helper to 211 Raises: 212 Exception: When the binary could not be fetched or could not be pushed to 213 the device. 214 """ 215 if (device.serial, install_path) in _installed_helpers: 216 return 217 host_path = FetchPath(binary_name, 'android', device.GetABI()) 218 if not host_path: 219 raise Exception( 220 '%s binary could not be fetched as %s', binary_name, host_path) 221 device.PushChangedFiles([(host_path, install_path)]) 222 device.RunShellCommand(['chmod', '777', install_path], check_return=True) 223 _installed_helpers.add((device.serial, install_path)) 224 225 226def _FetchReferenceBrowserBinary(platform): 227 os_name = platform.GetOSName() 228 if _IsChromeOSLocalMode(os_name): 229 os_name = 'linux' 230 arch_name = platform.GetArchName() 231 manager = binary_manager.BinaryManager( 232 [CHROME_BINARY_CONFIG]) 233 if os_name == 'android': 234 os_version = dependency_util.GetChromeApkOsVersion( 235 platform.GetOSVersionName()) 236 manager.FetchPath( 237 'chrome_stable', os_name, arch_name, os_version) 238 else: 239 manager.FetchPath( 240 'chrome_stable', os_name, arch_name) 241 242 243def UpdateDependency(dependency, dep_local_path, version, 244 os_name=None, arch_name=None): 245 config = os.path.join( 246 util.GetTelemetryDir(), 'telemetry', 'binary_dependencies.json') 247 248 if not os_name: 249 assert not arch_name, 'arch_name is specified but not os_name' 250 os_name = py_utils.GetHostOsName() 251 arch_name = py_utils.GetHostArchName() 252 else: 253 assert arch_name, 'os_name is specified but not arch_name' 254 255 dep_platform = '%s_%s' % (os_name, arch_name) 256 257 c = base_config.BaseConfig(config, writable=True) 258 try: 259 old_version = c.GetVersion(dependency, dep_platform) 260 print 'Updating from version: {}'.format(old_version) 261 except ValueError: 262 raise RuntimeError( 263 ('binary_dependencies.json entry for %s missing or invalid; please add ' 264 'it first! (need download_path and path_within_archive)') % 265 dep_platform) 266 267 if dep_local_path: 268 c.AddCloudStorageDependencyUpdateJob( 269 dependency, dep_platform, dep_local_path, version=version, 270 execute_job=True) 271