1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import contextlib
6import logging
7import os
8
9import py_utils
10from py_utils import binary_manager
11from py_utils import cloud_storage
12from py_utils import dependency_util
13import dependency_manager
14from dependency_manager import base_config
15
16from devil import devil_env
17
18
19from telemetry.core import exceptions
20from telemetry.core import util
21
22
23TELEMETRY_PROJECT_CONFIG = os.path.join(
24    util.GetTelemetryDir(), 'telemetry', 'binary_dependencies.json')
25
26
27CHROME_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'common', 'py_utils',
28                                    'py_utils', 'chrome_binaries.json')
29
30
31SUPPORTED_DEP_PLATFORMS = (
32    'linux_aarch64', 'linux_x86_64', 'linux_armv7l', 'linux_mips',
33    'mac_x86_64',
34    'win_x86', 'win_AMD64',
35    'android_arm64-v8a', 'android_armeabi-v7a', 'android_arm', 'android_x64',
36    'android_x86'
37)
38
39PLATFORMS_TO_DOWNLOAD_FOLDER_MAP = {
40    'linux_aarch64': 'bin/linux/aarch64',
41    'linux_x86_64': 'bin/linux/x86_64',
42    'linux_armv7l': 'bin/linux/armv7l',
43    'linux_mips': 'bin/linux/mips',
44    'mac_x86_64': 'bin/mac/x86_64',
45    'win_x86': 'bin/win/x86',
46    'win_AMD64': 'bin/win/AMD64',
47    'android_arm64-v8a': 'bin/android/arm64-v8a',
48    'android_armeabi-v7a': 'bin/android/armeabi-v7a',
49    'android_arm': 'bin/android/arm',
50    'android_x64': 'bin/android/x64',
51    'android_x86': 'bin/android/x86',
52}
53
54NoPathFoundError = dependency_manager.NoPathFoundError
55CloudStorageError = dependency_manager.CloudStorageError
56
57
58_binary_manager = None
59_installed_helpers = set()
60
61
62TELEMETRY_BINARY_BASE_CS_FOLDER = 'binary_dependencies'
63TELEMETRY_BINARY_CS_BUCKET = cloud_storage.PUBLIC_BUCKET
64
65def NeedsInit():
66  return not _binary_manager
67
68
69def InitDependencyManager(client_configs):
70  if GetBinaryManager():
71    raise exceptions.InitializationError(
72        'Trying to re-initialize the binary manager with config %s'
73        % client_configs)
74  configs = []
75  if client_configs:
76    configs += client_configs
77  configs += [TELEMETRY_PROJECT_CONFIG, CHROME_BINARY_CONFIG]
78  SetBinaryManager(binary_manager.BinaryManager(configs))
79
80  devil_env.config.Initialize()
81
82
83@contextlib.contextmanager
84def TemporarilyReplaceBinaryManager(manager):
85  old_manager = GetBinaryManager()
86  try:
87    SetBinaryManager(manager)
88    yield
89  finally:
90    SetBinaryManager(old_manager)
91
92
93def GetBinaryManager():
94  return _binary_manager
95
96
97def SetBinaryManager(manager):
98  global _binary_manager # pylint: disable=global-statement
99  _binary_manager = manager
100
101
102def _IsChromeOSLocalMode(os_name):
103  """Determines if we're running telemetry on a Chrome OS device.
104
105  Used to differentiate local mode (telemetry running on the CrOS DUT) from
106  remote mode (running telemetry on another platform that communicates with
107  the CrOS DUT over SSH).
108  """
109  return os_name == 'chromeos' and py_utils.GetHostOsName() == 'chromeos'
110
111
112def FetchPath(binary_name, os_name, arch, os_version=None):
113  """ Return a path to the appropriate executable for <binary_name>, downloading
114      from cloud storage if needed, or None if it cannot be found.
115  """
116  if GetBinaryManager() is None:
117    raise exceptions.InitializationError(
118        'Called FetchPath with uninitialized binary manager.')
119  return GetBinaryManager().FetchPath(
120      binary_name, 'linux' if _IsChromeOSLocalMode(os_name) else os_name,
121      arch, os_version)
122
123
124def LocalPath(binary_name, os_name, arch, os_version=None):
125  """ Return a local path to the given binary name, or None if an executable
126      cannot be found. Will not download the executable.
127      """
128  if GetBinaryManager() is None:
129    raise exceptions.InitializationError(
130        'Called LocalPath with uninitialized binary manager.')
131  return GetBinaryManager().LocalPath(binary_name, os_name, arch, os_version)
132
133
134def FetchBinaryDependencies(
135    platform, client_configs, fetch_reference_chrome_binary):
136  """ Fetch all binary dependenencies for the given |platform|.
137
138  Note: we don't fetch browser binaries by default because the size of the
139  binary is about 2Gb, and it requires cloud storage permission to
140  chrome-telemetry bucket.
141
142  Args:
143    platform: an instance of telemetry.core.platform
144    client_configs: A list of paths (string) to dependencies json files.
145    fetch_reference_chrome_binary: whether to fetch reference chrome binary for
146      the given platform.
147  """
148  configs = [
149      dependency_manager.BaseConfig(TELEMETRY_PROJECT_CONFIG),
150  ]
151  dep_manager = dependency_manager.DependencyManager(configs)
152  os_name = platform.GetOSName()
153  # If we're running directly on a Chrome OS device, fetch the binaries for
154  # linux instead, which should be compatible with CrOS. Otherwise, if we're
155  # running remotely on CrOS, fetch the binaries for the host platform like
156  # we do with android below.
157  if _IsChromeOSLocalMode(os_name):
158    os_name = 'linux'
159  target_platform = '%s_%s' % (os_name, platform.GetArchName())
160  dep_manager.PrefetchPaths(target_platform)
161
162  host_platform = None
163  fetch_devil_deps = False
164  if os_name in ('android', 'chromeos'):
165    host_platform = '%s_%s' % (
166        py_utils.GetHostOsName(), py_utils.GetHostArchName())
167    dep_manager.PrefetchPaths(host_platform)
168    if os_name == 'android':
169      if host_platform == 'linux_x86_64':
170        fetch_devil_deps = True
171      else:
172        logging.error('Devil only supports 64 bit linux as a host platform. '
173                      'Android tests may fail.')
174
175  if fetch_reference_chrome_binary:
176    _FetchReferenceBrowserBinary(platform)
177
178  # For now, handle client config separately because the BUILD.gn & .isolate of
179  # telemetry tests in chromium src failed to include the files specified in its
180  # client config.
181  # (https://github.com/catapult-project/catapult/issues/2192)
182  # For now this is ok because the client configs usually don't include cloud
183  # storage infos.
184  # TODO(crbug.com/1111556): remove the logic of swallowing exception once the
185  # issue is fixed on Chromium side.
186  if client_configs:
187    manager = dependency_manager.DependencyManager(
188        list(dependency_manager.BaseConfig(c) for c in client_configs))
189    try:
190      manager.PrefetchPaths(target_platform)
191      if host_platform is not None:
192        manager.PrefetchPaths(host_platform)
193
194    except dependency_manager.NoPathFoundError as e:
195      logging.error('Error when trying to prefetch paths for %s: %s',
196                    target_platform, e.message)
197
198  if fetch_devil_deps:
199    devil_env.config.Initialize()
200    devil_env.config.PrefetchPaths(arch=platform.GetArchName())
201    devil_env.config.PrefetchPaths()
202
203
204def ReinstallAndroidHelperIfNeeded(binary_name, install_path, device):
205  """ Install a binary helper to a specific location.
206
207  Args:
208    binary_name: (str) The name of the binary from binary_dependencies.json
209    install_path: (str) The path to install the binary at
210    device: (device_utils.DeviceUtils) a device to install the helper to
211  Raises:
212    Exception: When the binary could not be fetched or could not be pushed to
213        the device.
214  """
215  if (device.serial, install_path) in _installed_helpers:
216    return
217  host_path = FetchPath(binary_name, 'android', device.GetABI())
218  if not host_path:
219    raise Exception(
220        '%s binary could not be fetched as %s', binary_name, host_path)
221  device.PushChangedFiles([(host_path, install_path)])
222  device.RunShellCommand(['chmod', '777', install_path], check_return=True)
223  _installed_helpers.add((device.serial, install_path))
224
225
226def _FetchReferenceBrowserBinary(platform):
227  os_name = platform.GetOSName()
228  if _IsChromeOSLocalMode(os_name):
229    os_name = 'linux'
230  arch_name = platform.GetArchName()
231  manager = binary_manager.BinaryManager(
232      [CHROME_BINARY_CONFIG])
233  if os_name == 'android':
234    os_version = dependency_util.GetChromeApkOsVersion(
235        platform.GetOSVersionName())
236    manager.FetchPath(
237        'chrome_stable', os_name, arch_name, os_version)
238  else:
239    manager.FetchPath(
240        'chrome_stable', os_name, arch_name)
241
242
243def UpdateDependency(dependency, dep_local_path, version,
244                     os_name=None, arch_name=None):
245  config = os.path.join(
246      util.GetTelemetryDir(), 'telemetry', 'binary_dependencies.json')
247
248  if not os_name:
249    assert not arch_name, 'arch_name is specified but not os_name'
250    os_name = py_utils.GetHostOsName()
251    arch_name = py_utils.GetHostArchName()
252  else:
253    assert arch_name, 'os_name is specified but not arch_name'
254
255  dep_platform = '%s_%s' % (os_name, arch_name)
256
257  c = base_config.BaseConfig(config, writable=True)
258  try:
259    old_version = c.GetVersion(dependency, dep_platform)
260    print 'Updating from version: {}'.format(old_version)
261  except ValueError:
262    raise RuntimeError(
263        ('binary_dependencies.json entry for %s missing or invalid; please add '
264         'it first! (need download_path and path_within_archive)') %
265        dep_platform)
266
267  if dep_local_path:
268    c.AddCloudStorageDependencyUpdateJob(
269        dependency, dep_platform, dep_local_path, version=version,
270        execute_job=True)
271