1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5# pylint: disable=W0212
6
7import fcntl
8import inspect
9import logging
10import os
11import psutil
12import re
13import textwrap
14
15from devil import base_error
16from devil import devil_env
17from devil.android import device_errors
18from devil.android.constants import file_system
19from devil.android.sdk import adb_wrapper
20from devil.android.valgrind_tools import base_tool
21from devil.utils import cmd_helper
22
23logger = logging.getLogger(__name__)
24
25# If passed as the device port, this will tell the forwarder to allocate
26# a dynamic port on the device. The actual port can then be retrieved with
27# Forwarder.DevicePortForHostPort.
28DYNAMIC_DEVICE_PORT = 0
29
30PORT_REGEX = re.compile(r'(?P<device_port>\d+):(?P<host_port>\d+)')
31
32
33def _GetProcessStartTime(pid):
34  p = psutil.Process(pid)
35  if inspect.ismethod(p.create_time):
36    return p.create_time()
37  else:  # Process.create_time is a property in old versions of psutil.
38    return p.create_time
39
40
41def _DumpHostLog():
42  # The host forwarder daemon logs to /tmp/host_forwarder_log, so print the end
43  # of that.
44  try:
45    with open('/tmp/host_forwarder_log') as host_forwarder_log:
46      logger.info('Last 50 lines of the host forwarder daemon log:')
47      for line in host_forwarder_log.read().splitlines()[-50:]:
48        logger.info('    %s', line)
49  except Exception:  # pylint: disable=broad-except
50    # Grabbing the host forwarder log is best-effort. Ignore all errors.
51    logger.warning('Failed to get the contents of host_forwarder_log.')
52
53
54def _LogMapFailureDiagnostics(device):
55  _DumpHostLog()
56
57  # The device forwarder daemon logs to the logcat, so print the end of that.
58  try:
59    logger.info('Last 50 lines of logcat:')
60    for logcat_line in device.adb.Logcat(dump=True)[-50:]:
61      logger.info('    %s', logcat_line)
62  except (device_errors.CommandFailedError,
63          device_errors.DeviceUnreachableError):
64    # Grabbing the device forwarder log is also best-effort. Ignore all errors.
65    logger.warning('Failed to get the contents of the logcat.')
66
67  # Log alive device forwarders.
68  try:
69    ps_out = device.RunShellCommand(['ps'], check_return=True)
70    logger.info('Currently running device_forwarders:')
71    for line in ps_out:
72      if 'device_forwarder' in line:
73        logger.info('    %s', line)
74  except (device_errors.CommandFailedError,
75          device_errors.DeviceUnreachableError):
76    logger.warning('Failed to list currently running device_forwarder '
77                   'instances.')
78
79
80class _FileLock(object):
81  """With statement-aware implementation of a file lock.
82
83  File locks are needed for cross-process synchronization when the
84  multiprocessing Python module is used.
85  """
86
87  def __init__(self, path):
88    self._fd = -1
89    self._path = path
90
91  def __enter__(self):
92    self._fd = os.open(self._path, os.O_RDONLY | os.O_CREAT)
93    if self._fd < 0:
94      raise Exception('Could not open file %s for reading' % self._path)
95    fcntl.flock(self._fd, fcntl.LOCK_EX)
96
97  def __exit__(self, _exception_type, _exception_value, traceback):
98    fcntl.flock(self._fd, fcntl.LOCK_UN)
99    os.close(self._fd)
100
101
102class HostForwarderError(base_error.BaseError):
103  """Exception for failures involving host_forwarder."""
104
105  def __init__(self, message):
106    super(HostForwarderError, self).__init__(message)
107
108
109class Forwarder(object):
110  """Thread-safe class to manage port forwards from the device to the host."""
111
112  _DEVICE_FORWARDER_FOLDER = (file_system.TEST_EXECUTABLE_DIR + '/forwarder/')
113  _DEVICE_FORWARDER_PATH = (
114      file_system.TEST_EXECUTABLE_DIR + '/forwarder/device_forwarder')
115  _LOCK_PATH = '/tmp/chrome.forwarder.lock'
116  # Defined in host_forwarder_main.cc
117  _HOST_FORWARDER_LOG = '/tmp/host_forwarder_log'
118
119  _TIMEOUT = 60  # seconds
120
121  _instance = None
122
123  @staticmethod
124  def Map(port_pairs, device, tool=None):
125    """Runs the forwarder.
126
127    Args:
128      port_pairs: A list of tuples (device_port, host_port) to forward. Note
129                 that you can specify 0 as a device_port, in which case a
130                 port will by dynamically assigned on the device. You can
131                 get the number of the assigned port using the
132                 DevicePortForHostPort method.
133      device: A DeviceUtils instance.
134      tool: Tool class to use to get wrapper, if necessary, for executing the
135            forwarder (see valgrind_tools.py).
136
137    Raises:
138      Exception on failure to forward the port.
139    """
140    if not tool:
141      tool = base_tool.BaseTool()
142    with _FileLock(Forwarder._LOCK_PATH):
143      instance = Forwarder._GetInstanceLocked(tool)
144      instance._InitDeviceLocked(device, tool)
145
146      device_serial = str(device)
147      map_arg_lists = [[
148          '--adb=' + adb_wrapper.AdbWrapper.GetAdbPath(),
149          '--serial-id=' + device_serial, '--map',
150          str(device_port),
151          str(host_port)
152      ] for device_port, host_port in port_pairs]
153      logger.info('Forwarding using commands: %s', map_arg_lists)
154
155      for map_arg_list in map_arg_lists:
156        try:
157          map_cmd = [instance._host_forwarder_path] + map_arg_list
158          (exit_code, output) = cmd_helper.GetCmdStatusAndOutputWithTimeout(
159              map_cmd, Forwarder._TIMEOUT)
160        except cmd_helper.TimeoutError as e:
161          raise HostForwarderError(
162              '`%s` timed out:\n%s' % (' '.join(map_cmd), e.output))
163        except OSError as e:
164          if e.errno == 2:
165            raise HostForwarderError('Unable to start host forwarder. '
166                                     'Make sure you have built host_forwarder.')
167          else:
168            raise
169        if exit_code != 0:
170          try:
171            instance._KillDeviceLocked(device, tool)
172          except (device_errors.CommandFailedError,
173                  device_errors.DeviceUnreachableError):
174            # We don't want the failure to kill the device forwarder to
175            # supersede the original failure to map.
176            logger.warning(
177                'Failed to kill the device forwarder after map failure: %s',
178                str(e))
179          _LogMapFailureDiagnostics(device)
180          formatted_output = ('\n'.join(output)
181                              if isinstance(output, list) else output)
182          raise HostForwarderError(
183              '`%s` exited with %d:\n%s' % (' '.join(map_cmd), exit_code,
184                                            formatted_output))
185        for line in output.splitlines():
186          match = PORT_REGEX.match(line)
187          if match:
188            break
189        if not match:
190          raise HostForwarderError('Unable to find device_port:host_port in '
191                                   'host forwarder output: %s' % output)
192        device_port = int(match.groupdict()['device_port'])
193        host_port = int(match.groupdict()['host_port'])
194        serial_with_port = (device_serial, device_port)
195        instance._device_to_host_port_map[serial_with_port] = host_port
196        instance._host_to_device_port_map[host_port] = serial_with_port
197        logger.info('Forwarding device port: %d to host port: %d.', device_port,
198                    host_port)
199
200  @staticmethod
201  def UnmapDevicePort(device_port, device):
202    """Unmaps a previously forwarded device port.
203
204    Args:
205      device: A DeviceUtils instance.
206      device_port: A previously forwarded port (through Map()).
207    """
208    with _FileLock(Forwarder._LOCK_PATH):
209      Forwarder._UnmapDevicePortLocked(device_port, device)
210
211  @staticmethod
212  def UnmapAllDevicePorts(device):
213    """Unmaps all the previously forwarded ports for the provided device.
214
215    Args:
216      device: A DeviceUtils instance.
217      port_pairs: A list of tuples (device_port, host_port) to unmap.
218    """
219    with _FileLock(Forwarder._LOCK_PATH):
220      instance = Forwarder._GetInstanceLocked(None)
221      unmap_all_cmd = [
222          instance._host_forwarder_path,
223          '--adb=%s' % adb_wrapper.AdbWrapper.GetAdbPath(),
224          '--serial-id=%s' % device.serial, '--unmap-all'
225      ]
226      try:
227        exit_code, output = cmd_helper.GetCmdStatusAndOutputWithTimeout(
228            unmap_all_cmd, Forwarder._TIMEOUT)
229      except cmd_helper.TimeoutError as e:
230        raise HostForwarderError(
231            '`%s` timed out:\n%s' % (' '.join(unmap_all_cmd), e.output))
232      if exit_code != 0:
233        error_msg = [
234            '`%s` exited with %d' % (' '.join(unmap_all_cmd), exit_code)
235        ]
236        if isinstance(output, list):
237          error_msg += output
238        else:
239          error_msg += [output]
240        raise HostForwarderError('\n'.join(error_msg))
241
242      # Clean out any entries from the device & host map.
243      device_map = instance._device_to_host_port_map
244      host_map = instance._host_to_device_port_map
245      for device_serial_and_port, host_port in device_map.items():
246        device_serial = device_serial_and_port[0]
247        if device_serial == device.serial:
248          del device_map[device_serial_and_port]
249          del host_map[host_port]
250
251      # Kill the device forwarder.
252      tool = base_tool.BaseTool()
253      instance._KillDeviceLocked(device, tool)
254
255  @staticmethod
256  def DevicePortForHostPort(host_port):
257    """Returns the device port that corresponds to a given host port."""
258    with _FileLock(Forwarder._LOCK_PATH):
259      serial_and_port = Forwarder._GetInstanceLocked(
260          None)._host_to_device_port_map.get(host_port)
261      return serial_and_port[1] if serial_and_port else None
262
263  @staticmethod
264  def RemoveHostLog():
265    if os.path.exists(Forwarder._HOST_FORWARDER_LOG):
266      os.unlink(Forwarder._HOST_FORWARDER_LOG)
267
268  @staticmethod
269  def GetHostLog():
270    if not os.path.exists(Forwarder._HOST_FORWARDER_LOG):
271      return ''
272    with file(Forwarder._HOST_FORWARDER_LOG, 'r') as f:
273      return f.read()
274
275  @staticmethod
276  def _GetInstanceLocked(tool):
277    """Returns the singleton instance.
278
279    Note that the global lock must be acquired before calling this method.
280
281    Args:
282      tool: Tool class to use to get wrapper, if necessary, for executing the
283            forwarder (see valgrind_tools.py).
284    """
285    if not Forwarder._instance:
286      Forwarder._instance = Forwarder(tool)
287    return Forwarder._instance
288
289  def __init__(self, tool):
290    """Constructs a new instance of Forwarder.
291
292    Note that Forwarder is a singleton therefore this constructor should be
293    called only once.
294
295    Args:
296      tool: Tool class to use to get wrapper, if necessary, for executing the
297            forwarder (see valgrind_tools.py).
298    """
299    assert not Forwarder._instance
300    self._tool = tool
301    self._initialized_devices = set()
302    self._device_to_host_port_map = dict()
303    self._host_to_device_port_map = dict()
304    self._host_forwarder_path = devil_env.config.FetchPath('forwarder_host')
305    assert os.path.exists(self._host_forwarder_path), 'Please build forwarder2'
306    self._InitHostLocked()
307
308  @staticmethod
309  def _UnmapDevicePortLocked(device_port, device):
310    """Internal method used by UnmapDevicePort().
311
312    Note that the global lock must be acquired before calling this method.
313    """
314    instance = Forwarder._GetInstanceLocked(None)
315    serial = str(device)
316    serial_with_port = (serial, device_port)
317    if serial_with_port not in instance._device_to_host_port_map:
318      logger.error('Trying to unmap non-forwarded port %d', device_port)
319      return
320
321    host_port = instance._device_to_host_port_map[serial_with_port]
322    del instance._device_to_host_port_map[serial_with_port]
323    del instance._host_to_device_port_map[host_port]
324
325    unmap_cmd = [
326        instance._host_forwarder_path,
327        '--adb=%s' % adb_wrapper.AdbWrapper.GetAdbPath(),
328        '--serial-id=%s' % serial, '--unmap',
329        str(device_port)
330    ]
331    try:
332      (exit_code, output) = cmd_helper.GetCmdStatusAndOutputWithTimeout(
333          unmap_cmd, Forwarder._TIMEOUT)
334    except cmd_helper.TimeoutError as e:
335      raise HostForwarderError(
336          '`%s` timed out:\n%s' % (' '.join(unmap_cmd), e.output))
337    if exit_code != 0:
338      logger.error('`%s` exited with %d:\n%s', ' '.join(unmap_cmd), exit_code,
339                   '\n'.join(output) if isinstance(output, list) else output)
340
341  @staticmethod
342  def _GetPidForLock():
343    """Returns the PID used for host_forwarder initialization.
344
345    The PID of the "sharder" is used to handle multiprocessing. The "sharder"
346    is the initial process that forks that is the parent process.
347    """
348    return os.getpgrp()
349
350  def _InitHostLocked(self):
351    """Initializes the host forwarder daemon.
352
353    Note that the global lock must be acquired before calling this method. This
354    method kills any existing host_forwarder process that could be stale.
355    """
356    # See if the host_forwarder daemon was already initialized by a concurrent
357    # process or thread (in case multi-process sharding is not used).
358    # TODO(crbug.com/762005): Consider using a different implemention; relying
359    # on matching the string represantion of the process start time seems
360    # fragile.
361    pid_for_lock = Forwarder._GetPidForLock()
362    fd = os.open(Forwarder._LOCK_PATH, os.O_RDWR | os.O_CREAT)
363    with os.fdopen(fd, 'r+') as pid_file:
364      pid_with_start_time = pid_file.readline()
365      if pid_with_start_time:
366        (pid, process_start_time) = pid_with_start_time.split(':')
367        if pid == str(pid_for_lock):
368          if process_start_time == str(_GetProcessStartTime(pid_for_lock)):
369            return
370      self._KillHostLocked()
371      pid_file.seek(0)
372      pid_file.write(
373          '%s:%s' % (pid_for_lock, str(_GetProcessStartTime(pid_for_lock))))
374      pid_file.truncate()
375
376  def _InitDeviceLocked(self, device, tool):
377    """Initializes the device_forwarder daemon for a specific device (once).
378
379    Note that the global lock must be acquired before calling this method. This
380    method kills any existing device_forwarder daemon on the device that could
381    be stale, pushes the latest version of the daemon (to the device) and starts
382    it.
383
384    Args:
385      device: A DeviceUtils instance.
386      tool: Tool class to use to get wrapper, if necessary, for executing the
387            forwarder (see valgrind_tools.py).
388    """
389    device_serial = str(device)
390    if device_serial in self._initialized_devices:
391      return
392    try:
393      self._KillDeviceLocked(device, tool)
394    except device_errors.CommandFailedError:
395      logger.warning('Failed to kill device forwarder. Rebooting.')
396      device.Reboot()
397    forwarder_device_path_on_host = devil_env.config.FetchPath(
398        'forwarder_device', device=device)
399    forwarder_device_path_on_device = (
400        Forwarder._DEVICE_FORWARDER_FOLDER
401        if os.path.isdir(forwarder_device_path_on_host) else
402        Forwarder._DEVICE_FORWARDER_PATH)
403    device.PushChangedFiles([(forwarder_device_path_on_host,
404                              forwarder_device_path_on_device)])
405
406    cmd = [Forwarder._DEVICE_FORWARDER_PATH]
407    wrapper = tool.GetUtilWrapper()
408    if wrapper:
409      cmd.insert(0, wrapper)
410    device.RunShellCommand(
411        cmd,
412        env={'LD_LIBRARY_PATH': Forwarder._DEVICE_FORWARDER_FOLDER},
413        check_return=True)
414    self._initialized_devices.add(device_serial)
415
416  @staticmethod
417  def KillHost():
418    """Kills the forwarder process running on the host."""
419    with _FileLock(Forwarder._LOCK_PATH):
420      Forwarder._GetInstanceLocked(None)._KillHostLocked()
421
422  def _KillHostLocked(self):
423    """Kills the forwarder process running on the host.
424
425    Note that the global lock must be acquired before calling this method.
426    """
427    logger.info('Killing host_forwarder.')
428    try:
429      kill_cmd = [self._host_forwarder_path, '--kill-server']
430      (exit_code, output) = cmd_helper.GetCmdStatusAndOutputWithTimeout(
431          kill_cmd, Forwarder._TIMEOUT)
432      if exit_code != 0:
433        logger.warning('Forwarder unable to shut down:\n%s', output)
434        kill_cmd = ['pkill', '-9', 'host_forwarder']
435        (exit_code, output) = cmd_helper.GetCmdStatusAndOutputWithTimeout(
436            kill_cmd, Forwarder._TIMEOUT)
437        if exit_code == -9:
438          # pkill can exit with -9, which indicates that it was killed. It's
439          # possible that the forwarder was still killed, though, which will
440          # be checked later.
441          logging.warning('pkilling host forwarder returned -9.')
442        if exit_code in (0, 1):
443          # pkill exits with a 0 if it was able to signal at least one process.
444          # pkill exits with a 1 if it wasn't able to signal a process because
445          # no matching process existed. We're ok with either.
446          return
447
448        _, ps_output = cmd_helper.GetCmdStatusAndOutputWithTimeout(
449            ['ps', 'aux'], Forwarder._TIMEOUT)
450        host_forwarder_lines = [line for line in ps_output.splitlines()
451                                if 'host_forwarder' in line]
452        if host_forwarder_lines:
453          logger.error('Remaining host_forwarder processes:\n  %s',
454                       '\n  '.join(host_forwarder_lines))
455        else:
456          logger.error('No remaining host_forwarder processes?')
457          return
458        _DumpHostLog()
459        error_msg = textwrap.dedent("""\
460            `{kill_cmd}` failed to kill host_forwarder.
461              exit_code: {exit_code}
462              output:
463            {output}
464            """)
465        raise HostForwarderError(
466            error_msg.format(
467                kill_cmd=' '.join(kill_cmd), exit_code=str(exit_code),
468                output='\n'.join('    %s' % l for l in output.splitlines())))
469    except cmd_helper.TimeoutError as e:
470      raise HostForwarderError(
471          '`%s` timed out:\n%s' % (' '.join(kill_cmd), e.output))
472
473  @staticmethod
474  def KillDevice(device, tool=None):
475    """Kills the forwarder process running on the device.
476
477    Args:
478      device: Instance of DeviceUtils for talking to the device.
479      tool: Wrapper tool (e.g. valgrind) that can be used to execute the device
480            forwarder (see valgrind_tools.py).
481    """
482    with _FileLock(Forwarder._LOCK_PATH):
483      Forwarder._GetInstanceLocked(None)._KillDeviceLocked(
484          device, tool or base_tool.BaseTool())
485
486  def _KillDeviceLocked(self, device, tool):
487    """Kills the forwarder process running on the device.
488
489    Note that the global lock must be acquired before calling this method.
490
491    Args:
492      device: Instance of DeviceUtils for talking to the device.
493      tool: Wrapper tool (e.g. valgrind) that can be used to execute the device
494            forwarder (see valgrind_tools.py).
495    """
496    logger.info('Killing device_forwarder.')
497    self._initialized_devices.discard(device.serial)
498    if not device.FileExists(Forwarder._DEVICE_FORWARDER_PATH):
499      return
500
501    cmd = [Forwarder._DEVICE_FORWARDER_PATH, '--kill-server']
502    wrapper = tool.GetUtilWrapper()
503    if wrapper:
504      cmd.insert(0, wrapper)
505    device.RunShellCommand(
506        cmd,
507        env={'LD_LIBRARY_PATH': Forwarder._DEVICE_FORWARDER_FOLDER},
508        check_return=True)
509