1#!/usr/bin/python2.7
2# Copyright 2016 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Prints all non-system dependencies for the given module.
7
8The primary use-case for this script is to generate the list of python modules
9required for .isolate files.
10
11This script should be compatible with Python 2 and Python 3.
12"""
13
14import argparse
15import os
16import pipes
17import sys
18
19# Don't use any helper modules, or else they will end up in the results.
20
21
22_SRC_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
23
24
25def ComputePythonDependencies():
26  """Gets the paths of imported non-system python modules.
27
28  A path is assumed to be a "system" import if it is outside of chromium's
29  src/. The paths will be relative to the current directory.
30  """
31  module_paths = (m.__file__ for m in sys.modules.values()
32                  if m and hasattr(m, '__file__'))
33
34  src_paths = set()
35  for path in module_paths:
36    if path == __file__:
37      continue
38    path = os.path.abspath(path)
39    if not path.startswith(_SRC_ROOT):
40      continue
41
42    if (path.endswith('.pyc')
43        or (path.endswith('c') and not os.path.splitext(path)[1])):
44      path = path[:-1]
45    src_paths.add(path)
46
47  return src_paths
48
49
50def _NormalizeCommandLine(options):
51  """Returns a string that when run from SRC_ROOT replicates the command."""
52  args = ['build/print_python_deps.py']
53  root = os.path.relpath(options.root, _SRC_ROOT)
54  if root != '.':
55    args.extend(('--root', root))
56  if options.output:
57    args.extend(('--output', os.path.relpath(options.output, _SRC_ROOT)))
58  if options.gn_paths:
59    args.extend(('--gn-paths',))
60  for whitelist in sorted(options.whitelists):
61    args.extend(('--whitelist', os.path.relpath(whitelist, _SRC_ROOT)))
62  args.append(os.path.relpath(options.module, _SRC_ROOT))
63  return ' '.join(pipes.quote(x) for x in args)
64
65
66def _FindPythonInDirectory(directory):
67  """Returns an iterable of all non-test python files in the given directory."""
68  files = []
69  for root, _dirnames, filenames in os.walk(directory):
70    for filename in filenames:
71      if filename.endswith('.py') and not filename.endswith('_test.py'):
72        yield os.path.join(root, filename)
73
74
75def _GetTargetPythonVersion(module):
76  """Heuristically determines the target module's Python version."""
77  with open(module) as f:
78    shebang = f.readline().strip()
79  default_version = 2
80  if shebang.startswith('#!'):
81    # Examples:
82    # '#!/usr/bin/python'
83    # '#!/usr/bin/python2.7'
84    # '#!/usr/bin/python3'
85    # '#!/usr/bin/env python3'
86    # '#!/usr/bin/env vpython'
87    # '#!/usr/bin/env vpython3'
88    exec_name = os.path.basename(shebang[2:].split(' ')[-1])
89    for python_prefix in ['python', 'vpython']:
90      if exec_name.startswith(python_prefix):
91        version_string = exec_name[len(python_prefix):]
92        break
93    else:
94      raise ValueError('Invalid shebang: ' + shebang)
95    if version_string:
96      return int(float(version_string))
97  return default_version
98
99
100def _ImportModuleByPath(module_path):
101  """Imports a module by its source file."""
102  sys.path[0] = os.path.dirname(module_path)
103  if sys.version_info[0] == 2:
104    import imp  # Python 2 only, since it's deprecated in Python 3.
105    imp.load_source('NAME', module_path)
106  else:
107    # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
108    module_name = os.path.splitext(os.path.basename(module_path))[0]
109    import importlib.util  # Python 3 only, since it's unavailable in Python 2.
110    spec = importlib.util.spec_from_file_location(module_name, module_path)
111    module = importlib.util.module_from_spec(spec)
112    sys.modules[module_name] = module
113    spec.loader.exec_module(module)
114
115
116def main():
117  parser = argparse.ArgumentParser(
118      description='Prints all non-system dependencies for the given module.')
119  parser.add_argument('module',
120                      help='The python module to analyze.')
121  parser.add_argument('--root', default='.',
122                      help='Directory to make paths relative to.')
123  parser.add_argument('--output',
124                      help='Write output to a file rather than stdout.')
125  parser.add_argument('--inplace', action='store_true',
126                      help='Write output to a file with the same path as the '
127                      'module, but with a .pydeps extension. Also sets the '
128                      'root to the module\'s directory.')
129  parser.add_argument('--no-header', action='store_true',
130                      help='Do not write the "# Generated by" header.')
131  parser.add_argument('--gn-paths', action='store_true',
132                      help='Write paths as //foo/bar/baz.py')
133  parser.add_argument('--did-relaunch', action='store_true',
134                      help=argparse.SUPPRESS)
135  parser.add_argument('--whitelist', default=[], action='append',
136                      dest='whitelists',
137                      help='Recursively include all non-test python files '
138                      'within this directory. May be specified multiple times.')
139  options = parser.parse_args()
140
141  if options.inplace:
142    if options.output:
143      parser.error('Cannot use --inplace and --output at the same time!')
144    if not options.module.endswith('.py'):
145      parser.error('Input module path should end with .py suffix!')
146    options.output = options.module + 'deps'
147    options.root = os.path.dirname(options.module)
148
149  target_version = _GetTargetPythonVersion(options.module)
150  assert target_version in [2, 3]
151  current_version = sys.version_info[0]
152
153  # Trybots run with vpython as default Python, but with a different config
154  # from //.vpython. To make the is_vpython test work, and to match the behavior
155  # of dev machines, the shebang line must be run with python2.7.
156  #
157  # E.g. $HOME/.vpython-root/dd50d3/bin/python
158  # E.g. /b/s/w/ir/cache/vpython/ab5c79/bin/python
159  is_vpython = 'vpython' in sys.executable
160  if not is_vpython or target_version != current_version:
161    # Prevent infinite relaunch if something goes awry.
162    assert not options.did_relaunch
163    # Re-launch using vpython will cause us to pick up modules specified in
164    # //.vpython, but does not cause it to pick up modules defined inline via
165    # [VPYTHON:BEGIN] ... [VPYTHON:END] comments.
166    # TODO(agrieve): Add support for this if the need ever arises.
167    vpython_to_use = {2: 'vpython', 3: 'vpython3'}[target_version]
168    os.execvp(vpython_to_use, [vpython_to_use] + sys.argv + ['--did-relaunch'])
169
170  # Replace the path entry for print_python_deps.py with the one for the given
171  # module.
172  try:
173    _ImportModuleByPath(options.module)
174  except Exception:
175    # Output extra diagnostics when loading the script fails.
176    sys.stderr.write('Error running print_python_deps.py.\n')
177    sys.stderr.write('is_vpython={}\n'.format(is_vpython))
178    sys.stderr.write('did_relanuch={}\n'.format(options.did_relaunch))
179    sys.stderr.write('python={}\n'.format(sys.executable))
180    raise
181
182  paths_set = ComputePythonDependencies()
183  for path in options.whitelists:
184    paths_set.update(os.path.abspath(p) for p in _FindPythonInDirectory(path))
185
186  paths = [os.path.relpath(p, options.root) for p in paths_set]
187
188  normalized_cmdline = _NormalizeCommandLine(options)
189  out = open(options.output, 'w') if options.output else sys.stdout
190  with out:
191    if not options.no_header:
192      out.write('# Generated by running:\n')
193      out.write('#   %s\n' % normalized_cmdline)
194    prefix = '//' if options.gn_paths else ''
195    for path in sorted(paths):
196      out.write(prefix + path + '\n')
197
198
199if __name__ == '__main__':
200  sys.exit(main())
201