1#!/usr/bin/env python
2# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS.  All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""Setup links to a Chromium checkout for WebRTC.
11
12WebRTC standalone shares a lot of dependencies and build tools with Chromium.
13To do this, many of the paths of a Chromium checkout is emulated by creating
14symlinks to files and directories. This script handles the setup of symlinks to
15achieve this.
16
17It also handles cleanup of the legacy Subversion-based approach that was used
18before Chrome switched over their master repo from Subversion to Git.
19"""
20
21
22import ctypes
23import errno
24import logging
25import optparse
26import os
27import shelve
28import shutil
29import subprocess
30import sys
31import textwrap
32
33
34DIRECTORIES = [
35  'build',
36  'buildtools',
37  'testing',
38  'third_party/binutils',
39  'third_party/boringssl',
40  'third_party/colorama',
41  'third_party/drmemory',
42  'third_party/expat',
43  'third_party/icu',
44  'third_party/instrumented_libraries',
45  'third_party/jsoncpp',
46  'third_party/libjpeg',
47  'third_party/libjpeg_turbo',
48  'third_party/libsrtp',
49  'third_party/libudev',
50  'third_party/libvpx_new',
51  'third_party/libyuv',
52  'third_party/llvm-build',
53  'third_party/lss',
54  'third_party/nss',
55  'third_party/ocmock',
56  'third_party/openmax_dl',
57  'third_party/opus',
58  'third_party/proguard',
59  'third_party/protobuf',
60  'third_party/sqlite',
61  'third_party/syzygy',
62  'third_party/usrsctp',
63  'third_party/yasm',
64  'third_party/zlib',
65  'tools/clang',
66  'tools/generate_library_loader',
67  'tools/gn',
68  'tools/gyp',
69  'tools/memory',
70  'tools/protoc_wrapper',
71  'tools/python',
72  'tools/swarming_client',
73  'tools/valgrind',
74  'tools/vim',
75  'tools/win',
76  'tools/xdisplaycheck',
77]
78
79from sync_chromium import get_target_os_list
80target_os = get_target_os_list()
81if 'android' in target_os:
82  DIRECTORIES += [
83    'base',
84    'third_party/android_platform',
85    'third_party/android_tools',
86    'third_party/appurify-python',
87    'third_party/ashmem',
88    'third_party/ijar',
89    'third_party/jsr-305',
90    'third_party/junit',
91    'third_party/libevent',
92    'third_party/libxml',
93    'third_party/mockito',
94    'third_party/modp_b64',
95    'third_party/requests',
96    'third_party/robolectric',
97    'tools/android',
98    'tools/grit',
99  ]
100if 'ios' in target_os:
101  DIRECTORIES.append('third_party/class-dump')
102
103FILES = {
104  'tools/isolate_driver.py': None,
105  'third_party/BUILD.gn': None,
106}
107
108ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
109CHROMIUM_CHECKOUT = os.path.join('chromium', 'src')
110LINKS_DB = 'links'
111
112# Version management to make future upgrades/downgrades easier to support.
113SCHEMA_VERSION = 1
114
115
116def query_yes_no(question, default=False):
117  """Ask a yes/no question via raw_input() and return their answer.
118
119  Modified from http://stackoverflow.com/a/3041990.
120  """
121  prompt = " [%s/%%s]: "
122  prompt = prompt % ('Y' if default is True  else 'y')
123  prompt = prompt % ('N' if default is False else 'n')
124
125  if default is None:
126    default = 'INVALID'
127
128  while True:
129    sys.stdout.write(question + prompt)
130    choice = raw_input().lower()
131    if choice == '' and default != 'INVALID':
132      return default
133
134    if 'yes'.startswith(choice):
135      return True
136    elif 'no'.startswith(choice):
137      return False
138
139    print "Please respond with 'yes' or 'no' (or 'y' or 'n')."
140
141
142# Actions
143class Action(object):
144  def __init__(self, dangerous):
145    self.dangerous = dangerous
146
147  def announce(self, planning):
148    """Log a description of this action.
149
150    Args:
151      planning - True iff we're in the planning stage, False if we're in the
152                 doit stage.
153    """
154    pass
155
156  def doit(self, links_db):
157    """Execute the action, recording what we did to links_db, if necessary."""
158    pass
159
160
161class Remove(Action):
162  def __init__(self, path, dangerous):
163    super(Remove, self).__init__(dangerous)
164    self._priority = 0
165    self._path = path
166
167  def announce(self, planning):
168    log = logging.warn
169    filesystem_type = 'file'
170    if not self.dangerous:
171      log = logging.info
172      filesystem_type = 'link'
173    if planning:
174      log('Planning to remove %s: %s', filesystem_type, self._path)
175    else:
176      log('Removing %s: %s', filesystem_type, self._path)
177
178  def doit(self, _):
179    os.remove(self._path)
180
181
182class Rmtree(Action):
183  def __init__(self, path):
184    super(Rmtree, self).__init__(dangerous=True)
185    self._priority = 0
186    self._path = path
187
188  def announce(self, planning):
189    if planning:
190      logging.warn('Planning to remove directory: %s', self._path)
191    else:
192      logging.warn('Removing directory: %s', self._path)
193
194  def doit(self, _):
195    if sys.platform.startswith('win'):
196      # shutil.rmtree() doesn't work on Windows if any of the directories are
197      # read-only, which svn repositories are.
198      subprocess.check_call(['rd', '/q', '/s', self._path], shell=True)
199    else:
200      shutil.rmtree(self._path)
201
202
203class Makedirs(Action):
204  def __init__(self, path):
205    super(Makedirs, self).__init__(dangerous=False)
206    self._priority = 1
207    self._path = path
208
209  def doit(self, _):
210    try:
211      os.makedirs(self._path)
212    except OSError as e:
213      if e.errno != errno.EEXIST:
214        raise
215
216
217class Symlink(Action):
218  def __init__(self, source_path, link_path):
219    super(Symlink, self).__init__(dangerous=False)
220    self._priority = 2
221    self._source_path = source_path
222    self._link_path = link_path
223
224  def announce(self, planning):
225    if planning:
226      logging.info(
227          'Planning to create link from %s to %s', self._link_path,
228          self._source_path)
229    else:
230      logging.debug(
231          'Linking from %s to %s', self._link_path, self._source_path)
232
233  def doit(self, links_db):
234    # Files not in the root directory need relative path calculation.
235    # On Windows, use absolute paths instead since NTFS doesn't seem to support
236    # relative paths for symlinks.
237    if sys.platform.startswith('win'):
238      source_path = os.path.abspath(self._source_path)
239    else:
240      if os.path.dirname(self._link_path) != self._link_path:
241        source_path = os.path.relpath(self._source_path,
242                                      os.path.dirname(self._link_path))
243
244    os.symlink(source_path, os.path.abspath(self._link_path))
245    links_db[self._source_path] = self._link_path
246
247
248class LinkError(IOError):
249  """Failed to create a link."""
250  pass
251
252
253# Handles symlink creation on the different platforms.
254if sys.platform.startswith('win'):
255  def symlink(source_path, link_path):
256    flag = 1 if os.path.isdir(source_path) else 0
257    if not ctypes.windll.kernel32.CreateSymbolicLinkW(
258        unicode(link_path), unicode(source_path), flag):
259      raise OSError('Failed to create symlink to %s. Notice that only NTFS '
260                    'version 5.0 and up has all the needed APIs for '
261                    'creating symlinks.' % source_path)
262  os.symlink = symlink
263
264
265class WebRTCLinkSetup(object):
266  def __init__(self, links_db, force=False, dry_run=False, prompt=False):
267    self._force = force
268    self._dry_run = dry_run
269    self._prompt = prompt
270    self._links_db = links_db
271
272  def CreateLinks(self, on_bot):
273    logging.debug('CreateLinks')
274    # First, make a plan of action
275    actions = []
276
277    for source_path, link_path in FILES.iteritems():
278      actions += self._ActionForPath(
279          source_path, link_path, check_fn=os.path.isfile, check_msg='files')
280    for source_dir in DIRECTORIES:
281      actions += self._ActionForPath(
282          source_dir, None, check_fn=os.path.isdir,
283          check_msg='directories')
284
285    if not on_bot and self._force:
286      # When making the manual switch from legacy SVN checkouts to the new
287      # Git-based Chromium DEPS, the .gclient_entries file that contains cached
288      # URLs for all DEPS entries must be removed to avoid future sync problems.
289      entries_file = os.path.join(os.path.dirname(ROOT_DIR), '.gclient_entries')
290      if os.path.exists(entries_file):
291        actions.append(Remove(entries_file, dangerous=True))
292
293    actions.sort()
294
295    if self._dry_run:
296      for action in actions:
297        action.announce(planning=True)
298      logging.info('Not doing anything because dry-run was specified.')
299      sys.exit(0)
300
301    if any(a.dangerous for a in actions):
302      logging.warn('Dangerous actions:')
303      for action in (a for a in actions if a.dangerous):
304        action.announce(planning=True)
305      print
306
307      if not self._force:
308        logging.error(textwrap.dedent("""\
309        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
310                              A C T I O N     R E Q I R E D
311        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
312
313        Because chromium/src is transitioning to Git (from SVN), we needed to
314        change the way that the WebRTC standalone checkout works. Instead of
315        individually syncing subdirectories of Chromium in SVN, we're now
316        syncing Chromium (and all of its DEPS, as defined by its own DEPS file),
317        into the `chromium/src` directory.
318
319        As such, all Chromium directories which are currently pulled by DEPS are
320        now replaced with a symlink into the full Chromium checkout.
321
322        To avoid disrupting developers, we've chosen to not delete your
323        directories forcibly, in case you have some work in progress in one of
324        them :).
325
326        ACTION REQUIRED:
327        Before running `gclient sync|runhooks` again, you must run:
328        %s%s --force
329
330        Which will replace all directories which now must be symlinks, after
331        prompting with a summary of the work-to-be-done.
332        """), 'python ' if sys.platform.startswith('win') else '', sys.argv[0])
333        sys.exit(1)
334      elif self._prompt:
335        if not query_yes_no('Would you like to perform the above plan?'):
336          sys.exit(1)
337
338    for action in actions:
339      action.announce(planning=False)
340      action.doit(self._links_db)
341
342    if not on_bot and self._force:
343      logging.info('Completed!\n\nNow run `gclient sync|runhooks` again to '
344                   'let the remaining hooks (that probably were interrupted) '
345                   'execute.')
346
347  def CleanupLinks(self):
348    logging.debug('CleanupLinks')
349    for source, link_path  in self._links_db.iteritems():
350      if source == 'SCHEMA_VERSION':
351        continue
352      if os.path.islink(link_path) or sys.platform.startswith('win'):
353        # os.path.islink() always returns false on Windows
354        # See http://bugs.python.org/issue13143.
355        logging.debug('Removing link to %s at %s', source, link_path)
356        if not self._dry_run:
357          if os.path.exists(link_path):
358            if sys.platform.startswith('win') and os.path.isdir(link_path):
359              subprocess.check_call(['rmdir', '/q', '/s', link_path],
360                                    shell=True)
361            else:
362              os.remove(link_path)
363          del self._links_db[source]
364
365  @staticmethod
366  def _ActionForPath(source_path, link_path=None, check_fn=None,
367                     check_msg=None):
368    """Create zero or more Actions to link to a file or directory.
369
370    This will be a symlink on POSIX platforms. On Windows this requires
371    that NTFS is version 5.0 or higher (Vista or newer).
372
373    Args:
374      source_path: Path relative to the Chromium checkout root.
375        For readability, the path may contain slashes, which will
376        automatically be converted to the right path delimiter on Windows.
377      link_path: The location for the link to create. If omitted it will be the
378        same path as source_path.
379      check_fn: A function returning true if the type of filesystem object is
380        correct for the attempted call. Otherwise an error message with
381        check_msg will be printed.
382      check_msg: String used to inform the user of an invalid attempt to create
383        a file.
384    Returns:
385      A list of Action objects.
386    """
387    def fix_separators(path):
388      if sys.platform.startswith('win'):
389        return path.replace(os.altsep, os.sep)
390      else:
391        return path
392
393    assert check_fn
394    assert check_msg
395    link_path = link_path or source_path
396    link_path = fix_separators(link_path)
397
398    source_path = fix_separators(source_path)
399    source_path = os.path.join(CHROMIUM_CHECKOUT, source_path)
400    if os.path.exists(source_path) and not check_fn:
401      raise LinkError('_LinkChromiumPath can only be used to link to %s: '
402                      'Tried to link to: %s' % (check_msg, source_path))
403
404    if not os.path.exists(source_path):
405      logging.debug('Silently ignoring missing source: %s. This is to avoid '
406                    'errors on platform-specific dependencies.', source_path)
407      return []
408
409    actions = []
410
411    if os.path.exists(link_path) or os.path.islink(link_path):
412      if os.path.islink(link_path):
413        actions.append(Remove(link_path, dangerous=False))
414      elif os.path.isfile(link_path):
415        actions.append(Remove(link_path, dangerous=True))
416      elif os.path.isdir(link_path):
417        actions.append(Rmtree(link_path))
418      else:
419        raise LinkError('Don\'t know how to plan: %s' % link_path)
420
421    # Create parent directories to the target link if needed.
422    target_parent_dirs = os.path.dirname(link_path)
423    if (target_parent_dirs and
424        target_parent_dirs != link_path and
425        not os.path.exists(target_parent_dirs)):
426      actions.append(Makedirs(target_parent_dirs))
427
428    actions.append(Symlink(source_path, link_path))
429
430    return actions
431
432def _initialize_database(filename):
433  links_database = shelve.open(filename)
434
435  # Wipe the database if this version of the script ends up looking at a
436  # newer (future) version of the links db, just to be sure.
437  version = links_database.get('SCHEMA_VERSION')
438  if version and version != SCHEMA_VERSION:
439    logging.info('Found database with schema version %s while this script only '
440                 'supports %s. Wiping previous database contents.', version,
441                 SCHEMA_VERSION)
442    links_database.clear()
443  links_database['SCHEMA_VERSION'] = SCHEMA_VERSION
444  return links_database
445
446
447def main():
448  on_bot = os.environ.get('CHROME_HEADLESS') == '1'
449
450  parser = optparse.OptionParser()
451  parser.add_option('-d', '--dry-run', action='store_true', default=False,
452                    help='Print what would be done, but don\'t perform any '
453                         'operations. This will automatically set logging to '
454                         'verbose.')
455  parser.add_option('-c', '--clean-only', action='store_true', default=False,
456                    help='Only clean previously created links, don\'t create '
457                         'new ones. This will automatically set logging to '
458                         'verbose.')
459  parser.add_option('-f', '--force', action='store_true', default=on_bot,
460                    help='Force link creation. CAUTION: This deletes existing '
461                         'folders and files in the locations where links are '
462                         'about to be created.')
463  parser.add_option('-n', '--no-prompt', action='store_false', dest='prompt',
464                    default=(not on_bot),
465                    help='Prompt if we\'re planning to do a dangerous action')
466  parser.add_option('-v', '--verbose', action='store_const',
467                    const=logging.DEBUG, default=logging.INFO,
468                    help='Print verbose output for debugging.')
469  options, _ = parser.parse_args()
470
471  if options.dry_run or options.force or options.clean_only:
472    options.verbose = logging.DEBUG
473  logging.basicConfig(format='%(message)s', level=options.verbose)
474
475  # Work from the root directory of the checkout.
476  script_dir = os.path.dirname(os.path.abspath(__file__))
477  os.chdir(script_dir)
478
479  if sys.platform.startswith('win'):
480    def is_admin():
481      try:
482        return os.getuid() == 0
483      except AttributeError:
484        return ctypes.windll.shell32.IsUserAnAdmin() != 0
485    if not is_admin():
486      logging.error('On Windows, you now need to have administrator '
487                    'privileges for the shell running %s (or '
488                    '`gclient sync|runhooks`).\nPlease start another command '
489                    'prompt as Administrator and try again.', sys.argv[0])
490      return 1
491
492  if not os.path.exists(CHROMIUM_CHECKOUT):
493    logging.error('Cannot find a Chromium checkout at %s. Did you run "gclient '
494                  'sync" before running this script?', CHROMIUM_CHECKOUT)
495    return 2
496
497  links_database = _initialize_database(LINKS_DB)
498  try:
499    symlink_creator = WebRTCLinkSetup(links_database, options.force,
500                                      options.dry_run, options.prompt)
501    symlink_creator.CleanupLinks()
502    if not options.clean_only:
503      symlink_creator.CreateLinks(on_bot)
504  except LinkError as e:
505    print >> sys.stderr, e.message
506    return 3
507  finally:
508    links_database.close()
509  return 0
510
511
512if __name__ == '__main__':
513  sys.exit(main())
514