1#!/usr/bin/env python
2# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS.  All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""Setup links to a Chromium checkout for WebRTC.
11
12WebRTC standalone shares a lot of dependencies and build tools with Chromium.
13To do this, many of the paths of a Chromium checkout is emulated by creating
14symlinks to files and directories. This script handles the setup of symlinks to
15achieve this.
16
17It also handles cleanup of the legacy Subversion-based approach that was used
18before Chrome switched over their master repo from Subversion to Git.
19"""
20
21
22import ctypes
23import errno
24import logging
25import optparse
26import os
27import shelve
28import shutil
29import subprocess
30import sys
31import textwrap
32
33
34DIRECTORIES = [
35  'build',
36  'buildtools',
37  'mojo',  # TODO(kjellander): Remove, see webrtc:5629.
38  'native_client',
39  'net',
40  'testing',
41  'third_party/binutils',
42  'third_party/drmemory',
43  'third_party/instrumented_libraries',
44  'third_party/libjpeg',
45  'third_party/libjpeg_turbo',
46  'third_party/llvm-build',
47  'third_party/lss',
48  'third_party/yasm',
49  'third_party/WebKit',  # TODO(kjellander): Remove, see webrtc:5629.
50  'tools/clang',
51  'tools/gn',
52  'tools/gyp',
53  'tools/memory',
54  'tools/python',
55  'tools/swarming_client',
56  'tools/valgrind',
57  'tools/vim',
58  'tools/win',
59]
60
61from sync_chromium import get_target_os_list
62target_os = get_target_os_list()
63if 'android' in target_os:
64  DIRECTORIES += [
65    'base',
66    'third_party/android_platform',
67    'third_party/android_tools',
68    'third_party/appurify-python',
69    'third_party/ashmem',
70    'third_party/catapult',
71    'third_party/icu',
72    'third_party/ijar',
73    'third_party/jsr-305',
74    'third_party/junit',
75    'third_party/libxml',
76    'third_party/mockito',
77    'third_party/modp_b64',
78    'third_party/protobuf',
79    'third_party/requests',
80    'third_party/robolectric',
81    'tools/android',
82    'tools/grit',
83  ]
84if 'ios' in target_os:
85  DIRECTORIES.append('third_party/class-dump')
86
87FILES = {
88  'tools/isolate_driver.py': None,
89  'third_party/BUILD.gn': None,
90}
91
92ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
93CHROMIUM_CHECKOUT = os.path.join('chromium', 'src')
94LINKS_DB = 'links'
95
96# Version management to make future upgrades/downgrades easier to support.
97SCHEMA_VERSION = 1
98
99
100def query_yes_no(question, default=False):
101  """Ask a yes/no question via raw_input() and return their answer.
102
103  Modified from http://stackoverflow.com/a/3041990.
104  """
105  prompt = " [%s/%%s]: "
106  prompt = prompt % ('Y' if default is True  else 'y')
107  prompt = prompt % ('N' if default is False else 'n')
108
109  if default is None:
110    default = 'INVALID'
111
112  while True:
113    sys.stdout.write(question + prompt)
114    choice = raw_input().lower()
115    if choice == '' and default != 'INVALID':
116      return default
117
118    if 'yes'.startswith(choice):
119      return True
120    elif 'no'.startswith(choice):
121      return False
122
123    print "Please respond with 'yes' or 'no' (or 'y' or 'n')."
124
125
126# Actions
127class Action(object):
128  def __init__(self, dangerous):
129    self.dangerous = dangerous
130
131  def announce(self, planning):
132    """Log a description of this action.
133
134    Args:
135      planning - True iff we're in the planning stage, False if we're in the
136                 doit stage.
137    """
138    pass
139
140  def doit(self, links_db):
141    """Execute the action, recording what we did to links_db, if necessary."""
142    pass
143
144
145class Remove(Action):
146  def __init__(self, path, dangerous):
147    super(Remove, self).__init__(dangerous)
148    self._priority = 0
149    self._path = path
150
151  def announce(self, planning):
152    log = logging.warn
153    filesystem_type = 'file'
154    if not self.dangerous:
155      log = logging.info
156      filesystem_type = 'link'
157    if planning:
158      log('Planning to remove %s: %s', filesystem_type, self._path)
159    else:
160      log('Removing %s: %s', filesystem_type, self._path)
161
162  def doit(self, _):
163    os.remove(self._path)
164
165
166class Rmtree(Action):
167  def __init__(self, path):
168    super(Rmtree, self).__init__(dangerous=True)
169    self._priority = 0
170    self._path = path
171
172  def announce(self, planning):
173    if planning:
174      logging.warn('Planning to remove directory: %s', self._path)
175    else:
176      logging.warn('Removing directory: %s', self._path)
177
178  def doit(self, _):
179    if sys.platform.startswith('win'):
180      # shutil.rmtree() doesn't work on Windows if any of the directories are
181      # read-only, which svn repositories are.
182      subprocess.check_call(['rd', '/q', '/s', self._path], shell=True)
183    else:
184      shutil.rmtree(self._path)
185
186
187class Makedirs(Action):
188  def __init__(self, path):
189    super(Makedirs, self).__init__(dangerous=False)
190    self._priority = 1
191    self._path = path
192
193  def doit(self, _):
194    try:
195      os.makedirs(self._path)
196    except OSError as e:
197      if e.errno != errno.EEXIST:
198        raise
199
200
201class Symlink(Action):
202  def __init__(self, source_path, link_path):
203    super(Symlink, self).__init__(dangerous=False)
204    self._priority = 2
205    self._source_path = source_path
206    self._link_path = link_path
207
208  def announce(self, planning):
209    if planning:
210      logging.info(
211          'Planning to create link from %s to %s', self._link_path,
212          self._source_path)
213    else:
214      logging.debug(
215          'Linking from %s to %s', self._link_path, self._source_path)
216
217  def doit(self, links_db):
218    # Files not in the root directory need relative path calculation.
219    # On Windows, use absolute paths instead since NTFS doesn't seem to support
220    # relative paths for symlinks.
221    if sys.platform.startswith('win'):
222      source_path = os.path.abspath(self._source_path)
223    else:
224      if os.path.dirname(self._link_path) != self._link_path:
225        source_path = os.path.relpath(self._source_path,
226                                      os.path.dirname(self._link_path))
227
228    os.symlink(source_path, os.path.abspath(self._link_path))
229    links_db[self._source_path] = self._link_path
230
231
232class LinkError(IOError):
233  """Failed to create a link."""
234  pass
235
236
237# Handles symlink creation on the different platforms.
238if sys.platform.startswith('win'):
239  def symlink(source_path, link_path):
240    flag = 1 if os.path.isdir(source_path) else 0
241    if not ctypes.windll.kernel32.CreateSymbolicLinkW(
242        unicode(link_path), unicode(source_path), flag):
243      raise OSError('Failed to create symlink to %s. Notice that only NTFS '
244                    'version 5.0 and up has all the needed APIs for '
245                    'creating symlinks.' % source_path)
246  os.symlink = symlink
247
248
249class WebRTCLinkSetup(object):
250  def __init__(self, links_db, force=False, dry_run=False, prompt=False):
251    self._force = force
252    self._dry_run = dry_run
253    self._prompt = prompt
254    self._links_db = links_db
255
256  def CreateLinks(self, on_bot):
257    logging.debug('CreateLinks')
258    # First, make a plan of action
259    actions = []
260
261    for source_path, link_path in FILES.iteritems():
262      actions += self._ActionForPath(
263          source_path, link_path, check_fn=os.path.isfile, check_msg='files')
264    for source_dir in DIRECTORIES:
265      actions += self._ActionForPath(
266          source_dir, None, check_fn=os.path.isdir,
267          check_msg='directories')
268
269    if not on_bot and self._force:
270      # When making the manual switch from legacy SVN checkouts to the new
271      # Git-based Chromium DEPS, the .gclient_entries file that contains cached
272      # URLs for all DEPS entries must be removed to avoid future sync problems.
273      entries_file = os.path.join(os.path.dirname(ROOT_DIR), '.gclient_entries')
274      if os.path.exists(entries_file):
275        actions.append(Remove(entries_file, dangerous=True))
276
277    actions.sort()
278
279    if self._dry_run:
280      for action in actions:
281        action.announce(planning=True)
282      logging.info('Not doing anything because dry-run was specified.')
283      sys.exit(0)
284
285    if any(a.dangerous for a in actions):
286      logging.warn('Dangerous actions:')
287      for action in (a for a in actions if a.dangerous):
288        action.announce(planning=True)
289      print
290
291      if not self._force:
292        logging.error(textwrap.dedent("""\
293        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
294                              A C T I O N     R E Q I R E D
295        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
296
297        Because chromium/src is transitioning to Git (from SVN), we needed to
298        change the way that the WebRTC standalone checkout works. Instead of
299        individually syncing subdirectories of Chromium in SVN, we're now
300        syncing Chromium (and all of its DEPS, as defined by its own DEPS file),
301        into the `chromium/src` directory.
302
303        As such, all Chromium directories which are currently pulled by DEPS are
304        now replaced with a symlink into the full Chromium checkout.
305
306        To avoid disrupting developers, we've chosen to not delete your
307        directories forcibly, in case you have some work in progress in one of
308        them :).
309
310        ACTION REQUIRED:
311        Before running `gclient sync|runhooks` again, you must run:
312        %s%s --force
313
314        Which will replace all directories which now must be symlinks, after
315        prompting with a summary of the work-to-be-done.
316        """), 'python ' if sys.platform.startswith('win') else '', sys.argv[0])
317        sys.exit(1)
318      elif self._prompt:
319        if not query_yes_no('Would you like to perform the above plan?'):
320          sys.exit(1)
321
322    for action in actions:
323      action.announce(planning=False)
324      action.doit(self._links_db)
325
326    if not on_bot and self._force:
327      logging.info('Completed!\n\nNow run `gclient sync|runhooks` again to '
328                   'let the remaining hooks (that probably were interrupted) '
329                   'execute.')
330
331  def CleanupLinks(self):
332    logging.debug('CleanupLinks')
333    for source, link_path  in self._links_db.iteritems():
334      if source == 'SCHEMA_VERSION':
335        continue
336      if os.path.islink(link_path) or sys.platform.startswith('win'):
337        # os.path.islink() always returns false on Windows
338        # See http://bugs.python.org/issue13143.
339        logging.debug('Removing link to %s at %s', source, link_path)
340        if not self._dry_run:
341          if os.path.exists(link_path):
342            if sys.platform.startswith('win') and os.path.isdir(link_path):
343              subprocess.check_call(['rmdir', '/q', '/s', link_path],
344                                    shell=True)
345            else:
346              os.remove(link_path)
347          del self._links_db[source]
348
349  @staticmethod
350  def _ActionForPath(source_path, link_path=None, check_fn=None,
351                     check_msg=None):
352    """Create zero or more Actions to link to a file or directory.
353
354    This will be a symlink on POSIX platforms. On Windows this requires
355    that NTFS is version 5.0 or higher (Vista or newer).
356
357    Args:
358      source_path: Path relative to the Chromium checkout root.
359        For readability, the path may contain slashes, which will
360        automatically be converted to the right path delimiter on Windows.
361      link_path: The location for the link to create. If omitted it will be the
362        same path as source_path.
363      check_fn: A function returning true if the type of filesystem object is
364        correct for the attempted call. Otherwise an error message with
365        check_msg will be printed.
366      check_msg: String used to inform the user of an invalid attempt to create
367        a file.
368    Returns:
369      A list of Action objects.
370    """
371    def fix_separators(path):
372      if sys.platform.startswith('win'):
373        return path.replace(os.altsep, os.sep)
374      else:
375        return path
376
377    assert check_fn
378    assert check_msg
379    link_path = link_path or source_path
380    link_path = fix_separators(link_path)
381
382    source_path = fix_separators(source_path)
383    source_path = os.path.join(CHROMIUM_CHECKOUT, source_path)
384    if os.path.exists(source_path) and not check_fn:
385      raise LinkError('_LinkChromiumPath can only be used to link to %s: '
386                      'Tried to link to: %s' % (check_msg, source_path))
387
388    if not os.path.exists(source_path):
389      logging.debug('Silently ignoring missing source: %s. This is to avoid '
390                    'errors on platform-specific dependencies.', source_path)
391      return []
392
393    actions = []
394
395    if os.path.exists(link_path) or os.path.islink(link_path):
396      if os.path.islink(link_path):
397        actions.append(Remove(link_path, dangerous=False))
398      elif os.path.isfile(link_path):
399        actions.append(Remove(link_path, dangerous=True))
400      elif os.path.isdir(link_path):
401        actions.append(Rmtree(link_path))
402      else:
403        raise LinkError('Don\'t know how to plan: %s' % link_path)
404
405    # Create parent directories to the target link if needed.
406    target_parent_dirs = os.path.dirname(link_path)
407    if (target_parent_dirs and
408        target_parent_dirs != link_path and
409        not os.path.exists(target_parent_dirs)):
410      actions.append(Makedirs(target_parent_dirs))
411
412    actions.append(Symlink(source_path, link_path))
413
414    return actions
415
416def _initialize_database(filename):
417  links_database = shelve.open(filename)
418
419  # Wipe the database if this version of the script ends up looking at a
420  # newer (future) version of the links db, just to be sure.
421  version = links_database.get('SCHEMA_VERSION')
422  if version and version != SCHEMA_VERSION:
423    logging.info('Found database with schema version %s while this script only '
424                 'supports %s. Wiping previous database contents.', version,
425                 SCHEMA_VERSION)
426    links_database.clear()
427  links_database['SCHEMA_VERSION'] = SCHEMA_VERSION
428  return links_database
429
430
431def main():
432  on_bot = os.environ.get('CHROME_HEADLESS') == '1'
433
434  parser = optparse.OptionParser()
435  parser.add_option('-d', '--dry-run', action='store_true', default=False,
436                    help='Print what would be done, but don\'t perform any '
437                         'operations. This will automatically set logging to '
438                         'verbose.')
439  parser.add_option('-c', '--clean-only', action='store_true', default=False,
440                    help='Only clean previously created links, don\'t create '
441                         'new ones. This will automatically set logging to '
442                         'verbose.')
443  parser.add_option('-f', '--force', action='store_true', default=on_bot,
444                    help='Force link creation. CAUTION: This deletes existing '
445                         'folders and files in the locations where links are '
446                         'about to be created.')
447  parser.add_option('-n', '--no-prompt', action='store_false', dest='prompt',
448                    default=(not on_bot),
449                    help='Prompt if we\'re planning to do a dangerous action')
450  parser.add_option('-v', '--verbose', action='store_const',
451                    const=logging.DEBUG, default=logging.INFO,
452                    help='Print verbose output for debugging.')
453  options, _ = parser.parse_args()
454
455  if options.dry_run or options.force or options.clean_only:
456    options.verbose = logging.DEBUG
457  logging.basicConfig(format='%(message)s', level=options.verbose)
458
459  # Work from the root directory of the checkout.
460  script_dir = os.path.dirname(os.path.abspath(__file__))
461  os.chdir(script_dir)
462
463  if sys.platform.startswith('win'):
464    def is_admin():
465      try:
466        return os.getuid() == 0
467      except AttributeError:
468        return ctypes.windll.shell32.IsUserAnAdmin() != 0
469    if not is_admin():
470      logging.error('On Windows, you now need to have administrator '
471                    'privileges for the shell running %s (or '
472                    '`gclient sync|runhooks`).\nPlease start another command '
473                    'prompt as Administrator and try again.', sys.argv[0])
474      return 1
475
476  if not os.path.exists(CHROMIUM_CHECKOUT):
477    logging.error('Cannot find a Chromium checkout at %s. Did you run "gclient '
478                  'sync" before running this script?', CHROMIUM_CHECKOUT)
479    return 2
480
481  links_database = _initialize_database(LINKS_DB)
482  try:
483    symlink_creator = WebRTCLinkSetup(links_database, options.force,
484                                      options.dry_run, options.prompt)
485    symlink_creator.CleanupLinks()
486    if not options.clean_only:
487      symlink_creator.CreateLinks(on_bot)
488  except LinkError as e:
489    print >> sys.stderr, e.message
490    return 3
491  finally:
492    links_database.close()
493  return 0
494
495
496if __name__ == '__main__':
497  sys.exit(main())
498