1#!/usr/bin/env python 2# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3# 4# Use of this source code is governed by a BSD-style license 5# that can be found in the LICENSE file in the root of the source 6# tree. An additional intellectual property rights grant can be found 7# in the file PATENTS. All contributing project authors may 8# be found in the AUTHORS file in the root of the source tree. 9 10"""Setup links to a Chromium checkout for WebRTC. 11 12WebRTC standalone shares a lot of dependencies and build tools with Chromium. 13To do this, many of the paths of a Chromium checkout is emulated by creating 14symlinks to files and directories. This script handles the setup of symlinks to 15achieve this. 16 17It also handles cleanup of the legacy Subversion-based approach that was used 18before Chrome switched over their master repo from Subversion to Git. 19""" 20 21 22import ctypes 23import errno 24import logging 25import optparse 26import os 27import shelve 28import shutil 29import subprocess 30import sys 31import textwrap 32 33 34DIRECTORIES = [ 35 'build', 36 'buildtools', 37 'testing', 38 'third_party/binutils', 39 'third_party/boringssl', 40 'third_party/colorama', 41 'third_party/drmemory', 42 'third_party/expat', 43 'third_party/icu', 44 'third_party/instrumented_libraries', 45 'third_party/jsoncpp', 46 'third_party/libjpeg', 47 'third_party/libjpeg_turbo', 48 'third_party/libsrtp', 49 'third_party/libudev', 50 'third_party/libvpx_new', 51 'third_party/libyuv', 52 'third_party/llvm-build', 53 'third_party/lss', 54 'third_party/nss', 55 'third_party/ocmock', 56 'third_party/openmax_dl', 57 'third_party/opus', 58 'third_party/proguard', 59 'third_party/protobuf', 60 'third_party/sqlite', 61 'third_party/syzygy', 62 'third_party/usrsctp', 63 'third_party/yasm', 64 'third_party/zlib', 65 'tools/clang', 66 'tools/generate_library_loader', 67 'tools/gn', 68 'tools/gyp', 69 'tools/memory', 70 'tools/protoc_wrapper', 71 'tools/python', 72 'tools/swarming_client', 73 'tools/valgrind', 74 'tools/vim', 75 'tools/win', 76 'tools/xdisplaycheck', 77] 78 79from sync_chromium import get_target_os_list 80target_os = get_target_os_list() 81if 'android' in target_os: 82 DIRECTORIES += [ 83 'base', 84 'third_party/android_platform', 85 'third_party/android_tools', 86 'third_party/appurify-python', 87 'third_party/ashmem', 88 'third_party/ijar', 89 'third_party/jsr-305', 90 'third_party/junit', 91 'third_party/libevent', 92 'third_party/libxml', 93 'third_party/mockito', 94 'third_party/modp_b64', 95 'third_party/requests', 96 'third_party/robolectric', 97 'tools/android', 98 'tools/grit', 99 ] 100if 'ios' in target_os: 101 DIRECTORIES.append('third_party/class-dump') 102 103FILES = { 104 'tools/isolate_driver.py': None, 105 'third_party/BUILD.gn': None, 106} 107 108ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 109CHROMIUM_CHECKOUT = os.path.join('chromium', 'src') 110LINKS_DB = 'links' 111 112# Version management to make future upgrades/downgrades easier to support. 113SCHEMA_VERSION = 1 114 115 116def query_yes_no(question, default=False): 117 """Ask a yes/no question via raw_input() and return their answer. 118 119 Modified from http://stackoverflow.com/a/3041990. 120 """ 121 prompt = " [%s/%%s]: " 122 prompt = prompt % ('Y' if default is True else 'y') 123 prompt = prompt % ('N' if default is False else 'n') 124 125 if default is None: 126 default = 'INVALID' 127 128 while True: 129 sys.stdout.write(question + prompt) 130 choice = raw_input().lower() 131 if choice == '' and default != 'INVALID': 132 return default 133 134 if 'yes'.startswith(choice): 135 return True 136 elif 'no'.startswith(choice): 137 return False 138 139 print "Please respond with 'yes' or 'no' (or 'y' or 'n')." 140 141 142# Actions 143class Action(object): 144 def __init__(self, dangerous): 145 self.dangerous = dangerous 146 147 def announce(self, planning): 148 """Log a description of this action. 149 150 Args: 151 planning - True iff we're in the planning stage, False if we're in the 152 doit stage. 153 """ 154 pass 155 156 def doit(self, links_db): 157 """Execute the action, recording what we did to links_db, if necessary.""" 158 pass 159 160 161class Remove(Action): 162 def __init__(self, path, dangerous): 163 super(Remove, self).__init__(dangerous) 164 self._priority = 0 165 self._path = path 166 167 def announce(self, planning): 168 log = logging.warn 169 filesystem_type = 'file' 170 if not self.dangerous: 171 log = logging.info 172 filesystem_type = 'link' 173 if planning: 174 log('Planning to remove %s: %s', filesystem_type, self._path) 175 else: 176 log('Removing %s: %s', filesystem_type, self._path) 177 178 def doit(self, _): 179 os.remove(self._path) 180 181 182class Rmtree(Action): 183 def __init__(self, path): 184 super(Rmtree, self).__init__(dangerous=True) 185 self._priority = 0 186 self._path = path 187 188 def announce(self, planning): 189 if planning: 190 logging.warn('Planning to remove directory: %s', self._path) 191 else: 192 logging.warn('Removing directory: %s', self._path) 193 194 def doit(self, _): 195 if sys.platform.startswith('win'): 196 # shutil.rmtree() doesn't work on Windows if any of the directories are 197 # read-only, which svn repositories are. 198 subprocess.check_call(['rd', '/q', '/s', self._path], shell=True) 199 else: 200 shutil.rmtree(self._path) 201 202 203class Makedirs(Action): 204 def __init__(self, path): 205 super(Makedirs, self).__init__(dangerous=False) 206 self._priority = 1 207 self._path = path 208 209 def doit(self, _): 210 try: 211 os.makedirs(self._path) 212 except OSError as e: 213 if e.errno != errno.EEXIST: 214 raise 215 216 217class Symlink(Action): 218 def __init__(self, source_path, link_path): 219 super(Symlink, self).__init__(dangerous=False) 220 self._priority = 2 221 self._source_path = source_path 222 self._link_path = link_path 223 224 def announce(self, planning): 225 if planning: 226 logging.info( 227 'Planning to create link from %s to %s', self._link_path, 228 self._source_path) 229 else: 230 logging.debug( 231 'Linking from %s to %s', self._link_path, self._source_path) 232 233 def doit(self, links_db): 234 # Files not in the root directory need relative path calculation. 235 # On Windows, use absolute paths instead since NTFS doesn't seem to support 236 # relative paths for symlinks. 237 if sys.platform.startswith('win'): 238 source_path = os.path.abspath(self._source_path) 239 else: 240 if os.path.dirname(self._link_path) != self._link_path: 241 source_path = os.path.relpath(self._source_path, 242 os.path.dirname(self._link_path)) 243 244 os.symlink(source_path, os.path.abspath(self._link_path)) 245 links_db[self._source_path] = self._link_path 246 247 248class LinkError(IOError): 249 """Failed to create a link.""" 250 pass 251 252 253# Handles symlink creation on the different platforms. 254if sys.platform.startswith('win'): 255 def symlink(source_path, link_path): 256 flag = 1 if os.path.isdir(source_path) else 0 257 if not ctypes.windll.kernel32.CreateSymbolicLinkW( 258 unicode(link_path), unicode(source_path), flag): 259 raise OSError('Failed to create symlink to %s. Notice that only NTFS ' 260 'version 5.0 and up has all the needed APIs for ' 261 'creating symlinks.' % source_path) 262 os.symlink = symlink 263 264 265class WebRTCLinkSetup(object): 266 def __init__(self, links_db, force=False, dry_run=False, prompt=False): 267 self._force = force 268 self._dry_run = dry_run 269 self._prompt = prompt 270 self._links_db = links_db 271 272 def CreateLinks(self, on_bot): 273 logging.debug('CreateLinks') 274 # First, make a plan of action 275 actions = [] 276 277 for source_path, link_path in FILES.iteritems(): 278 actions += self._ActionForPath( 279 source_path, link_path, check_fn=os.path.isfile, check_msg='files') 280 for source_dir in DIRECTORIES: 281 actions += self._ActionForPath( 282 source_dir, None, check_fn=os.path.isdir, 283 check_msg='directories') 284 285 if not on_bot and self._force: 286 # When making the manual switch from legacy SVN checkouts to the new 287 # Git-based Chromium DEPS, the .gclient_entries file that contains cached 288 # URLs for all DEPS entries must be removed to avoid future sync problems. 289 entries_file = os.path.join(os.path.dirname(ROOT_DIR), '.gclient_entries') 290 if os.path.exists(entries_file): 291 actions.append(Remove(entries_file, dangerous=True)) 292 293 actions.sort() 294 295 if self._dry_run: 296 for action in actions: 297 action.announce(planning=True) 298 logging.info('Not doing anything because dry-run was specified.') 299 sys.exit(0) 300 301 if any(a.dangerous for a in actions): 302 logging.warn('Dangerous actions:') 303 for action in (a for a in actions if a.dangerous): 304 action.announce(planning=True) 305 print 306 307 if not self._force: 308 logging.error(textwrap.dedent("""\ 309 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 310 A C T I O N R E Q I R E D 311 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 312 313 Because chromium/src is transitioning to Git (from SVN), we needed to 314 change the way that the WebRTC standalone checkout works. Instead of 315 individually syncing subdirectories of Chromium in SVN, we're now 316 syncing Chromium (and all of its DEPS, as defined by its own DEPS file), 317 into the `chromium/src` directory. 318 319 As such, all Chromium directories which are currently pulled by DEPS are 320 now replaced with a symlink into the full Chromium checkout. 321 322 To avoid disrupting developers, we've chosen to not delete your 323 directories forcibly, in case you have some work in progress in one of 324 them :). 325 326 ACTION REQUIRED: 327 Before running `gclient sync|runhooks` again, you must run: 328 %s%s --force 329 330 Which will replace all directories which now must be symlinks, after 331 prompting with a summary of the work-to-be-done. 332 """), 'python ' if sys.platform.startswith('win') else '', sys.argv[0]) 333 sys.exit(1) 334 elif self._prompt: 335 if not query_yes_no('Would you like to perform the above plan?'): 336 sys.exit(1) 337 338 for action in actions: 339 action.announce(planning=False) 340 action.doit(self._links_db) 341 342 if not on_bot and self._force: 343 logging.info('Completed!\n\nNow run `gclient sync|runhooks` again to ' 344 'let the remaining hooks (that probably were interrupted) ' 345 'execute.') 346 347 def CleanupLinks(self): 348 logging.debug('CleanupLinks') 349 for source, link_path in self._links_db.iteritems(): 350 if source == 'SCHEMA_VERSION': 351 continue 352 if os.path.islink(link_path) or sys.platform.startswith('win'): 353 # os.path.islink() always returns false on Windows 354 # See http://bugs.python.org/issue13143. 355 logging.debug('Removing link to %s at %s', source, link_path) 356 if not self._dry_run: 357 if os.path.exists(link_path): 358 if sys.platform.startswith('win') and os.path.isdir(link_path): 359 subprocess.check_call(['rmdir', '/q', '/s', link_path], 360 shell=True) 361 else: 362 os.remove(link_path) 363 del self._links_db[source] 364 365 @staticmethod 366 def _ActionForPath(source_path, link_path=None, check_fn=None, 367 check_msg=None): 368 """Create zero or more Actions to link to a file or directory. 369 370 This will be a symlink on POSIX platforms. On Windows this requires 371 that NTFS is version 5.0 or higher (Vista or newer). 372 373 Args: 374 source_path: Path relative to the Chromium checkout root. 375 For readability, the path may contain slashes, which will 376 automatically be converted to the right path delimiter on Windows. 377 link_path: The location for the link to create. If omitted it will be the 378 same path as source_path. 379 check_fn: A function returning true if the type of filesystem object is 380 correct for the attempted call. Otherwise an error message with 381 check_msg will be printed. 382 check_msg: String used to inform the user of an invalid attempt to create 383 a file. 384 Returns: 385 A list of Action objects. 386 """ 387 def fix_separators(path): 388 if sys.platform.startswith('win'): 389 return path.replace(os.altsep, os.sep) 390 else: 391 return path 392 393 assert check_fn 394 assert check_msg 395 link_path = link_path or source_path 396 link_path = fix_separators(link_path) 397 398 source_path = fix_separators(source_path) 399 source_path = os.path.join(CHROMIUM_CHECKOUT, source_path) 400 if os.path.exists(source_path) and not check_fn: 401 raise LinkError('_LinkChromiumPath can only be used to link to %s: ' 402 'Tried to link to: %s' % (check_msg, source_path)) 403 404 if not os.path.exists(source_path): 405 logging.debug('Silently ignoring missing source: %s. This is to avoid ' 406 'errors on platform-specific dependencies.', source_path) 407 return [] 408 409 actions = [] 410 411 if os.path.exists(link_path) or os.path.islink(link_path): 412 if os.path.islink(link_path): 413 actions.append(Remove(link_path, dangerous=False)) 414 elif os.path.isfile(link_path): 415 actions.append(Remove(link_path, dangerous=True)) 416 elif os.path.isdir(link_path): 417 actions.append(Rmtree(link_path)) 418 else: 419 raise LinkError('Don\'t know how to plan: %s' % link_path) 420 421 # Create parent directories to the target link if needed. 422 target_parent_dirs = os.path.dirname(link_path) 423 if (target_parent_dirs and 424 target_parent_dirs != link_path and 425 not os.path.exists(target_parent_dirs)): 426 actions.append(Makedirs(target_parent_dirs)) 427 428 actions.append(Symlink(source_path, link_path)) 429 430 return actions 431 432def _initialize_database(filename): 433 links_database = shelve.open(filename) 434 435 # Wipe the database if this version of the script ends up looking at a 436 # newer (future) version of the links db, just to be sure. 437 version = links_database.get('SCHEMA_VERSION') 438 if version and version != SCHEMA_VERSION: 439 logging.info('Found database with schema version %s while this script only ' 440 'supports %s. Wiping previous database contents.', version, 441 SCHEMA_VERSION) 442 links_database.clear() 443 links_database['SCHEMA_VERSION'] = SCHEMA_VERSION 444 return links_database 445 446 447def main(): 448 on_bot = os.environ.get('CHROME_HEADLESS') == '1' 449 450 parser = optparse.OptionParser() 451 parser.add_option('-d', '--dry-run', action='store_true', default=False, 452 help='Print what would be done, but don\'t perform any ' 453 'operations. This will automatically set logging to ' 454 'verbose.') 455 parser.add_option('-c', '--clean-only', action='store_true', default=False, 456 help='Only clean previously created links, don\'t create ' 457 'new ones. This will automatically set logging to ' 458 'verbose.') 459 parser.add_option('-f', '--force', action='store_true', default=on_bot, 460 help='Force link creation. CAUTION: This deletes existing ' 461 'folders and files in the locations where links are ' 462 'about to be created.') 463 parser.add_option('-n', '--no-prompt', action='store_false', dest='prompt', 464 default=(not on_bot), 465 help='Prompt if we\'re planning to do a dangerous action') 466 parser.add_option('-v', '--verbose', action='store_const', 467 const=logging.DEBUG, default=logging.INFO, 468 help='Print verbose output for debugging.') 469 options, _ = parser.parse_args() 470 471 if options.dry_run or options.force or options.clean_only: 472 options.verbose = logging.DEBUG 473 logging.basicConfig(format='%(message)s', level=options.verbose) 474 475 # Work from the root directory of the checkout. 476 script_dir = os.path.dirname(os.path.abspath(__file__)) 477 os.chdir(script_dir) 478 479 if sys.platform.startswith('win'): 480 def is_admin(): 481 try: 482 return os.getuid() == 0 483 except AttributeError: 484 return ctypes.windll.shell32.IsUserAnAdmin() != 0 485 if not is_admin(): 486 logging.error('On Windows, you now need to have administrator ' 487 'privileges for the shell running %s (or ' 488 '`gclient sync|runhooks`).\nPlease start another command ' 489 'prompt as Administrator and try again.', sys.argv[0]) 490 return 1 491 492 if not os.path.exists(CHROMIUM_CHECKOUT): 493 logging.error('Cannot find a Chromium checkout at %s. Did you run "gclient ' 494 'sync" before running this script?', CHROMIUM_CHECKOUT) 495 return 2 496 497 links_database = _initialize_database(LINKS_DB) 498 try: 499 symlink_creator = WebRTCLinkSetup(links_database, options.force, 500 options.dry_run, options.prompt) 501 symlink_creator.CleanupLinks() 502 if not options.clean_only: 503 symlink_creator.CreateLinks(on_bot) 504 except LinkError as e: 505 print >> sys.stderr, e.message 506 return 3 507 finally: 508 links_database.close() 509 return 0 510 511 512if __name__ == '__main__': 513 sys.exit(main()) 514