1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Meta checkout dependency manager for Git.""" 7# Files 8# .gclient : Current client configuration, written by 'config' command. 9# Format is a Python script defining 'solutions', a list whose 10# entries each are maps binding the strings "name" and "url" 11# to strings specifying the name and location of the client 12# module, as well as "custom_deps" to a map similar to the 13# deps section of the DEPS file below, as well as 14# "custom_hooks" to a list similar to the hooks sections of 15# the DEPS file below. 16# .gclient_entries : A cache constructed by 'update' command. Format is a 17# Python script defining 'entries', a list of the names 18# of all modules in the client 19# <module>/DEPS : Python script defining var 'deps' as a map from each 20# requisite submodule name to a URL where it can be found (via 21# one SCM) 22# 23# Hooks 24# .gclient and DEPS files may optionally contain a list named "hooks" to 25# allow custom actions to be performed based on files that have changed in the 26# working copy as a result of a "sync"/"update" or "revert" operation. This 27# can be prevented by using --nohooks (hooks run by default). Hooks can also 28# be forced to run with the "runhooks" operation. If "sync" is run with 29# --force, all known but not suppressed hooks will run regardless of the state 30# of the working copy. 31# 32# Each item in a "hooks" list is a dict, containing these two keys: 33# "pattern" The associated value is a string containing a regular 34# expression. When a file whose pathname matches the expression 35# is checked out, updated, or reverted, the hook's "action" will 36# run. 37# "action" A list describing a command to run along with its arguments, if 38# any. An action command will run at most one time per gclient 39# invocation, regardless of how many files matched the pattern. 40# The action is executed in the same directory as the .gclient 41# file. If the first item in the list is the string "python", 42# the current Python interpreter (sys.executable) will be used 43# to run the command. If the list contains string 44# "$matching_files" it will be removed from the list and the list 45# will be extended by the list of matching files. 46# "name" An optional string specifying the group to which a hook belongs 47# for overriding and organizing. 48# 49# Example: 50# hooks = [ 51# { "pattern": "\\.(gif|jpe?g|pr0n|png)$", 52# "action": ["python", "image_indexer.py", "--all"]}, 53# { "pattern": ".", 54# "name": "gyp", 55# "action": ["python", "src/build/gyp_chromium"]}, 56# ] 57# 58# Pre-DEPS Hooks 59# DEPS files may optionally contain a list named "pre_deps_hooks". These are 60# the same as normal hooks, except that they run before the DEPS are 61# processed. Pre-DEPS run with "sync" and "revert" unless the --noprehooks 62# flag is used. 63# 64# Specifying a target OS 65# An optional key named "target_os" may be added to a gclient file to specify 66# one or more additional operating systems that should be considered when 67# processing the deps_os/hooks_os dict of a DEPS file. 68# 69# Example: 70# target_os = [ "android" ] 71# 72# If the "target_os_only" key is also present and true, then *only* the 73# operating systems listed in "target_os" will be used. 74# 75# Example: 76# target_os = [ "ios" ] 77# target_os_only = True 78# 79# Specifying a target CPU 80# To specify a target CPU, the variables target_cpu and target_cpu_only 81# are available and are analogous to target_os and target_os_only. 82 83from __future__ import print_function 84 85__version__ = '0.7' 86 87import collections 88import copy 89import json 90import logging 91import optparse 92import os 93import platform 94import posixpath 95import pprint 96import re 97import sys 98import time 99 100try: 101 import urlparse 102except ImportError: # For Py3 compatibility 103 import urllib.parse as urlparse 104 105import detect_host_arch 106import fix_encoding 107import gclient_eval 108import gclient_scm 109import gclient_paths 110import gclient_utils 111import git_cache 112import metrics 113import metrics_utils 114from third_party.repo.progress import Progress 115import subcommand 116import subprocess2 117import setup_color 118 119from third_party import six 120 121# TODO(crbug.com/953884): Remove this when python3 migration is done. 122if six.PY3: 123 # pylint: disable=redefined-builtin 124 basestring = str 125 126 127DEPOT_TOOLS_DIR = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) 128 129# Singleton object to represent an unset cache_dir (as opposed to a disabled 130# one, e.g. if a spec explicitly says `cache_dir = None`.) 131UNSET_CACHE_DIR = object() 132 133 134class GNException(Exception): 135 pass 136 137 138def ToGNString(value, allow_dicts = True): 139 """Returns a stringified GN equivalent of the Python value. 140 141 allow_dicts indicates if this function will allow converting dictionaries 142 to GN scopes. This is only possible at the top level, you can't nest a 143 GN scope in a list, so this should be set to False for recursive calls.""" 144 if isinstance(value, basestring): 145 if value.find('\n') >= 0: 146 raise GNException("Trying to print a string with a newline in it.") 147 return '"' + \ 148 value.replace('\\', '\\\\').replace('"', '\\"').replace('$', '\\$') + \ 149 '"' 150 151 if sys.version_info.major == 2 and isinstance(value, unicode): 152 return ToGNString(value.encode('utf-8')) 153 154 if isinstance(value, bool): 155 if value: 156 return "true" 157 return "false" 158 159 # NOTE: some type handling removed compared to chromium/src copy. 160 161 raise GNException("Unsupported type when printing to GN.") 162 163 164class Hook(object): 165 """Descriptor of command ran before/after sync or on demand.""" 166 167 def __init__(self, action, pattern=None, name=None, cwd=None, condition=None, 168 variables=None, verbose=False, cwd_base=None): 169 """Constructor. 170 171 Arguments: 172 action (list of basestring): argv of the command to run 173 pattern (basestring regex): noop with git; deprecated 174 name (basestring): optional name; no effect on operation 175 cwd (basestring): working directory to use 176 condition (basestring): condition when to run the hook 177 variables (dict): variables for evaluating the condition 178 """ 179 self._action = gclient_utils.freeze(action) 180 self._pattern = pattern 181 self._name = name 182 self._cwd = cwd 183 self._condition = condition 184 self._variables = variables 185 self._verbose = verbose 186 self._cwd_base = cwd_base 187 188 @staticmethod 189 def from_dict(d, variables=None, verbose=False, conditions=None, 190 cwd_base=None): 191 """Creates a Hook instance from a dict like in the DEPS file.""" 192 # Merge any local and inherited conditions. 193 gclient_eval.UpdateCondition(d, 'and', conditions) 194 return Hook( 195 d['action'], 196 d.get('pattern'), 197 d.get('name'), 198 d.get('cwd'), 199 d.get('condition'), 200 variables=variables, 201 # Always print the header if not printing to a TTY. 202 verbose=verbose or not setup_color.IS_TTY, 203 cwd_base=cwd_base) 204 205 @property 206 def action(self): 207 return self._action 208 209 @property 210 def pattern(self): 211 return self._pattern 212 213 @property 214 def name(self): 215 return self._name 216 217 @property 218 def condition(self): 219 return self._condition 220 221 @property 222 def effective_cwd(self): 223 cwd = self._cwd_base 224 if self._cwd: 225 cwd = os.path.join(cwd, self._cwd) 226 return cwd 227 228 def matches(self, file_list): 229 """Returns true if the pattern matches any of files in the list.""" 230 if not self._pattern: 231 return True 232 pattern = re.compile(self._pattern) 233 return bool([f for f in file_list if pattern.search(f)]) 234 235 def run(self): 236 """Executes the hook's command (provided the condition is met).""" 237 if (self._condition and 238 not gclient_eval.EvaluateCondition(self._condition, self._variables)): 239 return 240 241 cmd = [arg for arg in self._action] 242 243 if cmd[0] == 'python': 244 cmd[0] = 'vpython' 245 if cmd[0] == 'vpython' and _detect_host_os() == 'win': 246 cmd[0] += '.bat' 247 248 try: 249 start_time = time.time() 250 gclient_utils.CheckCallAndFilter( 251 cmd, cwd=self.effective_cwd, print_stdout=True, show_header=True, 252 always_show_header=self._verbose) 253 except (gclient_utils.Error, subprocess2.CalledProcessError) as e: 254 # Use a discrete exit status code of 2 to indicate that a hook action 255 # failed. Users of this script may wish to treat hook action failures 256 # differently from VC failures. 257 print('Error: %s' % str(e), file=sys.stderr) 258 sys.exit(2) 259 finally: 260 elapsed_time = time.time() - start_time 261 if elapsed_time > 10: 262 print("Hook '%s' took %.2f secs" % ( 263 gclient_utils.CommandToStr(cmd), elapsed_time)) 264 265 266class DependencySettings(object): 267 """Immutable configuration settings.""" 268 def __init__( 269 self, parent, url, managed, custom_deps, custom_vars, 270 custom_hooks, deps_file, should_process, relative, condition): 271 # These are not mutable: 272 self._parent = parent 273 self._deps_file = deps_file 274 self._url = url 275 # The condition as string (or None). Useful to keep e.g. for flatten. 276 self._condition = condition 277 # 'managed' determines whether or not this dependency is synced/updated by 278 # gclient after gclient checks it out initially. The difference between 279 # 'managed' and 'should_process' is that the user specifies 'managed' via 280 # the --unmanaged command-line flag or a .gclient config, where 281 # 'should_process' is dynamically set by gclient if it goes over its 282 # recursion limit and controls gclient's behavior so it does not misbehave. 283 self._managed = managed 284 self._should_process = should_process 285 # If this is a recursed-upon sub-dependency, and the parent has 286 # use_relative_paths set, then this dependency should check out its own 287 # dependencies relative to that parent's path for this, rather than 288 # relative to the .gclient file. 289 self._relative = relative 290 # This is a mutable value which has the list of 'target_os' OSes listed in 291 # the current deps file. 292 self.local_target_os = None 293 294 # These are only set in .gclient and not in DEPS files. 295 self._custom_vars = custom_vars or {} 296 self._custom_deps = custom_deps or {} 297 self._custom_hooks = custom_hooks or [] 298 299 # Post process the url to remove trailing slashes. 300 if isinstance(self.url, basestring): 301 # urls are sometime incorrectly written as proto://host/path/@rev. Replace 302 # it to proto://host/path@rev. 303 self.set_url(self.url.replace('/@', '@')) 304 elif not isinstance(self.url, (None.__class__)): 305 raise gclient_utils.Error( 306 ('dependency url must be either string or None, ' 307 'instead of %s') % self.url.__class__.__name__) 308 309 # Make any deps_file path platform-appropriate. 310 if self._deps_file: 311 for sep in ['/', '\\']: 312 self._deps_file = self._deps_file.replace(sep, os.sep) 313 314 @property 315 def deps_file(self): 316 return self._deps_file 317 318 @property 319 def managed(self): 320 return self._managed 321 322 @property 323 def parent(self): 324 return self._parent 325 326 @property 327 def root(self): 328 """Returns the root node, a GClient object.""" 329 if not self.parent: 330 # This line is to signal pylint that it could be a GClient instance. 331 return self or GClient(None, None) 332 return self.parent.root 333 334 @property 335 def should_process(self): 336 """True if this dependency should be processed, i.e. checked out.""" 337 return self._should_process 338 339 @property 340 def custom_vars(self): 341 return self._custom_vars.copy() 342 343 @property 344 def custom_deps(self): 345 return self._custom_deps.copy() 346 347 @property 348 def custom_hooks(self): 349 return self._custom_hooks[:] 350 351 @property 352 def url(self): 353 """URL after variable expansion.""" 354 return self._url 355 356 @property 357 def condition(self): 358 return self._condition 359 360 @property 361 def target_os(self): 362 if self.local_target_os is not None: 363 return tuple(set(self.local_target_os).union(self.parent.target_os)) 364 else: 365 return self.parent.target_os 366 367 @property 368 def target_cpu(self): 369 return self.parent.target_cpu 370 371 def set_url(self, url): 372 self._url = url 373 374 def get_custom_deps(self, name, url): 375 """Returns a custom deps if applicable.""" 376 if self.parent: 377 url = self.parent.get_custom_deps(name, url) 378 # None is a valid return value to disable a dependency. 379 return self.custom_deps.get(name, url) 380 381 382class Dependency(gclient_utils.WorkItem, DependencySettings): 383 """Object that represents a dependency checkout.""" 384 385 def __init__(self, parent, name, url, managed, custom_deps, 386 custom_vars, custom_hooks, deps_file, should_process, 387 should_recurse, relative, condition, print_outbuf=False): 388 gclient_utils.WorkItem.__init__(self, name) 389 DependencySettings.__init__( 390 self, parent, url, managed, custom_deps, custom_vars, 391 custom_hooks, deps_file, should_process, relative, condition) 392 393 # This is in both .gclient and DEPS files: 394 self._deps_hooks = [] 395 396 self._pre_deps_hooks = [] 397 398 # Calculates properties: 399 self._dependencies = [] 400 self._vars = {} 401 402 # A cache of the files affected by the current operation, necessary for 403 # hooks. 404 self._file_list = [] 405 # List of host names from which dependencies are allowed. 406 # Default is an empty set, meaning unspecified in DEPS file, and hence all 407 # hosts will be allowed. Non-empty set means whitelist of hosts. 408 # allowed_hosts var is scoped to its DEPS file, and so it isn't recursive. 409 self._allowed_hosts = frozenset() 410 self._gn_args_from = None 411 # Spec for .gni output to write (if any). 412 self._gn_args_file = None 413 self._gn_args = [] 414 # If it is not set to True, the dependency wasn't processed for its child 415 # dependency, i.e. its DEPS wasn't read. 416 self._deps_parsed = False 417 # This dependency has been processed, i.e. checked out 418 self._processed = False 419 # This dependency had its pre-DEPS hooks run 420 self._pre_deps_hooks_ran = False 421 # This dependency had its hook run 422 self._hooks_ran = False 423 # This is the scm used to checkout self.url. It may be used by dependencies 424 # to get the datetime of the revision we checked out. 425 self._used_scm = None 426 self._used_revision = None 427 # The actual revision we ended up getting, or None if that information is 428 # unavailable 429 self._got_revision = None 430 # Whether this dependency should use relative paths. 431 self._use_relative_paths = False 432 433 # recursedeps is a mutable value that selectively overrides the default 434 # 'no recursion' setting on a dep-by-dep basis. 435 # 436 # It will be a dictionary of {deps_name: depfile_namee} 437 self.recursedeps = {} 438 439 # Whether we should process this dependency's DEPS file. 440 self._should_recurse = should_recurse 441 442 self._OverrideUrl() 443 # This is inherited from WorkItem. We want the URL to be a resource. 444 if self.url and isinstance(self.url, basestring): 445 # The url is usually given to gclient either as https://blah@123 446 # or just https://blah. The @123 portion is irrelevant. 447 self.resources.append(self.url.split('@')[0]) 448 449 # Controls whether we want to print git's output when we first clone the 450 # dependency 451 self.print_outbuf = print_outbuf 452 453 if not self.name and self.parent: 454 raise gclient_utils.Error('Dependency without name') 455 456 def _OverrideUrl(self): 457 """Resolves the parsed url from the parent hierarchy.""" 458 parsed_url = self.get_custom_deps(self._name, self.url) 459 if parsed_url != self.url: 460 logging.info('Dependency(%s)._OverrideUrl(%s) -> %s', self._name, 461 self.url, parsed_url) 462 self.set_url(parsed_url) 463 return 464 465 if self.url is None: 466 logging.info('Dependency(%s)._OverrideUrl(None) -> None', self._name) 467 return 468 469 if not isinstance(self.url, basestring): 470 raise gclient_utils.Error('Unknown url type') 471 472 # self.url is a local path 473 path, at, rev = self.url.partition('@') 474 if os.path.isdir(path): 475 return 476 477 # self.url is a URL 478 parsed_url = urlparse.urlparse(self.url) 479 if parsed_url[0] or re.match(r'^\w+\@[\w\.-]+\:[\w\/]+', parsed_url[2]): 480 return 481 482 # self.url is relative to the parent's URL. 483 if not path.startswith('/'): 484 raise gclient_utils.Error( 485 'relative DEPS entry \'%s\' must begin with a slash' % self.url) 486 487 parent_url = self.parent.url 488 parent_path = self.parent.url.split('@')[0] 489 if os.path.isdir(parent_path): 490 # Parent's URL is a local path. Get parent's URL dirname and append 491 # self.url. 492 parent_path = os.path.dirname(parent_path) 493 parsed_url = parent_path + path.replace('/', os.sep) + at + rev 494 else: 495 # Parent's URL is a URL. Get parent's URL, strip from the last '/' 496 # (equivalent to unix dirname) and append self.url. 497 parsed_url = parent_url[:parent_url.rfind('/')] + self.url 498 499 logging.info('Dependency(%s)._OverrideUrl(%s) -> %s', self.name, 500 self.url, parsed_url) 501 self.set_url(parsed_url) 502 503 def PinToActualRevision(self): 504 """Updates self.url to the revision checked out on disk.""" 505 if self.url is None: 506 return 507 url = None 508 scm = self.CreateSCM() 509 if os.path.isdir(scm.checkout_path): 510 revision = scm.revinfo(None, None, None) 511 url = '%s@%s' % (gclient_utils.SplitUrlRevision(self.url)[0], revision) 512 self.set_url(url) 513 514 def ToLines(self): 515 s = [] 516 condition_part = ([' "condition": %r,' % self.condition] 517 if self.condition else []) 518 s.extend([ 519 ' # %s' % self.hierarchy(include_url=False), 520 ' "%s": {' % (self.name,), 521 ' "url": "%s",' % (self.url,), 522 ] + condition_part + [ 523 ' },', 524 '', 525 ]) 526 return s 527 528 @property 529 def requirements(self): 530 """Calculate the list of requirements.""" 531 requirements = set() 532 # self.parent is implicitly a requirement. This will be recursive by 533 # definition. 534 if self.parent and self.parent.name: 535 requirements.add(self.parent.name) 536 537 # For a tree with at least 2 levels*, the leaf node needs to depend 538 # on the level higher up in an orderly way. 539 # This becomes messy for >2 depth as the DEPS file format is a dictionary, 540 # thus unsorted, while the .gclient format is a list thus sorted. 541 # 542 # Interestingly enough, the following condition only works in the case we 543 # want: self is a 2nd level node. 3rd level node wouldn't need this since 544 # they already have their parent as a requirement. 545 if self.parent and self.parent.parent and not self.parent.parent.parent: 546 requirements |= set(i.name for i in self.root.dependencies if i.name) 547 548 if self.name: 549 requirements |= set( 550 obj.name for obj in self.root.subtree(False) 551 if (obj is not self 552 and obj.name and 553 self.name.startswith(posixpath.join(obj.name, '')))) 554 requirements = tuple(sorted(requirements)) 555 logging.info('Dependency(%s).requirements = %s' % (self.name, requirements)) 556 return requirements 557 558 @property 559 def should_recurse(self): 560 return self._should_recurse 561 562 def verify_validity(self): 563 """Verifies that this Dependency is fine to add as a child of another one. 564 565 Returns True if this entry should be added, False if it is a duplicate of 566 another entry. 567 """ 568 logging.info('Dependency(%s).verify_validity()' % self.name) 569 if self.name in [s.name for s in self.parent.dependencies]: 570 raise gclient_utils.Error( 571 'The same name "%s" appears multiple times in the deps section' % 572 self.name) 573 if not self.should_process: 574 # Return early, no need to set requirements. 575 return not any(d.name == self.name for d in self.root.subtree(True)) 576 577 # This require a full tree traversal with locks. 578 siblings = [d for d in self.root.subtree(False) if d.name == self.name] 579 for sibling in siblings: 580 # Allow to have only one to be None or ''. 581 if self.url != sibling.url and bool(self.url) == bool(sibling.url): 582 raise gclient_utils.Error( 583 ('Dependency %s specified more than once:\n' 584 ' %s [%s]\n' 585 'vs\n' 586 ' %s [%s]') % ( 587 self.name, 588 sibling.hierarchy(), 589 sibling.url, 590 self.hierarchy(), 591 self.url)) 592 # In theory we could keep it as a shadow of the other one. In 593 # practice, simply ignore it. 594 logging.warning("Won't process duplicate dependency %s" % sibling) 595 return False 596 return True 597 598 def _postprocess_deps(self, deps, rel_prefix): 599 """Performs post-processing of deps compared to what's in the DEPS file.""" 600 # Make sure the dict is mutable, e.g. in case it's frozen. 601 deps = dict(deps) 602 603 # If a line is in custom_deps, but not in the solution, we want to append 604 # this line to the solution. 605 for dep_name, dep_info in self.custom_deps.items(): 606 if dep_name not in deps: 607 deps[dep_name] = {'url': dep_info, 'dep_type': 'git'} 608 609 # Make child deps conditional on any parent conditions. This ensures that, 610 # when flattened, recursed entries have the correct restrictions, even if 611 # not explicitly set in the recursed DEPS file. For instance, if 612 # "src/ios_foo" is conditional on "checkout_ios=True", then anything 613 # recursively included by "src/ios_foo/DEPS" should also require 614 # "checkout_ios=True". 615 if self.condition: 616 for value in deps.values(): 617 gclient_eval.UpdateCondition(value, 'and', self.condition) 618 619 if rel_prefix: 620 logging.warning('use_relative_paths enabled.') 621 rel_deps = {} 622 for d, url in deps.items(): 623 # normpath is required to allow DEPS to use .. in their 624 # dependency local path. 625 rel_deps[os.path.normpath(os.path.join(rel_prefix, d))] = url 626 logging.warning('Updating deps by prepending %s.', rel_prefix) 627 deps = rel_deps 628 629 return deps 630 631 def _deps_to_objects(self, deps, use_relative_paths): 632 """Convert a deps dict to a dict of Dependency objects.""" 633 deps_to_add = [] 634 for name, dep_value in deps.items(): 635 should_process = self.should_process 636 if dep_value is None: 637 continue 638 639 condition = dep_value.get('condition') 640 dep_type = dep_value.get('dep_type') 641 642 if condition and not self._get_option('process_all_deps', False): 643 should_process = should_process and gclient_eval.EvaluateCondition( 644 condition, self.get_vars()) 645 646 # The following option is only set by the 'revinfo' command. 647 if self._get_option('ignore_dep_type', None) == dep_type: 648 continue 649 650 if dep_type == 'cipd': 651 cipd_root = self.GetCipdRoot() 652 for package in dep_value.get('packages', []): 653 deps_to_add.append( 654 CipdDependency( 655 parent=self, 656 name=name, 657 dep_value=package, 658 cipd_root=cipd_root, 659 custom_vars=self.custom_vars, 660 should_process=should_process, 661 relative=use_relative_paths, 662 condition=condition)) 663 else: 664 url = dep_value.get('url') 665 deps_to_add.append( 666 GitDependency( 667 parent=self, 668 name=name, 669 url=url, 670 managed=True, 671 custom_deps=None, 672 custom_vars=self.custom_vars, 673 custom_hooks=None, 674 deps_file=self.recursedeps.get(name, self.deps_file), 675 should_process=should_process, 676 should_recurse=name in self.recursedeps, 677 relative=use_relative_paths, 678 condition=condition)) 679 680 deps_to_add.sort(key=lambda x: x.name) 681 return deps_to_add 682 683 def ParseDepsFile(self): 684 """Parses the DEPS file for this dependency.""" 685 assert not self.deps_parsed 686 assert not self.dependencies 687 688 deps_content = None 689 690 # First try to locate the configured deps file. If it's missing, fallback 691 # to DEPS. 692 deps_files = [self.deps_file] 693 if 'DEPS' not in deps_files: 694 deps_files.append('DEPS') 695 for deps_file in deps_files: 696 filepath = os.path.join(self.root.root_dir, self.name, deps_file) 697 if os.path.isfile(filepath): 698 logging.info( 699 'ParseDepsFile(%s): %s file found at %s', self.name, deps_file, 700 filepath) 701 break 702 logging.info( 703 'ParseDepsFile(%s): No %s file found at %s', self.name, deps_file, 704 filepath) 705 706 if os.path.isfile(filepath): 707 deps_content = gclient_utils.FileRead(filepath) 708 logging.debug('ParseDepsFile(%s) read:\n%s', self.name, deps_content) 709 710 local_scope = {} 711 if deps_content: 712 try: 713 local_scope = gclient_eval.Parse( 714 deps_content, filepath, self.get_vars(), self.get_builtin_vars()) 715 except SyntaxError as e: 716 gclient_utils.SyntaxErrorToError(filepath, e) 717 718 if 'allowed_hosts' in local_scope: 719 try: 720 self._allowed_hosts = frozenset(local_scope.get('allowed_hosts')) 721 except TypeError: # raised if non-iterable 722 pass 723 if not self._allowed_hosts: 724 logging.warning("allowed_hosts is specified but empty %s", 725 self._allowed_hosts) 726 raise gclient_utils.Error( 727 'ParseDepsFile(%s): allowed_hosts must be absent ' 728 'or a non-empty iterable' % self.name) 729 730 self._gn_args_from = local_scope.get('gclient_gn_args_from') 731 self._gn_args_file = local_scope.get('gclient_gn_args_file') 732 self._gn_args = local_scope.get('gclient_gn_args', []) 733 # It doesn't make sense to set all of these, since setting gn_args_from to 734 # another DEPS will make gclient ignore any other local gn_args* settings. 735 assert not (self._gn_args_from and self._gn_args_file), \ 736 'Only specify one of "gclient_gn_args_from" or ' \ 737 '"gclient_gn_args_file + gclient_gn_args".' 738 739 self._vars = local_scope.get('vars', {}) 740 if self.parent: 741 for key, value in self.parent.get_vars().items(): 742 if key in self._vars: 743 self._vars[key] = value 744 # Since we heavily post-process things, freeze ones which should 745 # reflect original state of DEPS. 746 self._vars = gclient_utils.freeze(self._vars) 747 748 # If use_relative_paths is set in the DEPS file, regenerate 749 # the dictionary using paths relative to the directory containing 750 # the DEPS file. Also update recursedeps if use_relative_paths is 751 # enabled. 752 # If the deps file doesn't set use_relative_paths, but the parent did 753 # (and therefore set self.relative on this Dependency object), then we 754 # want to modify the deps and recursedeps by prepending the parent 755 # directory of this dependency. 756 self._use_relative_paths = local_scope.get('use_relative_paths', False) 757 rel_prefix = None 758 if self._use_relative_paths: 759 rel_prefix = self.name 760 elif self._relative: 761 rel_prefix = os.path.dirname(self.name) 762 763 if 'recursion' in local_scope: 764 logging.warning( 765 '%s: Ignoring recursion = %d.', self.name, local_scope['recursion']) 766 767 if 'recursedeps' in local_scope: 768 for ent in local_scope['recursedeps']: 769 if isinstance(ent, basestring): 770 self.recursedeps[ent] = self.deps_file 771 else: # (depname, depsfilename) 772 self.recursedeps[ent[0]] = ent[1] 773 logging.warning('Found recursedeps %r.', repr(self.recursedeps)) 774 775 if rel_prefix: 776 logging.warning('Updating recursedeps by prepending %s.', rel_prefix) 777 rel_deps = {} 778 for depname, options in self.recursedeps.items(): 779 rel_deps[ 780 os.path.normpath(os.path.join(rel_prefix, depname))] = options 781 self.recursedeps = rel_deps 782 # To get gn_args from another DEPS, that DEPS must be recursed into. 783 if self._gn_args_from: 784 assert self.recursedeps and self._gn_args_from in self.recursedeps, \ 785 'The "gclient_gn_args_from" value must be in recursedeps.' 786 787 # If present, save 'target_os' in the local_target_os property. 788 if 'target_os' in local_scope: 789 self.local_target_os = local_scope['target_os'] 790 791 deps = local_scope.get('deps', {}) 792 deps_to_add = self._deps_to_objects( 793 self._postprocess_deps(deps, rel_prefix), self._use_relative_paths) 794 795 # compute which working directory should be used for hooks 796 if local_scope.get('use_relative_hooks', False): 797 print('use_relative_hooks is deprecated, please remove it from DEPS. ' + 798 '(it was merged in use_relative_paths)', file=sys.stderr) 799 800 hooks_cwd = self.root.root_dir 801 if self._use_relative_paths: 802 hooks_cwd = os.path.join(hooks_cwd, self.name) 803 logging.warning('Updating hook base working directory to %s.', 804 hooks_cwd) 805 806 # override named sets of hooks by the custom hooks 807 hooks_to_run = [] 808 hook_names_to_suppress = [c.get('name', '') for c in self.custom_hooks] 809 for hook in local_scope.get('hooks', []): 810 if hook.get('name', '') not in hook_names_to_suppress: 811 hooks_to_run.append(hook) 812 813 # add the replacements and any additions 814 for hook in self.custom_hooks: 815 if 'action' in hook: 816 hooks_to_run.append(hook) 817 818 if self.should_recurse: 819 self._pre_deps_hooks = [ 820 Hook.from_dict(hook, variables=self.get_vars(), verbose=True, 821 conditions=self.condition, cwd_base=hooks_cwd) 822 for hook in local_scope.get('pre_deps_hooks', []) 823 ] 824 825 self.add_dependencies_and_close(deps_to_add, hooks_to_run, 826 hooks_cwd=hooks_cwd) 827 logging.info('ParseDepsFile(%s) done' % self.name) 828 829 def _get_option(self, attr, default): 830 obj = self 831 while not hasattr(obj, '_options'): 832 obj = obj.parent 833 return getattr(obj._options, attr, default) 834 835 def add_dependencies_and_close(self, deps_to_add, hooks, hooks_cwd=None): 836 """Adds the dependencies, hooks and mark the parsing as done.""" 837 if hooks_cwd == None: 838 hooks_cwd = self.root.root_dir 839 840 for dep in deps_to_add: 841 if dep.verify_validity(): 842 self.add_dependency(dep) 843 self._mark_as_parsed([ 844 Hook.from_dict( 845 h, variables=self.get_vars(), verbose=self.root._options.verbose, 846 conditions=self.condition, cwd_base=hooks_cwd) 847 for h in hooks 848 ]) 849 850 def findDepsFromNotAllowedHosts(self): 851 """Returns a list of dependencies from not allowed hosts. 852 853 If allowed_hosts is not set, allows all hosts and returns empty list. 854 """ 855 if not self._allowed_hosts: 856 return [] 857 bad_deps = [] 858 for dep in self._dependencies: 859 # Don't enforce this for custom_deps. 860 if dep.name in self._custom_deps: 861 continue 862 if isinstance(dep.url, basestring): 863 parsed_url = urlparse.urlparse(dep.url) 864 if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts: 865 bad_deps.append(dep) 866 return bad_deps 867 868 def FuzzyMatchUrl(self, candidates): 869 """Attempts to find this dependency in the list of candidates. 870 871 It looks first for the URL of this dependency in the list of 872 candidates. If it doesn't succeed, and the URL ends in '.git', it will try 873 looking for the URL minus '.git'. Finally it will try to look for the name 874 of the dependency. 875 876 Args: 877 candidates: list, dict. The list of candidates in which to look for this 878 dependency. It can contain URLs as above, or dependency names like 879 "src/some/dep". 880 881 Returns: 882 If this dependency is not found in the list of candidates, returns None. 883 Otherwise, it returns under which name did we find this dependency: 884 - Its parsed url: "https://example.com/src.git' 885 - Its parsed url minus '.git': "https://example.com/src" 886 - Its name: "src" 887 """ 888 if self.url: 889 origin, _ = gclient_utils.SplitUrlRevision(self.url) 890 if origin in candidates: 891 return origin 892 if origin.endswith('.git') and origin[:-len('.git')] in candidates: 893 return origin[:-len('.git')] 894 if origin + '.git' in candidates: 895 return origin + '.git' 896 if self.name in candidates: 897 return self.name 898 return None 899 900 # Arguments number differs from overridden method 901 # pylint: disable=arguments-differ 902 def run(self, revision_overrides, command, args, work_queue, options, 903 patch_refs, target_branches): 904 """Runs |command| then parse the DEPS file.""" 905 logging.info('Dependency(%s).run()' % self.name) 906 assert self._file_list == [] 907 if not self.should_process: 908 return 909 # When running runhooks, there's no need to consult the SCM. 910 # All known hooks are expected to run unconditionally regardless of working 911 # copy state, so skip the SCM status check. 912 run_scm = command not in ( 913 'flatten', 'runhooks', 'recurse', 'validate', None) 914 file_list = [] if not options.nohooks else None 915 revision_override = revision_overrides.pop( 916 self.FuzzyMatchUrl(revision_overrides), None) 917 if not revision_override and not self.managed: 918 revision_override = 'unmanaged' 919 if run_scm and self.url: 920 # Create a shallow copy to mutate revision. 921 options = copy.copy(options) 922 options.revision = revision_override 923 self._used_revision = options.revision 924 self._used_scm = self.CreateSCM(out_cb=work_queue.out_cb) 925 self._got_revision = self._used_scm.RunCommand(command, options, args, 926 file_list) 927 928 patch_repo = self.url.split('@')[0] 929 patch_ref = patch_refs.pop(self.FuzzyMatchUrl(patch_refs), None) 930 target_branch = target_branches.pop( 931 self.FuzzyMatchUrl(target_branches), None) 932 if command == 'update' and patch_ref is not None: 933 self._used_scm.apply_patch_ref(patch_repo, patch_ref, target_branch, 934 options, file_list) 935 936 if file_list: 937 file_list = [os.path.join(self.name, f.strip()) for f in file_list] 938 939 # TODO(phajdan.jr): We should know exactly when the paths are absolute. 940 # Convert all absolute paths to relative. 941 for i in range(len(file_list or [])): 942 # It depends on the command being executed (like runhooks vs sync). 943 if not os.path.isabs(file_list[i]): 944 continue 945 prefix = os.path.commonprefix( 946 [self.root.root_dir.lower(), file_list[i].lower()]) 947 file_list[i] = file_list[i][len(prefix):] 948 # Strip any leading path separators. 949 while file_list[i].startswith(('\\', '/')): 950 file_list[i] = file_list[i][1:] 951 952 if self.should_recurse: 953 self.ParseDepsFile() 954 955 self._run_is_done(file_list or []) 956 957 if self.should_recurse: 958 if command in ('update', 'revert') and not options.noprehooks: 959 self.RunPreDepsHooks() 960 # Parse the dependencies of this dependency. 961 for s in self.dependencies: 962 if s.should_process: 963 work_queue.enqueue(s) 964 965 if command == 'recurse': 966 # Skip file only checkout. 967 scm = self.GetScmName() 968 if not options.scm or scm in options.scm: 969 cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name)) 970 # Pass in the SCM type as an env variable. Make sure we don't put 971 # unicode strings in the environment. 972 env = os.environ.copy() 973 if scm: 974 env['GCLIENT_SCM'] = str(scm) 975 if self.url: 976 env['GCLIENT_URL'] = str(self.url) 977 env['GCLIENT_DEP_PATH'] = str(self.name) 978 if options.prepend_dir and scm == 'git': 979 print_stdout = False 980 def filter_fn(line): 981 """Git-specific path marshaling. It is optimized for git-grep.""" 982 983 def mod_path(git_pathspec): 984 match = re.match('^(\\S+?:)?([^\0]+)$', git_pathspec) 985 modified_path = os.path.join(self.name, match.group(2)) 986 branch = match.group(1) or '' 987 return '%s%s' % (branch, modified_path) 988 989 match = re.match('^Binary file ([^\0]+) matches$', line) 990 if match: 991 print('Binary file %s matches\n' % mod_path(match.group(1))) 992 return 993 994 items = line.split('\0') 995 if len(items) == 2 and items[1]: 996 print('%s : %s' % (mod_path(items[0]), items[1])) 997 elif len(items) >= 2: 998 # Multiple null bytes or a single trailing null byte indicate 999 # git is likely displaying filenames only (such as with -l) 1000 print('\n'.join(mod_path(path) for path in items if path)) 1001 else: 1002 print(line) 1003 else: 1004 print_stdout = True 1005 filter_fn = None 1006 1007 if self.url is None: 1008 print('Skipped omitted dependency %s' % cwd, file=sys.stderr) 1009 elif os.path.isdir(cwd): 1010 try: 1011 gclient_utils.CheckCallAndFilter( 1012 args, cwd=cwd, env=env, print_stdout=print_stdout, 1013 filter_fn=filter_fn, 1014 ) 1015 except subprocess2.CalledProcessError: 1016 if not options.ignore: 1017 raise 1018 else: 1019 print('Skipped missing %s' % cwd, file=sys.stderr) 1020 1021 def GetScmName(self): 1022 raise NotImplementedError() 1023 1024 def CreateSCM(self, out_cb=None): 1025 raise NotImplementedError() 1026 1027 def HasGNArgsFile(self): 1028 return self._gn_args_file is not None 1029 1030 def WriteGNArgsFile(self): 1031 lines = ['# Generated from %r' % self.deps_file] 1032 variables = self.get_vars() 1033 for arg in self._gn_args: 1034 value = variables[arg] 1035 if isinstance(value, gclient_eval.ConstantString): 1036 value = value.value 1037 elif isinstance(value, basestring): 1038 value = gclient_eval.EvaluateCondition(value, variables) 1039 lines.append('%s = %s' % (arg, ToGNString(value))) 1040 1041 # When use_relative_paths is set, gn_args_file is relative to this DEPS 1042 path_prefix = self.root.root_dir 1043 if self._use_relative_paths: 1044 path_prefix = os.path.join(path_prefix, self.name) 1045 1046 with open(os.path.join(path_prefix, self._gn_args_file), 'wb') as f: 1047 f.write('\n'.join(lines).encode('utf-8', 'replace')) 1048 1049 @gclient_utils.lockedmethod 1050 def _run_is_done(self, file_list): 1051 # Both these are kept for hooks that are run as a separate tree traversal. 1052 self._file_list = file_list 1053 self._processed = True 1054 1055 def GetHooks(self, options): 1056 """Evaluates all hooks, and return them in a flat list. 1057 1058 RunOnDeps() must have been called before to load the DEPS. 1059 """ 1060 result = [] 1061 if not self.should_process or not self.should_recurse: 1062 # Don't run the hook when it is above recursion_limit. 1063 return result 1064 # If "--force" was specified, run all hooks regardless of what files have 1065 # changed. 1066 if self.deps_hooks: 1067 # TODO(maruel): If the user is using git, then we don't know 1068 # what files have changed so we always run all hooks. It'd be nice to fix 1069 # that. 1070 result.extend(self.deps_hooks) 1071 for s in self.dependencies: 1072 result.extend(s.GetHooks(options)) 1073 return result 1074 1075 def RunHooksRecursively(self, options, progress): 1076 assert self.hooks_ran == False 1077 self._hooks_ran = True 1078 hooks = self.GetHooks(options) 1079 if progress: 1080 progress._total = len(hooks) 1081 for hook in hooks: 1082 if progress: 1083 progress.update(extra=hook.name or '') 1084 hook.run() 1085 if progress: 1086 progress.end() 1087 1088 def RunPreDepsHooks(self): 1089 assert self.processed 1090 assert self.deps_parsed 1091 assert not self.pre_deps_hooks_ran 1092 assert not self.hooks_ran 1093 for s in self.dependencies: 1094 assert not s.processed 1095 self._pre_deps_hooks_ran = True 1096 for hook in self.pre_deps_hooks: 1097 hook.run() 1098 1099 def GetCipdRoot(self): 1100 if self.root is self: 1101 # Let's not infinitely recurse. If this is root and isn't an 1102 # instance of GClient, do nothing. 1103 return None 1104 return self.root.GetCipdRoot() 1105 1106 def subtree(self, include_all): 1107 """Breadth first recursion excluding root node.""" 1108 dependencies = self.dependencies 1109 for d in dependencies: 1110 if d.should_process or include_all: 1111 yield d 1112 for d in dependencies: 1113 for i in d.subtree(include_all): 1114 yield i 1115 1116 @gclient_utils.lockedmethod 1117 def add_dependency(self, new_dep): 1118 self._dependencies.append(new_dep) 1119 1120 @gclient_utils.lockedmethod 1121 def _mark_as_parsed(self, new_hooks): 1122 self._deps_hooks.extend(new_hooks) 1123 self._deps_parsed = True 1124 1125 @property 1126 @gclient_utils.lockedmethod 1127 def dependencies(self): 1128 return tuple(self._dependencies) 1129 1130 @property 1131 @gclient_utils.lockedmethod 1132 def deps_hooks(self): 1133 return tuple(self._deps_hooks) 1134 1135 @property 1136 @gclient_utils.lockedmethod 1137 def pre_deps_hooks(self): 1138 return tuple(self._pre_deps_hooks) 1139 1140 @property 1141 @gclient_utils.lockedmethod 1142 def deps_parsed(self): 1143 """This is purely for debugging purposes. It's not used anywhere.""" 1144 return self._deps_parsed 1145 1146 @property 1147 @gclient_utils.lockedmethod 1148 def processed(self): 1149 return self._processed 1150 1151 @property 1152 @gclient_utils.lockedmethod 1153 def pre_deps_hooks_ran(self): 1154 return self._pre_deps_hooks_ran 1155 1156 @property 1157 @gclient_utils.lockedmethod 1158 def hooks_ran(self): 1159 return self._hooks_ran 1160 1161 @property 1162 @gclient_utils.lockedmethod 1163 def allowed_hosts(self): 1164 return self._allowed_hosts 1165 1166 @property 1167 @gclient_utils.lockedmethod 1168 def file_list(self): 1169 return tuple(self._file_list) 1170 1171 @property 1172 def used_scm(self): 1173 """SCMWrapper instance for this dependency or None if not processed yet.""" 1174 return self._used_scm 1175 1176 @property 1177 @gclient_utils.lockedmethod 1178 def got_revision(self): 1179 return self._got_revision 1180 1181 @property 1182 def file_list_and_children(self): 1183 result = list(self.file_list) 1184 for d in self.dependencies: 1185 result.extend(d.file_list_and_children) 1186 return tuple(result) 1187 1188 def __str__(self): 1189 out = [] 1190 for i in ('name', 'url', 'custom_deps', 1191 'custom_vars', 'deps_hooks', 'file_list', 'should_process', 1192 'processed', 'hooks_ran', 'deps_parsed', 'requirements', 1193 'allowed_hosts'): 1194 # First try the native property if it exists. 1195 if hasattr(self, '_' + i): 1196 value = getattr(self, '_' + i, False) 1197 else: 1198 value = getattr(self, i, False) 1199 if value: 1200 out.append('%s: %s' % (i, value)) 1201 1202 for d in self.dependencies: 1203 out.extend([' ' + x for x in str(d).splitlines()]) 1204 out.append('') 1205 return '\n'.join(out) 1206 1207 def __repr__(self): 1208 return '%s: %s' % (self.name, self.url) 1209 1210 def hierarchy(self, include_url=True): 1211 """Returns a human-readable hierarchical reference to a Dependency.""" 1212 def format_name(d): 1213 if include_url: 1214 return '%s(%s)' % (d.name, d.url) 1215 return d.name 1216 out = format_name(self) 1217 i = self.parent 1218 while i and i.name: 1219 out = '%s -> %s' % (format_name(i), out) 1220 i = i.parent 1221 return out 1222 1223 def hierarchy_data(self): 1224 """Returns a machine-readable hierarchical reference to a Dependency.""" 1225 d = self 1226 out = [] 1227 while d and d.name: 1228 out.insert(0, (d.name, d.url)) 1229 d = d.parent 1230 return tuple(out) 1231 1232 def get_builtin_vars(self): 1233 return { 1234 'checkout_android': 'android' in self.target_os, 1235 'checkout_chromeos': 'chromeos' in self.target_os, 1236 'checkout_fuchsia': 'fuchsia' in self.target_os, 1237 'checkout_ios': 'ios' in self.target_os, 1238 'checkout_linux': 'unix' in self.target_os, 1239 'checkout_mac': 'mac' in self.target_os, 1240 'checkout_win': 'win' in self.target_os, 1241 'host_os': _detect_host_os(), 1242 1243 'checkout_arm': 'arm' in self.target_cpu, 1244 'checkout_arm64': 'arm64' in self.target_cpu, 1245 'checkout_x86': 'x86' in self.target_cpu, 1246 'checkout_mips': 'mips' in self.target_cpu, 1247 'checkout_mips64': 'mips64' in self.target_cpu, 1248 'checkout_ppc': 'ppc' in self.target_cpu, 1249 'checkout_s390': 's390' in self.target_cpu, 1250 'checkout_x64': 'x64' in self.target_cpu, 1251 'host_cpu': detect_host_arch.HostArch(), 1252 } 1253 1254 def get_vars(self): 1255 """Returns a dictionary of effective variable values 1256 (DEPS file contents with applied custom_vars overrides).""" 1257 # Variable precedence (last has highest): 1258 # - DEPS vars 1259 # - parents, from first to last 1260 # - built-in 1261 # - custom_vars overrides 1262 result = {} 1263 result.update(self._vars) 1264 if self.parent: 1265 merge_vars(result, self.parent.get_vars()) 1266 # Provide some built-in variables. 1267 result.update(self.get_builtin_vars()) 1268 merge_vars(result, self.custom_vars) 1269 1270 return result 1271 1272 1273_PLATFORM_MAPPING = { 1274 'cygwin': 'win', 1275 'darwin': 'mac', 1276 'linux2': 'linux', 1277 'linux': 'linux', 1278 'win32': 'win', 1279 'aix6': 'aix', 1280} 1281 1282 1283def merge_vars(result, new_vars): 1284 for k, v in new_vars.items(): 1285 if k in result: 1286 if isinstance(result[k], gclient_eval.ConstantString): 1287 if isinstance(v, gclient_eval.ConstantString): 1288 result[k] = v 1289 else: 1290 result[k].value = v 1291 else: 1292 result[k] = v 1293 else: 1294 result[k] = v 1295 1296 1297def _detect_host_os(): 1298 return _PLATFORM_MAPPING[sys.platform] 1299 1300 1301class GitDependency(Dependency): 1302 """A Dependency object that represents a single git checkout.""" 1303 1304 #override 1305 def GetScmName(self): 1306 """Always 'git'.""" 1307 return 'git' 1308 1309 #override 1310 def CreateSCM(self, out_cb=None): 1311 """Create a Wrapper instance suitable for handling this git dependency.""" 1312 return gclient_scm.GitWrapper( 1313 self.url, self.root.root_dir, self.name, self.outbuf, out_cb, 1314 print_outbuf=self.print_outbuf) 1315 1316 1317class GClient(GitDependency): 1318 """Object that represent a gclient checkout. A tree of Dependency(), one per 1319 solution or DEPS entry.""" 1320 1321 DEPS_OS_CHOICES = { 1322 "aix6": "unix", 1323 "win32": "win", 1324 "win": "win", 1325 "cygwin": "win", 1326 "darwin": "mac", 1327 "mac": "mac", 1328 "unix": "unix", 1329 "linux": "unix", 1330 "linux2": "unix", 1331 "linux3": "unix", 1332 "android": "android", 1333 "ios": "ios", 1334 "fuchsia": "fuchsia", 1335 "chromeos": "chromeos", 1336 } 1337 1338 DEFAULT_CLIENT_FILE_TEXT = ("""\ 1339solutions = [ 1340 { "name" : %(solution_name)r, 1341 "url" : %(solution_url)r, 1342 "deps_file" : %(deps_file)r, 1343 "managed" : %(managed)r, 1344 "custom_deps" : { 1345 }, 1346 "custom_vars": %(custom_vars)r, 1347 }, 1348] 1349""") 1350 1351 DEFAULT_CLIENT_CACHE_DIR_TEXT = ("""\ 1352cache_dir = %(cache_dir)r 1353""") 1354 1355 1356 DEFAULT_SNAPSHOT_FILE_TEXT = ("""\ 1357# Snapshot generated with gclient revinfo --snapshot 1358solutions = %(solution_list)s 1359""") 1360 1361 def __init__(self, root_dir, options): 1362 # Do not change previous behavior. Only solution level and immediate DEPS 1363 # are processed. 1364 self._recursion_limit = 2 1365 super(GClient, self).__init__( 1366 parent=None, 1367 name=None, 1368 url=None, 1369 managed=True, 1370 custom_deps=None, 1371 custom_vars=None, 1372 custom_hooks=None, 1373 deps_file='unused', 1374 should_process=True, 1375 should_recurse=True, 1376 relative=None, 1377 condition=None, 1378 print_outbuf=True) 1379 1380 self._options = options 1381 if options.deps_os: 1382 enforced_os = options.deps_os.split(',') 1383 else: 1384 enforced_os = [self.DEPS_OS_CHOICES.get(sys.platform, 'unix')] 1385 if 'all' in enforced_os: 1386 enforced_os = self.DEPS_OS_CHOICES.values() 1387 self._enforced_os = tuple(set(enforced_os)) 1388 self._enforced_cpu = detect_host_arch.HostArch(), 1389 self._root_dir = root_dir 1390 self._cipd_root = None 1391 self.config_content = None 1392 1393 def _CheckConfig(self): 1394 """Verify that the config matches the state of the existing checked-out 1395 solutions.""" 1396 for dep in self.dependencies: 1397 if dep.managed and dep.url: 1398 scm = dep.CreateSCM() 1399 actual_url = scm.GetActualRemoteURL(self._options) 1400 if actual_url and not scm.DoesRemoteURLMatch(self._options): 1401 mirror = scm.GetCacheMirror() 1402 if mirror: 1403 mirror_string = '%s (exists=%s)' % (mirror.mirror_path, 1404 mirror.exists()) 1405 else: 1406 mirror_string = 'not used' 1407 raise gclient_utils.Error( 1408 ''' 1409Your .gclient file seems to be broken. The requested URL is different from what 1410is actually checked out in %(checkout_path)s. 1411 1412The .gclient file contains: 1413URL: %(expected_url)s (%(expected_scm)s) 1414Cache mirror: %(mirror_string)s 1415 1416The local checkout in %(checkout_path)s reports: 1417%(actual_url)s (%(actual_scm)s) 1418 1419You should ensure that the URL listed in .gclient is correct and either change 1420it or fix the checkout. 1421''' % { 1422 'checkout_path': os.path.join(self.root_dir, dep.name), 1423 'expected_url': dep.url, 1424 'expected_scm': dep.GetScmName(), 1425 'mirror_string': mirror_string, 1426 'actual_url': actual_url, 1427 'actual_scm': dep.GetScmName() 1428 }) 1429 1430 def SetConfig(self, content): 1431 assert not self.dependencies 1432 config_dict = {} 1433 self.config_content = content 1434 try: 1435 exec(content, config_dict) 1436 except SyntaxError as e: 1437 gclient_utils.SyntaxErrorToError('.gclient', e) 1438 1439 # Append any target OS that is not already being enforced to the tuple. 1440 target_os = config_dict.get('target_os', []) 1441 if config_dict.get('target_os_only', False): 1442 self._enforced_os = tuple(set(target_os)) 1443 else: 1444 self._enforced_os = tuple(set(self._enforced_os).union(target_os)) 1445 1446 # Append any target CPU that is not already being enforced to the tuple. 1447 target_cpu = config_dict.get('target_cpu', []) 1448 if config_dict.get('target_cpu_only', False): 1449 self._enforced_cpu = tuple(set(target_cpu)) 1450 else: 1451 self._enforced_cpu = tuple(set(self._enforced_cpu).union(target_cpu)) 1452 1453 cache_dir = config_dict.get('cache_dir', UNSET_CACHE_DIR) 1454 if cache_dir is not UNSET_CACHE_DIR: 1455 if cache_dir: 1456 cache_dir = os.path.join(self.root_dir, cache_dir) 1457 cache_dir = os.path.abspath(cache_dir) 1458 1459 git_cache.Mirror.SetCachePath(cache_dir) 1460 1461 if not target_os and config_dict.get('target_os_only', False): 1462 raise gclient_utils.Error('Can\'t use target_os_only if target_os is ' 1463 'not specified') 1464 1465 if not target_cpu and config_dict.get('target_cpu_only', False): 1466 raise gclient_utils.Error('Can\'t use target_cpu_only if target_cpu is ' 1467 'not specified') 1468 1469 deps_to_add = [] 1470 for s in config_dict.get('solutions', []): 1471 try: 1472 deps_to_add.append(GitDependency( 1473 parent=self, 1474 name=s['name'], 1475 url=s['url'], 1476 managed=s.get('managed', True), 1477 custom_deps=s.get('custom_deps', {}), 1478 custom_vars=s.get('custom_vars', {}), 1479 custom_hooks=s.get('custom_hooks', []), 1480 deps_file=s.get('deps_file', 'DEPS'), 1481 should_process=True, 1482 should_recurse=True, 1483 relative=None, 1484 condition=None, 1485 print_outbuf=True)) 1486 except KeyError: 1487 raise gclient_utils.Error('Invalid .gclient file. Solution is ' 1488 'incomplete: %s' % s) 1489 metrics.collector.add( 1490 'project_urls', 1491 [ 1492 dep.FuzzyMatchUrl(metrics_utils.KNOWN_PROJECT_URLS) 1493 for dep in deps_to_add 1494 if dep.FuzzyMatchUrl(metrics_utils.KNOWN_PROJECT_URLS) 1495 ] 1496 ) 1497 1498 self.add_dependencies_and_close(deps_to_add, config_dict.get('hooks', [])) 1499 logging.info('SetConfig() done') 1500 1501 def SaveConfig(self): 1502 gclient_utils.FileWrite(os.path.join(self.root_dir, 1503 self._options.config_filename), 1504 self.config_content) 1505 1506 @staticmethod 1507 def LoadCurrentConfig(options): 1508 """Searches for and loads a .gclient file relative to the current working 1509 dir. Returns a GClient object.""" 1510 if options.spec: 1511 client = GClient('.', options) 1512 client.SetConfig(options.spec) 1513 else: 1514 if options.verbose: 1515 print('Looking for %s starting from %s\n' % ( 1516 options.config_filename, os.getcwd())) 1517 path = gclient_paths.FindGclientRoot(os.getcwd(), options.config_filename) 1518 if not path: 1519 if options.verbose: 1520 print('Couldn\'t find configuration file.') 1521 return None 1522 client = GClient(path, options) 1523 client.SetConfig(gclient_utils.FileRead( 1524 os.path.join(path, options.config_filename))) 1525 1526 if (options.revisions and 1527 len(client.dependencies) > 1 and 1528 any('@' not in r for r in options.revisions)): 1529 print( 1530 ('You must specify the full solution name like --revision %s@%s\n' 1531 'when you have multiple solutions setup in your .gclient file.\n' 1532 'Other solutions present are: %s.') % ( 1533 client.dependencies[0].name, 1534 options.revisions[0], 1535 ', '.join(s.name for s in client.dependencies[1:])), 1536 file=sys.stderr) 1537 return client 1538 1539 def SetDefaultConfig(self, solution_name, deps_file, solution_url, 1540 managed=True, cache_dir=UNSET_CACHE_DIR, 1541 custom_vars=None): 1542 text = self.DEFAULT_CLIENT_FILE_TEXT 1543 format_dict = { 1544 'solution_name': solution_name, 1545 'solution_url': solution_url, 1546 'deps_file': deps_file, 1547 'managed': managed, 1548 'custom_vars': custom_vars or {}, 1549 } 1550 1551 if cache_dir is not UNSET_CACHE_DIR: 1552 text += self.DEFAULT_CLIENT_CACHE_DIR_TEXT 1553 format_dict['cache_dir'] = cache_dir 1554 1555 self.SetConfig(text % format_dict) 1556 1557 def _SaveEntries(self): 1558 """Creates a .gclient_entries file to record the list of unique checkouts. 1559 1560 The .gclient_entries file lives in the same directory as .gclient. 1561 """ 1562 # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It 1563 # makes testing a bit too fun. 1564 result = 'entries = {\n' 1565 for entry in self.root.subtree(False): 1566 result += ' %s: %s,\n' % (pprint.pformat(entry.name), 1567 pprint.pformat(entry.url)) 1568 result += '}\n' 1569 file_path = os.path.join(self.root_dir, self._options.entries_filename) 1570 logging.debug(result) 1571 gclient_utils.FileWrite(file_path, result) 1572 1573 def _ReadEntries(self): 1574 """Read the .gclient_entries file for the given client. 1575 1576 Returns: 1577 A sequence of solution names, which will be empty if there is the 1578 entries file hasn't been created yet. 1579 """ 1580 scope = {} 1581 filename = os.path.join(self.root_dir, self._options.entries_filename) 1582 if not os.path.exists(filename): 1583 return {} 1584 try: 1585 exec(gclient_utils.FileRead(filename), scope) 1586 except SyntaxError as e: 1587 gclient_utils.SyntaxErrorToError(filename, e) 1588 return scope.get('entries', {}) 1589 1590 def _EnforceRevisions(self): 1591 """Checks for revision overrides.""" 1592 revision_overrides = {} 1593 if self._options.head: 1594 return revision_overrides 1595 if not self._options.revisions: 1596 return revision_overrides 1597 solutions_names = [s.name for s in self.dependencies] 1598 index = 0 1599 for revision in self._options.revisions: 1600 if not '@' in revision: 1601 # Support for --revision 123 1602 revision = '%s@%s' % (solutions_names[index], revision) 1603 name, rev = revision.split('@', 1) 1604 revision_overrides[name] = rev 1605 index += 1 1606 return revision_overrides 1607 1608 def _EnforcePatchRefsAndBranches(self): 1609 """Checks for patch refs.""" 1610 patch_refs = {} 1611 target_branches = {} 1612 if not self._options.patch_refs: 1613 return patch_refs, target_branches 1614 for given_patch_ref in self._options.patch_refs: 1615 patch_repo, _, patch_ref = given_patch_ref.partition('@') 1616 if not patch_repo or not patch_ref or ':' not in patch_ref: 1617 raise gclient_utils.Error( 1618 'Wrong revision format: %s should be of the form ' 1619 'patch_repo@target_branch:patch_ref.' % given_patch_ref) 1620 target_branch, _, patch_ref = patch_ref.partition(':') 1621 target_branches[patch_repo] = target_branch 1622 patch_refs[patch_repo] = patch_ref 1623 return patch_refs, target_branches 1624 1625 def _RemoveUnversionedGitDirs(self): 1626 """Remove directories that are no longer part of the checkout. 1627 1628 Notify the user if there is an orphaned entry in their working copy. 1629 Only delete the directory if there are no changes in it, and 1630 delete_unversioned_trees is set to true. 1631 """ 1632 1633 entries = [i.name for i in self.root.subtree(False) if i.url] 1634 full_entries = [os.path.join(self.root_dir, e.replace('/', os.path.sep)) 1635 for e in entries] 1636 1637 for entry, prev_url in self._ReadEntries().items(): 1638 if not prev_url: 1639 # entry must have been overridden via .gclient custom_deps 1640 continue 1641 # Fix path separator on Windows. 1642 entry_fixed = entry.replace('/', os.path.sep) 1643 e_dir = os.path.join(self.root_dir, entry_fixed) 1644 # Use entry and not entry_fixed there. 1645 if (entry not in entries and 1646 (not any(path.startswith(entry + '/') for path in entries)) and 1647 os.path.exists(e_dir)): 1648 # The entry has been removed from DEPS. 1649 scm = gclient_scm.GitWrapper( 1650 prev_url, self.root_dir, entry_fixed, self.outbuf) 1651 1652 # Check to see if this directory is now part of a higher-up checkout. 1653 scm_root = None 1654 try: 1655 scm_root = gclient_scm.scm.GIT.GetCheckoutRoot(scm.checkout_path) 1656 except subprocess2.CalledProcessError: 1657 pass 1658 if not scm_root: 1659 logging.warning('Could not find checkout root for %s. Unable to ' 1660 'determine whether it is part of a higher-level ' 1661 'checkout, so not removing.' % entry) 1662 continue 1663 1664 # This is to handle the case of third_party/WebKit migrating from 1665 # being a DEPS entry to being part of the main project. 1666 # If the subproject is a Git project, we need to remove its .git 1667 # folder. Otherwise git operations on that folder will have different 1668 # effects depending on the current working directory. 1669 if os.path.abspath(scm_root) == os.path.abspath(e_dir): 1670 e_par_dir = os.path.join(e_dir, os.pardir) 1671 if gclient_scm.scm.GIT.IsInsideWorkTree(e_par_dir): 1672 par_scm_root = gclient_scm.scm.GIT.GetCheckoutRoot(e_par_dir) 1673 # rel_e_dir : relative path of entry w.r.t. its parent repo. 1674 rel_e_dir = os.path.relpath(e_dir, par_scm_root) 1675 if gclient_scm.scm.GIT.IsDirectoryVersioned( 1676 par_scm_root, rel_e_dir): 1677 save_dir = scm.GetGitBackupDirPath() 1678 # Remove any eventual stale backup dir for the same project. 1679 if os.path.exists(save_dir): 1680 gclient_utils.rmtree(save_dir) 1681 os.rename(os.path.join(e_dir, '.git'), save_dir) 1682 # When switching between the two states (entry/ is a subproject 1683 # -> entry/ is part of the outer project), it is very likely 1684 # that some files are changed in the checkout, unless we are 1685 # jumping *exactly* across the commit which changed just DEPS. 1686 # In such case we want to cleanup any eventual stale files 1687 # (coming from the old subproject) in order to end up with a 1688 # clean checkout. 1689 gclient_scm.scm.GIT.CleanupDir(par_scm_root, rel_e_dir) 1690 assert not os.path.exists(os.path.join(e_dir, '.git')) 1691 print('\nWARNING: \'%s\' has been moved from DEPS to a higher ' 1692 'level checkout. The git folder containing all the local' 1693 ' branches has been saved to %s.\n' 1694 'If you don\'t care about its state you can safely ' 1695 'remove that folder to free up space.' % (entry, save_dir)) 1696 continue 1697 1698 if scm_root in full_entries: 1699 logging.info('%s is part of a higher level checkout, not removing', 1700 scm.GetCheckoutRoot()) 1701 continue 1702 1703 file_list = [] 1704 scm.status(self._options, [], file_list) 1705 modified_files = file_list != [] 1706 if (not self._options.delete_unversioned_trees or 1707 (modified_files and not self._options.force)): 1708 # There are modified files in this entry. Keep warning until 1709 # removed. 1710 self.add_dependency( 1711 GitDependency( 1712 parent=self, 1713 name=entry, 1714 url=prev_url, 1715 managed=False, 1716 custom_deps={}, 1717 custom_vars={}, 1718 custom_hooks=[], 1719 deps_file=None, 1720 should_process=True, 1721 should_recurse=False, 1722 relative=None, 1723 condition=None)) 1724 if modified_files and self._options.delete_unversioned_trees: 1725 print('\nWARNING: \'%s\' is no longer part of this client.\n' 1726 'Despite running \'gclient sync -D\' no action was taken ' 1727 'as there are modifications.\nIt is recommended you revert ' 1728 'all changes or run \'gclient sync -D --force\' next ' 1729 'time.' % entry_fixed) 1730 else: 1731 print('\nWARNING: \'%s\' is no longer part of this client.\n' 1732 'It is recommended that you manually remove it or use ' 1733 '\'gclient sync -D\' next time.' % entry_fixed) 1734 else: 1735 # Delete the entry 1736 print('\n________ deleting \'%s\' in \'%s\'' % ( 1737 entry_fixed, self.root_dir)) 1738 gclient_utils.rmtree(e_dir) 1739 # record the current list of entries for next time 1740 self._SaveEntries() 1741 1742 def RunOnDeps(self, command, args, ignore_requirements=False, progress=True): 1743 """Runs a command on each dependency in a client and its dependencies. 1744 1745 Args: 1746 command: The command to use (e.g., 'status' or 'diff') 1747 args: list of str - extra arguments to add to the command line. 1748 """ 1749 if not self.dependencies: 1750 raise gclient_utils.Error('No solution specified') 1751 1752 revision_overrides = {} 1753 patch_refs = {} 1754 target_branches = {} 1755 # It's unnecessary to check for revision overrides for 'recurse'. 1756 # Save a few seconds by not calling _EnforceRevisions() in that case. 1757 if command not in ('diff', 'recurse', 'runhooks', 'status', 'revert', 1758 'validate'): 1759 self._CheckConfig() 1760 revision_overrides = self._EnforceRevisions() 1761 1762 if command == 'update': 1763 patch_refs, target_branches = self._EnforcePatchRefsAndBranches() 1764 # Disable progress for non-tty stdout. 1765 should_show_progress = ( 1766 setup_color.IS_TTY and not self._options.verbose and progress) 1767 pm = None 1768 if should_show_progress: 1769 if command in ('update', 'revert'): 1770 pm = Progress('Syncing projects', 1) 1771 elif command in ('recurse', 'validate'): 1772 pm = Progress(' '.join(args), 1) 1773 work_queue = gclient_utils.ExecutionQueue( 1774 self._options.jobs, pm, ignore_requirements=ignore_requirements, 1775 verbose=self._options.verbose) 1776 for s in self.dependencies: 1777 if s.should_process: 1778 work_queue.enqueue(s) 1779 work_queue.flush(revision_overrides, command, args, options=self._options, 1780 patch_refs=patch_refs, target_branches=target_branches) 1781 1782 if revision_overrides: 1783 print('Please fix your script, having invalid --revision flags will soon ' 1784 'be considered an error.', file=sys.stderr) 1785 1786 if patch_refs: 1787 raise gclient_utils.Error( 1788 'The following --patch-ref flags were not used. Please fix it:\n%s' % 1789 ('\n'.join( 1790 patch_repo + '@' + patch_ref 1791 for patch_repo, patch_ref in patch_refs.items()))) 1792 1793 # Once all the dependencies have been processed, it's now safe to write 1794 # out the gn_args_file and run the hooks. 1795 if command == 'update': 1796 gn_args_dep = self.dependencies[0] 1797 if gn_args_dep._gn_args_from: 1798 deps_map = dict([(dep.name, dep) for dep in gn_args_dep.dependencies]) 1799 gn_args_dep = deps_map.get(gn_args_dep._gn_args_from) 1800 if gn_args_dep and gn_args_dep.HasGNArgsFile(): 1801 gn_args_dep.WriteGNArgsFile() 1802 1803 self._RemoveUnversionedGitDirs() 1804 1805 # Sync CIPD dependencies once removed deps are deleted. In case a git 1806 # dependency was moved to CIPD, we want to remove the old git directory 1807 # first and then sync the CIPD dep. 1808 if self._cipd_root: 1809 self._cipd_root.run(command) 1810 1811 if not self._options.nohooks: 1812 if should_show_progress: 1813 pm = Progress('Running hooks', 1) 1814 self.RunHooksRecursively(self._options, pm) 1815 1816 1817 return 0 1818 1819 def PrintRevInfo(self): 1820 if not self.dependencies: 1821 raise gclient_utils.Error('No solution specified') 1822 # Load all the settings. 1823 work_queue = gclient_utils.ExecutionQueue( 1824 self._options.jobs, None, False, verbose=self._options.verbose) 1825 for s in self.dependencies: 1826 if s.should_process: 1827 work_queue.enqueue(s) 1828 work_queue.flush({}, None, [], options=self._options, patch_refs=None, 1829 target_branches=None) 1830 1831 def ShouldPrintRevision(dep): 1832 return (not self._options.filter 1833 or dep.FuzzyMatchUrl(self._options.filter)) 1834 1835 if self._options.snapshot: 1836 json_output = [] 1837 # First level at .gclient 1838 for d in self.dependencies: 1839 entries = {} 1840 def GrabDeps(dep): 1841 """Recursively grab dependencies.""" 1842 for d in dep.dependencies: 1843 d.PinToActualRevision() 1844 if ShouldPrintRevision(d): 1845 entries[d.name] = d.url 1846 GrabDeps(d) 1847 GrabDeps(d) 1848 json_output.append({ 1849 'name': d.name, 1850 'solution_url': d.url, 1851 'deps_file': d.deps_file, 1852 'managed': d.managed, 1853 'custom_deps': entries, 1854 }) 1855 if self._options.output_json == '-': 1856 print(json.dumps(json_output, indent=2, separators=(',', ': '))) 1857 elif self._options.output_json: 1858 with open(self._options.output_json, 'w') as f: 1859 json.dump(json_output, f) 1860 else: 1861 # Print the snapshot configuration file 1862 print(self.DEFAULT_SNAPSHOT_FILE_TEXT % { 1863 'solution_list': pprint.pformat(json_output, indent=2), 1864 }) 1865 else: 1866 entries = {} 1867 for d in self.root.subtree(False): 1868 if self._options.actual: 1869 d.PinToActualRevision() 1870 if ShouldPrintRevision(d): 1871 entries[d.name] = d.url 1872 if self._options.output_json: 1873 json_output = { 1874 name: { 1875 'url': rev.split('@')[0] if rev else None, 1876 'rev': rev.split('@')[1] if rev and '@' in rev else None, 1877 } 1878 for name, rev in entries.items() 1879 } 1880 if self._options.output_json == '-': 1881 print(json.dumps(json_output, indent=2, separators=(',', ': '))) 1882 else: 1883 with open(self._options.output_json, 'w') as f: 1884 json.dump(json_output, f) 1885 else: 1886 keys = sorted(entries.keys()) 1887 for x in keys: 1888 print('%s: %s' % (x, entries[x])) 1889 logging.info(str(self)) 1890 1891 def ParseDepsFile(self): 1892 """No DEPS to parse for a .gclient file.""" 1893 raise gclient_utils.Error('Internal error') 1894 1895 def PrintLocationAndContents(self): 1896 # Print out the .gclient file. This is longer than if we just printed the 1897 # client dict, but more legible, and it might contain helpful comments. 1898 print('Loaded .gclient config in %s:\n%s' % ( 1899 self.root_dir, self.config_content)) 1900 1901 def GetCipdRoot(self): 1902 if not self._cipd_root: 1903 self._cipd_root = gclient_scm.CipdRoot( 1904 self.root_dir, 1905 # TODO(jbudorick): Support other service URLs as necessary. 1906 # Service URLs should be constant over the scope of a cipd 1907 # root, so a var per DEPS file specifying the service URL 1908 # should suffice. 1909 'https://chrome-infra-packages.appspot.com') 1910 return self._cipd_root 1911 1912 @property 1913 def root_dir(self): 1914 """Root directory of gclient checkout.""" 1915 return self._root_dir 1916 1917 @property 1918 def enforced_os(self): 1919 """What deps_os entries that are to be parsed.""" 1920 return self._enforced_os 1921 1922 @property 1923 def target_os(self): 1924 return self._enforced_os 1925 1926 @property 1927 def target_cpu(self): 1928 return self._enforced_cpu 1929 1930 1931class CipdDependency(Dependency): 1932 """A Dependency object that represents a single CIPD package.""" 1933 1934 def __init__( 1935 self, parent, name, dep_value, cipd_root, 1936 custom_vars, should_process, relative, condition): 1937 package = dep_value['package'] 1938 version = dep_value['version'] 1939 url = urlparse.urljoin( 1940 cipd_root.service_url, '%s@%s' % (package, version)) 1941 super(CipdDependency, self).__init__( 1942 parent=parent, 1943 name=name + ':' + package, 1944 url=url, 1945 managed=None, 1946 custom_deps=None, 1947 custom_vars=custom_vars, 1948 custom_hooks=None, 1949 deps_file=None, 1950 should_process=should_process, 1951 should_recurse=False, 1952 relative=relative, 1953 condition=condition) 1954 self._cipd_package = None 1955 self._cipd_root = cipd_root 1956 # CIPD wants /-separated paths, even on Windows. 1957 native_subdir_path = os.path.relpath( 1958 os.path.join(self.root.root_dir, name), cipd_root.root_dir) 1959 self._cipd_subdir = posixpath.join(*native_subdir_path.split(os.sep)) 1960 self._package_name = package 1961 self._package_version = version 1962 1963 #override 1964 def run(self, revision_overrides, command, args, work_queue, options, 1965 patch_refs, target_branches): 1966 """Runs |command| then parse the DEPS file.""" 1967 logging.info('CipdDependency(%s).run()' % self.name) 1968 if not self.should_process: 1969 return 1970 self._CreatePackageIfNecessary() 1971 super(CipdDependency, self).run(revision_overrides, command, args, 1972 work_queue, options, patch_refs, 1973 target_branches) 1974 1975 def _CreatePackageIfNecessary(self): 1976 # We lazily create the CIPD package to make sure that only packages 1977 # that we want (as opposed to all packages defined in all DEPS files 1978 # we parse) get added to the root and subsequently ensured. 1979 if not self._cipd_package: 1980 self._cipd_package = self._cipd_root.add_package( 1981 self._cipd_subdir, self._package_name, self._package_version) 1982 1983 def ParseDepsFile(self): 1984 """CIPD dependencies are not currently allowed to have nested deps.""" 1985 self.add_dependencies_and_close([], []) 1986 1987 #override 1988 def verify_validity(self): 1989 """CIPD dependencies allow duplicate name for packages in same directory.""" 1990 logging.info('Dependency(%s).verify_validity()' % self.name) 1991 return True 1992 1993 #override 1994 def GetScmName(self): 1995 """Always 'cipd'.""" 1996 return 'cipd' 1997 1998 #override 1999 def CreateSCM(self, out_cb=None): 2000 """Create a Wrapper instance suitable for handling this CIPD dependency.""" 2001 self._CreatePackageIfNecessary() 2002 return gclient_scm.CipdWrapper( 2003 self.url, self.root.root_dir, self.name, self.outbuf, out_cb, 2004 root=self._cipd_root, package=self._cipd_package) 2005 2006 def hierarchy(self, include_url=False): 2007 return self.parent.hierarchy(include_url) + ' -> ' + self._cipd_subdir 2008 2009 def ToLines(self): 2010 """Return a list of lines representing this in a DEPS file.""" 2011 def escape_cipd_var(package): 2012 return package.replace('{', '{{').replace('}', '}}') 2013 2014 s = [] 2015 self._CreatePackageIfNecessary() 2016 if self._cipd_package.authority_for_subdir: 2017 condition_part = ([' "condition": %r,' % self.condition] 2018 if self.condition else []) 2019 s.extend([ 2020 ' # %s' % self.hierarchy(include_url=False), 2021 ' "%s": {' % (self.name.split(':')[0],), 2022 ' "packages": [', 2023 ]) 2024 for p in sorted( 2025 self._cipd_root.packages(self._cipd_subdir), 2026 key=lambda x: x.name): 2027 s.extend([ 2028 ' {', 2029 ' "package": "%s",' % escape_cipd_var(p.name), 2030 ' "version": "%s",' % p.version, 2031 ' },', 2032 ]) 2033 2034 s.extend([ 2035 ' ],', 2036 ' "dep_type": "cipd",', 2037 ] + condition_part + [ 2038 ' },', 2039 '', 2040 ]) 2041 return s 2042 2043 2044#### gclient commands. 2045 2046 2047@subcommand.usage('[command] [args ...]') 2048@metrics.collector.collect_metrics('gclient recurse') 2049def CMDrecurse(parser, args): 2050 """Operates [command args ...] on all the dependencies. 2051 2052 Runs a shell command on all entries. 2053 Sets GCLIENT_DEP_PATH environment variable as the dep's relative location to 2054 root directory of the checkout. 2055 """ 2056 # Stop parsing at the first non-arg so that these go through to the command 2057 parser.disable_interspersed_args() 2058 parser.add_option('-s', '--scm', action='append', default=[], 2059 help='Choose scm types to operate upon.') 2060 parser.add_option('-i', '--ignore', action='store_true', 2061 help='Ignore non-zero return codes from subcommands.') 2062 parser.add_option('--prepend-dir', action='store_true', 2063 help='Prepend relative dir for use with git <cmd> --null.') 2064 parser.add_option('--no-progress', action='store_true', 2065 help='Disable progress bar that shows sub-command updates') 2066 options, args = parser.parse_args(args) 2067 if not args: 2068 print('Need to supply a command!', file=sys.stderr) 2069 return 1 2070 root_and_entries = gclient_utils.GetGClientRootAndEntries() 2071 if not root_and_entries: 2072 print( 2073 'You need to run gclient sync at least once to use \'recurse\'.\n' 2074 'This is because .gclient_entries needs to exist and be up to date.', 2075 file=sys.stderr) 2076 return 1 2077 2078 # Normalize options.scm to a set() 2079 scm_set = set() 2080 for scm in options.scm: 2081 scm_set.update(scm.split(',')) 2082 options.scm = scm_set 2083 2084 options.nohooks = True 2085 client = GClient.LoadCurrentConfig(options) 2086 if not client: 2087 raise gclient_utils.Error('client not configured; see \'gclient config\'') 2088 return client.RunOnDeps('recurse', args, ignore_requirements=True, 2089 progress=not options.no_progress) 2090 2091 2092@subcommand.usage('[args ...]') 2093@metrics.collector.collect_metrics('gclient fetch') 2094def CMDfetch(parser, args): 2095 """Fetches upstream commits for all modules. 2096 2097 Completely git-specific. Simply runs 'git fetch [args ...]' for each module. 2098 """ 2099 (options, args) = parser.parse_args(args) 2100 return CMDrecurse(OptionParser(), [ 2101 '--jobs=%d' % options.jobs, '--scm=git', 'git', 'fetch'] + args) 2102 2103 2104class Flattener(object): 2105 """Flattens a gclient solution.""" 2106 2107 def __init__(self, client, pin_all_deps=False): 2108 """Constructor. 2109 2110 Arguments: 2111 client (GClient): client to flatten 2112 pin_all_deps (bool): whether to pin all deps, even if they're not pinned 2113 in DEPS 2114 """ 2115 self._client = client 2116 2117 self._deps_string = None 2118 self._deps_files = set() 2119 2120 self._allowed_hosts = set() 2121 self._deps = {} 2122 self._hooks = [] 2123 self._pre_deps_hooks = [] 2124 self._vars = {} 2125 2126 self._flatten(pin_all_deps=pin_all_deps) 2127 2128 @property 2129 def deps_string(self): 2130 assert self._deps_string is not None 2131 return self._deps_string 2132 2133 @property 2134 def deps_files(self): 2135 return self._deps_files 2136 2137 def _pin_dep(self, dep): 2138 """Pins a dependency to specific full revision sha. 2139 2140 Arguments: 2141 dep (Dependency): dependency to process 2142 """ 2143 if dep.url is None: 2144 return 2145 2146 # Make sure the revision is always fully specified (a hash), 2147 # as opposed to refs or tags which might change. Similarly, 2148 # shortened shas might become ambiguous; make sure to always 2149 # use full one for pinning. 2150 revision = gclient_utils.SplitUrlRevision(dep.url)[1] 2151 if not revision or not gclient_utils.IsFullGitSha(revision): 2152 dep.PinToActualRevision() 2153 2154 def _flatten(self, pin_all_deps=False): 2155 """Runs the flattener. Saves resulting DEPS string. 2156 2157 Arguments: 2158 pin_all_deps (bool): whether to pin all deps, even if they're not pinned 2159 in DEPS 2160 """ 2161 for solution in self._client.dependencies: 2162 self._add_dep(solution) 2163 self._flatten_dep(solution) 2164 2165 if pin_all_deps: 2166 for dep in self._deps.values(): 2167 self._pin_dep(dep) 2168 2169 def add_deps_file(dep): 2170 # Only include DEPS files referenced by recursedeps. 2171 if not dep.should_recurse: 2172 return 2173 deps_file = dep.deps_file 2174 deps_path = os.path.join(self._client.root_dir, dep.name, deps_file) 2175 if not os.path.exists(deps_path): 2176 # gclient has a fallback that if deps_file doesn't exist, it'll try 2177 # DEPS. Do the same here. 2178 deps_file = 'DEPS' 2179 deps_path = os.path.join(self._client.root_dir, dep.name, deps_file) 2180 if not os.path.exists(deps_path): 2181 return 2182 assert dep.url 2183 self._deps_files.add((dep.url, deps_file, dep.hierarchy_data())) 2184 for dep in self._deps.values(): 2185 add_deps_file(dep) 2186 2187 gn_args_dep = self._deps.get(self._client.dependencies[0]._gn_args_from, 2188 self._client.dependencies[0]) 2189 self._deps_string = '\n'.join( 2190 _GNSettingsToLines(gn_args_dep._gn_args_file, gn_args_dep._gn_args) + 2191 _AllowedHostsToLines(self._allowed_hosts) + 2192 _DepsToLines(self._deps) + 2193 _HooksToLines('hooks', self._hooks) + 2194 _HooksToLines('pre_deps_hooks', self._pre_deps_hooks) + 2195 _VarsToLines(self._vars) + 2196 ['# %s, %s' % (url, deps_file) 2197 for url, deps_file, _ in sorted(self._deps_files)] + 2198 ['']) # Ensure newline at end of file. 2199 2200 def _add_dep(self, dep): 2201 """Helper to add a dependency to flattened DEPS. 2202 2203 Arguments: 2204 dep (Dependency): dependency to add 2205 """ 2206 assert dep.name not in self._deps or self._deps.get(dep.name) == dep, ( 2207 dep.name, self._deps.get(dep.name)) 2208 if dep.url: 2209 self._deps[dep.name] = dep 2210 2211 def _flatten_dep(self, dep): 2212 """Visits a dependency in order to flatten it (see CMDflatten). 2213 2214 Arguments: 2215 dep (Dependency): dependency to process 2216 """ 2217 logging.debug('_flatten_dep(%s)', dep.name) 2218 2219 assert dep.deps_parsed, ( 2220 "Attempted to flatten %s but it has not been processed." % dep.name) 2221 2222 self._allowed_hosts.update(dep.allowed_hosts) 2223 2224 # Only include vars explicitly listed in the DEPS files or gclient solution, 2225 # not automatic, local overrides (i.e. not all of dep.get_vars()). 2226 hierarchy = dep.hierarchy(include_url=False) 2227 for key, value in dep._vars.items(): 2228 # Make sure there are no conflicting variables. It is fine however 2229 # to use same variable name, as long as the value is consistent. 2230 assert key not in self._vars or self._vars[key][1] == value, ( 2231 "dep:%s key:%s value:%s != %s" % ( 2232 dep.name, key, value, self._vars[key][1])) 2233 self._vars[key] = (hierarchy, value) 2234 # Override explicit custom variables. 2235 for key, value in dep.custom_vars.items(): 2236 # Do custom_vars that don't correspond to DEPS vars ever make sense? DEPS 2237 # conditionals shouldn't be using vars that aren't also defined in the 2238 # DEPS (presubmit actually disallows this), so any new custom_var must be 2239 # unused in the DEPS, so no need to add it to the flattened output either. 2240 if key not in self._vars: 2241 continue 2242 # Don't "override" existing vars if it's actually the same value. 2243 elif self._vars[key][1] == value: 2244 continue 2245 # Anything else is overriding a default value from the DEPS. 2246 self._vars[key] = (hierarchy + ' [custom_var override]', value) 2247 2248 self._pre_deps_hooks.extend([(dep, hook) for hook in dep.pre_deps_hooks]) 2249 self._hooks.extend([(dep, hook) for hook in dep.deps_hooks]) 2250 2251 for sub_dep in dep.dependencies: 2252 self._add_dep(sub_dep) 2253 2254 for d in dep.dependencies: 2255 if d.should_recurse: 2256 self._flatten_dep(d) 2257 2258 2259@metrics.collector.collect_metrics('gclient flatten') 2260def CMDflatten(parser, args): 2261 """Flattens the solutions into a single DEPS file.""" 2262 parser.add_option('--output-deps', help='Path to the output DEPS file') 2263 parser.add_option( 2264 '--output-deps-files', 2265 help=('Path to the output metadata about DEPS files referenced by ' 2266 'recursedeps.')) 2267 parser.add_option( 2268 '--pin-all-deps', action='store_true', 2269 help=('Pin all deps, even if not pinned in DEPS. CAVEAT: only does so ' 2270 'for checked out deps, NOT deps_os.')) 2271 options, args = parser.parse_args(args) 2272 2273 options.nohooks = True 2274 options.process_all_deps = True 2275 client = GClient.LoadCurrentConfig(options) 2276 2277 # Only print progress if we're writing to a file. Otherwise, progress updates 2278 # could obscure intended output. 2279 code = client.RunOnDeps('flatten', args, progress=options.output_deps) 2280 if code != 0: 2281 return code 2282 2283 flattener = Flattener(client, pin_all_deps=options.pin_all_deps) 2284 2285 if options.output_deps: 2286 with open(options.output_deps, 'w') as f: 2287 f.write(flattener.deps_string) 2288 else: 2289 print(flattener.deps_string) 2290 2291 deps_files = [{'url': d[0], 'deps_file': d[1], 'hierarchy': d[2]} 2292 for d in sorted(flattener.deps_files)] 2293 if options.output_deps_files: 2294 with open(options.output_deps_files, 'w') as f: 2295 json.dump(deps_files, f) 2296 2297 return 0 2298 2299 2300def _GNSettingsToLines(gn_args_file, gn_args): 2301 s = [] 2302 if gn_args_file: 2303 s.extend([ 2304 'gclient_gn_args_file = "%s"' % gn_args_file, 2305 'gclient_gn_args = %r' % gn_args, 2306 ]) 2307 return s 2308 2309 2310def _AllowedHostsToLines(allowed_hosts): 2311 """Converts |allowed_hosts| set to list of lines for output.""" 2312 if not allowed_hosts: 2313 return [] 2314 s = ['allowed_hosts = ['] 2315 for h in sorted(allowed_hosts): 2316 s.append(' "%s",' % h) 2317 s.extend([']', '']) 2318 return s 2319 2320 2321def _DepsToLines(deps): 2322 """Converts |deps| dict to list of lines for output.""" 2323 if not deps: 2324 return [] 2325 s = ['deps = {'] 2326 for _, dep in sorted(deps.items()): 2327 s.extend(dep.ToLines()) 2328 s.extend(['}', '']) 2329 return s 2330 2331 2332def _DepsOsToLines(deps_os): 2333 """Converts |deps_os| dict to list of lines for output.""" 2334 if not deps_os: 2335 return [] 2336 s = ['deps_os = {'] 2337 for dep_os, os_deps in sorted(deps_os.items()): 2338 s.append(' "%s": {' % dep_os) 2339 for name, dep in sorted(os_deps.items()): 2340 condition_part = ([' "condition": %r,' % dep.condition] 2341 if dep.condition else []) 2342 s.extend([ 2343 ' # %s' % dep.hierarchy(include_url=False), 2344 ' "%s": {' % (name,), 2345 ' "url": "%s",' % (dep.url,), 2346 ] + condition_part + [ 2347 ' },', 2348 '', 2349 ]) 2350 s.extend([' },', '']) 2351 s.extend(['}', '']) 2352 return s 2353 2354 2355def _HooksToLines(name, hooks): 2356 """Converts |hooks| list to list of lines for output.""" 2357 if not hooks: 2358 return [] 2359 s = ['%s = [' % name] 2360 for dep, hook in hooks: 2361 s.extend([ 2362 ' # %s' % dep.hierarchy(include_url=False), 2363 ' {', 2364 ]) 2365 if hook.name is not None: 2366 s.append(' "name": "%s",' % hook.name) 2367 if hook.pattern is not None: 2368 s.append(' "pattern": "%s",' % hook.pattern) 2369 if hook.condition is not None: 2370 s.append(' "condition": %r,' % hook.condition) 2371 # Flattened hooks need to be written relative to the root gclient dir 2372 cwd = os.path.relpath(os.path.normpath(hook.effective_cwd)) 2373 s.extend( 2374 [' "cwd": "%s",' % cwd] + 2375 [' "action": ['] + 2376 [' "%s",' % arg for arg in hook.action] + 2377 [' ]', ' },', ''] 2378 ) 2379 s.extend([']', '']) 2380 return s 2381 2382 2383def _HooksOsToLines(hooks_os): 2384 """Converts |hooks| list to list of lines for output.""" 2385 if not hooks_os: 2386 return [] 2387 s = ['hooks_os = {'] 2388 for hook_os, os_hooks in hooks_os.items(): 2389 s.append(' "%s": [' % hook_os) 2390 for dep, hook in os_hooks: 2391 s.extend([ 2392 ' # %s' % dep.hierarchy(include_url=False), 2393 ' {', 2394 ]) 2395 if hook.name is not None: 2396 s.append(' "name": "%s",' % hook.name) 2397 if hook.pattern is not None: 2398 s.append(' "pattern": "%s",' % hook.pattern) 2399 if hook.condition is not None: 2400 s.append(' "condition": %r,' % hook.condition) 2401 # Flattened hooks need to be written relative to the root gclient dir 2402 cwd = os.path.relpath(os.path.normpath(hook.effective_cwd)) 2403 s.extend( 2404 [' "cwd": "%s",' % cwd] + 2405 [' "action": ['] + 2406 [' "%s",' % arg for arg in hook.action] + 2407 [' ]', ' },', ''] 2408 ) 2409 s.extend([' ],', '']) 2410 s.extend(['}', '']) 2411 return s 2412 2413 2414def _VarsToLines(variables): 2415 """Converts |variables| dict to list of lines for output.""" 2416 if not variables: 2417 return [] 2418 s = ['vars = {'] 2419 for key, tup in sorted(variables.items()): 2420 hierarchy, value = tup 2421 s.extend([ 2422 ' # %s' % hierarchy, 2423 ' "%s": %r,' % (key, value), 2424 '', 2425 ]) 2426 s.extend(['}', '']) 2427 return s 2428 2429 2430@metrics.collector.collect_metrics('gclient grep') 2431def CMDgrep(parser, args): 2432 """Greps through git repos managed by gclient. 2433 2434 Runs 'git grep [args...]' for each module. 2435 """ 2436 # We can't use optparse because it will try to parse arguments sent 2437 # to git grep and throw an error. :-( 2438 if not args or re.match('(-h|--help)$', args[0]): 2439 print( 2440 'Usage: gclient grep [-j <N>] git-grep-args...\n\n' 2441 'Example: "gclient grep -j10 -A2 RefCountedBase" runs\n"git grep ' 2442 '-A2 RefCountedBase" on each of gclient\'s git\nrepos with up to ' 2443 '10 jobs.\n\nBonus: page output by appending "|& less -FRSX" to the' 2444 ' end of your query.', 2445 file=sys.stderr) 2446 return 1 2447 2448 jobs_arg = ['--jobs=1'] 2449 if re.match(r'(-j|--jobs=)\d+$', args[0]): 2450 jobs_arg, args = args[:1], args[1:] 2451 elif re.match(r'(-j|--jobs)$', args[0]): 2452 jobs_arg, args = args[:2], args[2:] 2453 2454 return CMDrecurse( 2455 parser, 2456 jobs_arg + ['--ignore', '--prepend-dir', '--no-progress', '--scm=git', 2457 'git', 'grep', '--null', '--color=Always'] + args) 2458 2459 2460@metrics.collector.collect_metrics('gclient root') 2461def CMDroot(parser, args): 2462 """Outputs the solution root (or current dir if there isn't one).""" 2463 (options, args) = parser.parse_args(args) 2464 client = GClient.LoadCurrentConfig(options) 2465 if client: 2466 print(os.path.abspath(client.root_dir)) 2467 else: 2468 print(os.path.abspath('.')) 2469 2470 2471@subcommand.usage('[url]') 2472@metrics.collector.collect_metrics('gclient config') 2473def CMDconfig(parser, args): 2474 """Creates a .gclient file in the current directory. 2475 2476 This specifies the configuration for further commands. After update/sync, 2477 top-level DEPS files in each module are read to determine dependent 2478 modules to operate on as well. If optional [url] parameter is 2479 provided, then configuration is read from a specified Subversion server 2480 URL. 2481 """ 2482 # We do a little dance with the --gclientfile option. 'gclient config' is the 2483 # only command where it's acceptable to have both '--gclientfile' and '--spec' 2484 # arguments. So, we temporarily stash any --gclientfile parameter into 2485 # options.output_config_file until after the (gclientfile xor spec) error 2486 # check. 2487 parser.remove_option('--gclientfile') 2488 parser.add_option('--gclientfile', dest='output_config_file', 2489 help='Specify an alternate .gclient file') 2490 parser.add_option('--name', 2491 help='overrides the default name for the solution') 2492 parser.add_option('--deps-file', default='DEPS', 2493 help='overrides the default name for the DEPS file for the ' 2494 'main solutions and all sub-dependencies') 2495 parser.add_option('--unmanaged', action='store_true', default=False, 2496 help='overrides the default behavior to make it possible ' 2497 'to have the main solution untouched by gclient ' 2498 '(gclient will check out unmanaged dependencies but ' 2499 'will never sync them)') 2500 parser.add_option('--cache-dir', default=UNSET_CACHE_DIR, 2501 help='Cache all git repos into this dir and do shared ' 2502 'clones from the cache, instead of cloning directly ' 2503 'from the remote. Pass "None" to disable cache, even ' 2504 'if globally enabled due to $GIT_CACHE_PATH.') 2505 parser.add_option('--custom-var', action='append', dest='custom_vars', 2506 default=[], 2507 help='overrides variables; key=value syntax') 2508 parser.set_defaults(config_filename=None) 2509 (options, args) = parser.parse_args(args) 2510 if options.output_config_file: 2511 setattr(options, 'config_filename', getattr(options, 'output_config_file')) 2512 if ((options.spec and args) or len(args) > 2 or 2513 (not options.spec and not args)): 2514 parser.error('Inconsistent arguments. Use either --spec or one or 2 args') 2515 2516 if (options.cache_dir is not UNSET_CACHE_DIR 2517 and options.cache_dir.lower() == 'none'): 2518 options.cache_dir = None 2519 2520 custom_vars = {} 2521 for arg in options.custom_vars: 2522 kv = arg.split('=', 1) 2523 if len(kv) != 2: 2524 parser.error('Invalid --custom-var argument: %r' % arg) 2525 custom_vars[kv[0]] = gclient_eval.EvaluateCondition(kv[1], {}) 2526 2527 client = GClient('.', options) 2528 if options.spec: 2529 client.SetConfig(options.spec) 2530 else: 2531 base_url = args[0].rstrip('/') 2532 if not options.name: 2533 name = base_url.split('/')[-1] 2534 if name.endswith('.git'): 2535 name = name[:-4] 2536 else: 2537 # specify an alternate relpath for the given URL. 2538 name = options.name 2539 if not os.path.abspath(os.path.join(os.getcwd(), name)).startswith( 2540 os.getcwd()): 2541 parser.error('Do not pass a relative path for --name.') 2542 if any(x in ('..', '.', '/', '\\') for x in name.split(os.sep)): 2543 parser.error('Do not include relative path components in --name.') 2544 2545 deps_file = options.deps_file 2546 client.SetDefaultConfig(name, deps_file, base_url, 2547 managed=not options.unmanaged, 2548 cache_dir=options.cache_dir, 2549 custom_vars=custom_vars) 2550 client.SaveConfig() 2551 return 0 2552 2553 2554@subcommand.epilog("""Example: 2555 gclient pack > patch.txt 2556 generate simple patch for configured client and dependences 2557""") 2558@metrics.collector.collect_metrics('gclient pack') 2559def CMDpack(parser, args): 2560 """Generates a patch which can be applied at the root of the tree. 2561 2562 Internally, runs 'git diff' on each checked out module and 2563 dependencies, and performs minimal postprocessing of the output. The 2564 resulting patch is printed to stdout and can be applied to a freshly 2565 checked out tree via 'patch -p0 < patchfile'. 2566 """ 2567 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', 2568 help='override deps for the specified (comma-separated) ' 2569 'platform(s); \'all\' will process all deps_os ' 2570 'references') 2571 parser.remove_option('--jobs') 2572 (options, args) = parser.parse_args(args) 2573 # Force jobs to 1 so the stdout is not annotated with the thread ids 2574 options.jobs = 1 2575 client = GClient.LoadCurrentConfig(options) 2576 if not client: 2577 raise gclient_utils.Error('client not configured; see \'gclient config\'') 2578 if options.verbose: 2579 client.PrintLocationAndContents() 2580 return client.RunOnDeps('pack', args) 2581 2582 2583@metrics.collector.collect_metrics('gclient status') 2584def CMDstatus(parser, args): 2585 """Shows modification status for every dependencies.""" 2586 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', 2587 help='override deps for the specified (comma-separated) ' 2588 'platform(s); \'all\' will process all deps_os ' 2589 'references') 2590 (options, args) = parser.parse_args(args) 2591 client = GClient.LoadCurrentConfig(options) 2592 if not client: 2593 raise gclient_utils.Error('client not configured; see \'gclient config\'') 2594 if options.verbose: 2595 client.PrintLocationAndContents() 2596 return client.RunOnDeps('status', args) 2597 2598 2599@subcommand.epilog("""Examples: 2600 gclient sync 2601 update files from SCM according to current configuration, 2602 *for modules which have changed since last update or sync* 2603 gclient sync --force 2604 update files from SCM according to current configuration, for 2605 all modules (useful for recovering files deleted from local copy) 2606 gclient sync --revision src@31000 2607 update src directory to r31000 2608 2609JSON output format: 2610If the --output-json option is specified, the following document structure will 2611be emitted to the provided file. 'null' entries may occur for subprojects which 2612are present in the gclient solution, but were not processed (due to custom_deps, 2613os_deps, etc.) 2614 2615{ 2616 "solutions" : { 2617 "<name>": { # <name> is the posix-normalized path to the solution. 2618 "revision": [<git id hex string>|null], 2619 "scm": ["git"|null], 2620 } 2621 } 2622} 2623""") 2624@metrics.collector.collect_metrics('gclient sync') 2625def CMDsync(parser, args): 2626 """Checkout/update all modules.""" 2627 parser.add_option('-f', '--force', action='store_true', 2628 help='force update even for unchanged modules') 2629 parser.add_option('-n', '--nohooks', action='store_true', 2630 help='don\'t run hooks after the update is complete') 2631 parser.add_option('-p', '--noprehooks', action='store_true', 2632 help='don\'t run pre-DEPS hooks', default=False) 2633 parser.add_option('-r', '--revision', action='append', 2634 dest='revisions', metavar='REV', default=[], 2635 help='Enforces revision/hash for the solutions with the ' 2636 'format src@rev. The src@ part is optional and can be ' 2637 'skipped. You can also specify URLs instead of paths ' 2638 'and gclient will find the solution corresponding to ' 2639 'the given URL. If a path is also specified, the URL ' 2640 'takes precedence. -r can be used multiple times when ' 2641 '.gclient has multiple solutions configured, and will ' 2642 'work even if the src@ part is skipped.') 2643 parser.add_option('--patch-ref', action='append', 2644 dest='patch_refs', metavar='GERRIT_REF', default=[], 2645 help='Patches the given reference with the format ' 2646 'dep@target-ref:patch-ref. ' 2647 'For |dep|, you can specify URLs as well as paths, ' 2648 'with URLs taking preference. ' 2649 '|patch-ref| will be applied to |dep|, rebased on top ' 2650 'of what |dep| was synced to, and a soft reset will ' 2651 'be done. Use --no-rebase-patch-ref and ' 2652 '--no-reset-patch-ref to disable this behavior. ' 2653 '|target-ref| is the target branch against which a ' 2654 'patch was created, it is used to determine which ' 2655 'commits from the |patch-ref| actually constitute a ' 2656 'patch.') 2657 parser.add_option('--with_branch_heads', action='store_true', 2658 help='Clone git "branch_heads" refspecs in addition to ' 2659 'the default refspecs. This adds about 1/2GB to a ' 2660 'full checkout. (git only)') 2661 parser.add_option('--with_tags', action='store_true', 2662 help='Clone git tags in addition to the default refspecs.') 2663 parser.add_option('-H', '--head', action='store_true', 2664 help='DEPRECATED: only made sense with safesync urls.') 2665 parser.add_option('-D', '--delete_unversioned_trees', action='store_true', 2666 help='Deletes from the working copy any dependencies that ' 2667 'have been removed since the last sync, as long as ' 2668 'there are no local modifications. When used with ' 2669 '--force, such dependencies are removed even if they ' 2670 'have local modifications. When used with --reset, ' 2671 'all untracked directories are removed from the ' 2672 'working copy, excluding those which are explicitly ' 2673 'ignored in the repository.') 2674 parser.add_option('-R', '--reset', action='store_true', 2675 help='resets any local changes before updating (git only)') 2676 parser.add_option('-M', '--merge', action='store_true', 2677 help='merge upstream changes instead of trying to ' 2678 'fast-forward or rebase') 2679 parser.add_option('-A', '--auto_rebase', action='store_true', 2680 help='Automatically rebase repositories against local ' 2681 'checkout during update (git only).') 2682 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', 2683 help='override deps for the specified (comma-separated) ' 2684 'platform(s); \'all\' will process all deps_os ' 2685 'references') 2686 parser.add_option('--process-all-deps', action='store_true', 2687 help='Check out all deps, even for different OS-es, ' 2688 'or with conditions evaluating to false') 2689 parser.add_option('--upstream', action='store_true', 2690 help='Make repo state match upstream branch.') 2691 parser.add_option('--output-json', 2692 help='Output a json document to this path containing ' 2693 'summary information about the sync.') 2694 parser.add_option('--no-history', action='store_true', 2695 help='GIT ONLY - Reduces the size/time of the checkout at ' 2696 'the cost of no history. Requires Git 1.9+') 2697 parser.add_option('--shallow', action='store_true', 2698 help='GIT ONLY - Do a shallow clone into the cache dir. ' 2699 'Requires Git 1.9+') 2700 parser.add_option('--no_bootstrap', '--no-bootstrap', 2701 action='store_true', 2702 help='Don\'t bootstrap from Google Storage.') 2703 parser.add_option('--ignore_locks', 2704 action='store_true', 2705 help='No longer used.') 2706 parser.add_option('--break_repo_locks', 2707 action='store_true', 2708 help='No longer used.') 2709 parser.add_option('--lock_timeout', type='int', default=5000, 2710 help='GIT ONLY - Deadline (in seconds) to wait for git ' 2711 'cache lock to become available. Default is %default.') 2712 parser.add_option('--no-rebase-patch-ref', action='store_false', 2713 dest='rebase_patch_ref', default=True, 2714 help='Bypass rebase of the patch ref after checkout.') 2715 parser.add_option('--no-reset-patch-ref', action='store_false', 2716 dest='reset_patch_ref', default=True, 2717 help='Bypass calling reset after patching the ref.') 2718 (options, args) = parser.parse_args(args) 2719 client = GClient.LoadCurrentConfig(options) 2720 2721 if not client: 2722 raise gclient_utils.Error('client not configured; see \'gclient config\'') 2723 2724 if options.ignore_locks: 2725 print('Warning: ignore_locks is no longer used. Please remove its usage.') 2726 2727 if options.break_repo_locks: 2728 print('Warning: break_repo_locks is no longer used. Please remove its ' 2729 'usage.') 2730 2731 if options.revisions and options.head: 2732 # TODO(maruel): Make it a parser.error if it doesn't break any builder. 2733 print('Warning: you cannot use both --head and --revision') 2734 2735 if options.verbose: 2736 client.PrintLocationAndContents() 2737 ret = client.RunOnDeps('update', args) 2738 if options.output_json: 2739 slns = {} 2740 for d in client.subtree(True): 2741 normed = d.name.replace('\\', '/').rstrip('/') + '/' 2742 slns[normed] = { 2743 'revision': d.got_revision, 2744 'scm': d.used_scm.name if d.used_scm else None, 2745 'url': str(d.url) if d.url else None, 2746 'was_processed': d.should_process, 2747 } 2748 with open(options.output_json, 'w') as f: 2749 json.dump({'solutions': slns}, f) 2750 return ret 2751 2752 2753CMDupdate = CMDsync 2754 2755 2756@metrics.collector.collect_metrics('gclient validate') 2757def CMDvalidate(parser, args): 2758 """Validates the .gclient and DEPS syntax.""" 2759 options, args = parser.parse_args(args) 2760 client = GClient.LoadCurrentConfig(options) 2761 rv = client.RunOnDeps('validate', args) 2762 if rv == 0: 2763 print('validate: SUCCESS') 2764 else: 2765 print('validate: FAILURE') 2766 return rv 2767 2768 2769@metrics.collector.collect_metrics('gclient diff') 2770def CMDdiff(parser, args): 2771 """Displays local diff for every dependencies.""" 2772 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', 2773 help='override deps for the specified (comma-separated) ' 2774 'platform(s); \'all\' will process all deps_os ' 2775 'references') 2776 (options, args) = parser.parse_args(args) 2777 client = GClient.LoadCurrentConfig(options) 2778 if not client: 2779 raise gclient_utils.Error('client not configured; see \'gclient config\'') 2780 if options.verbose: 2781 client.PrintLocationAndContents() 2782 return client.RunOnDeps('diff', args) 2783 2784 2785@metrics.collector.collect_metrics('gclient revert') 2786def CMDrevert(parser, args): 2787 """Reverts all modifications in every dependencies. 2788 2789 That's the nuclear option to get back to a 'clean' state. It removes anything 2790 that shows up in git status.""" 2791 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', 2792 help='override deps for the specified (comma-separated) ' 2793 'platform(s); \'all\' will process all deps_os ' 2794 'references') 2795 parser.add_option('-n', '--nohooks', action='store_true', 2796 help='don\'t run hooks after the revert is complete') 2797 parser.add_option('-p', '--noprehooks', action='store_true', 2798 help='don\'t run pre-DEPS hooks', default=False) 2799 parser.add_option('--upstream', action='store_true', 2800 help='Make repo state match upstream branch.') 2801 parser.add_option('--break_repo_locks', 2802 action='store_true', 2803 help='No longer used.') 2804 (options, args) = parser.parse_args(args) 2805 if options.break_repo_locks: 2806 print('Warning: break_repo_locks is no longer used. Please remove its ' + 2807 'usage.') 2808 2809 # --force is implied. 2810 options.force = True 2811 options.reset = False 2812 options.delete_unversioned_trees = False 2813 options.merge = False 2814 client = GClient.LoadCurrentConfig(options) 2815 if not client: 2816 raise gclient_utils.Error('client not configured; see \'gclient config\'') 2817 return client.RunOnDeps('revert', args) 2818 2819 2820@metrics.collector.collect_metrics('gclient runhooks') 2821def CMDrunhooks(parser, args): 2822 """Runs hooks for files that have been modified in the local working copy.""" 2823 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', 2824 help='override deps for the specified (comma-separated) ' 2825 'platform(s); \'all\' will process all deps_os ' 2826 'references') 2827 parser.add_option('-f', '--force', action='store_true', default=True, 2828 help='Deprecated. No effect.') 2829 (options, args) = parser.parse_args(args) 2830 client = GClient.LoadCurrentConfig(options) 2831 if not client: 2832 raise gclient_utils.Error('client not configured; see \'gclient config\'') 2833 if options.verbose: 2834 client.PrintLocationAndContents() 2835 options.force = True 2836 options.nohooks = False 2837 return client.RunOnDeps('runhooks', args) 2838 2839 2840@metrics.collector.collect_metrics('gclient revinfo') 2841def CMDrevinfo(parser, args): 2842 """Outputs revision info mapping for the client and its dependencies. 2843 2844 This allows the capture of an overall 'revision' for the source tree that 2845 can be used to reproduce the same tree in the future. It is only useful for 2846 'unpinned dependencies', i.e. DEPS/deps references without a git hash. 2847 A git branch name isn't 'pinned' since the actual commit can change. 2848 """ 2849 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', 2850 help='override deps for the specified (comma-separated) ' 2851 'platform(s); \'all\' will process all deps_os ' 2852 'references') 2853 parser.add_option('-a', '--actual', action='store_true', 2854 help='gets the actual checked out revisions instead of the ' 2855 'ones specified in the DEPS and .gclient files') 2856 parser.add_option('-s', '--snapshot', action='store_true', 2857 help='creates a snapshot .gclient file of the current ' 2858 'version of all repositories to reproduce the tree, ' 2859 'implies -a') 2860 parser.add_option('--filter', action='append', dest='filter', 2861 help='Display revision information only for the specified ' 2862 'dependencies (filtered by URL or path).') 2863 parser.add_option('--output-json', 2864 help='Output a json document to this path containing ' 2865 'information about the revisions.') 2866 parser.add_option('--ignore-dep-type', choices=['git', 'cipd'], 2867 help='Specify to skip processing of a certain type of dep.') 2868 (options, args) = parser.parse_args(args) 2869 client = GClient.LoadCurrentConfig(options) 2870 if not client: 2871 raise gclient_utils.Error('client not configured; see \'gclient config\'') 2872 client.PrintRevInfo() 2873 return 0 2874 2875 2876@metrics.collector.collect_metrics('gclient getdep') 2877def CMDgetdep(parser, args): 2878 """Gets revision information and variable values from a DEPS file.""" 2879 parser.add_option('--var', action='append', 2880 dest='vars', metavar='VAR', default=[], 2881 help='Gets the value of a given variable.') 2882 parser.add_option('-r', '--revision', action='append', 2883 dest='getdep_revisions', metavar='DEP', default=[], 2884 help='Gets the revision/version for the given dependency. ' 2885 'If it is a git dependency, dep must be a path. If it ' 2886 'is a CIPD dependency, dep must be of the form ' 2887 'path:package.') 2888 parser.add_option('--deps-file', default='DEPS', 2889 # TODO(ehmaldonado): Try to find the DEPS file pointed by 2890 # .gclient first. 2891 help='The DEPS file to be edited. Defaults to the DEPS ' 2892 'file in the current directory.') 2893 (options, args) = parser.parse_args(args) 2894 2895 if not os.path.isfile(options.deps_file): 2896 raise gclient_utils.Error( 2897 'DEPS file %s does not exist.' % options.deps_file) 2898 with open(options.deps_file) as f: 2899 contents = f.read() 2900 client = GClient.LoadCurrentConfig(options) 2901 if client is not None: 2902 builtin_vars = client.get_builtin_vars() 2903 else: 2904 logging.warning( 2905 'Couldn\'t find a valid gclient config. Will attempt to parse the DEPS ' 2906 'file without support for built-in variables.') 2907 builtin_vars = None 2908 local_scope = gclient_eval.Exec(contents, options.deps_file, 2909 builtin_vars=builtin_vars) 2910 2911 for var in options.vars: 2912 print(gclient_eval.GetVar(local_scope, var)) 2913 2914 for name in options.getdep_revisions: 2915 if ':' in name: 2916 name, _, package = name.partition(':') 2917 if not name or not package: 2918 parser.error( 2919 'Wrong CIPD format: %s:%s should be of the form path:pkg.' 2920 % (name, package)) 2921 print(gclient_eval.GetCIPD(local_scope, name, package)) 2922 else: 2923 print(gclient_eval.GetRevision(local_scope, name)) 2924 2925 2926@metrics.collector.collect_metrics('gclient setdep') 2927def CMDsetdep(parser, args): 2928 """Modifies dependency revisions and variable values in a DEPS file""" 2929 parser.add_option('--var', action='append', 2930 dest='vars', metavar='VAR=VAL', default=[], 2931 help='Sets a variable to the given value with the format ' 2932 'name=value.') 2933 parser.add_option('-r', '--revision', action='append', 2934 dest='setdep_revisions', metavar='DEP@REV', default=[], 2935 help='Sets the revision/version for the dependency with ' 2936 'the format dep@rev. If it is a git dependency, dep ' 2937 'must be a path and rev must be a git hash or ' 2938 'reference (e.g. src/dep@deadbeef). If it is a CIPD ' 2939 'dependency, dep must be of the form path:package and ' 2940 'rev must be the package version ' 2941 '(e.g. src/pkg:chromium/pkg@2.1-cr0).') 2942 parser.add_option('--deps-file', default='DEPS', 2943 # TODO(ehmaldonado): Try to find the DEPS file pointed by 2944 # .gclient first. 2945 help='The DEPS file to be edited. Defaults to the DEPS ' 2946 'file in the current directory.') 2947 (options, args) = parser.parse_args(args) 2948 if args: 2949 parser.error('Unused arguments: "%s"' % '" "'.join(args)) 2950 if not options.setdep_revisions and not options.vars: 2951 parser.error( 2952 'You must specify at least one variable or revision to modify.') 2953 2954 if not os.path.isfile(options.deps_file): 2955 raise gclient_utils.Error( 2956 'DEPS file %s does not exist.' % options.deps_file) 2957 with open(options.deps_file) as f: 2958 contents = f.read() 2959 2960 client = GClient.LoadCurrentConfig(options) 2961 if client is not None: 2962 builtin_vars = client.get_builtin_vars() 2963 else: 2964 logging.warning( 2965 'Couldn\'t find a valid gclient config. Will attempt to parse the DEPS ' 2966 'file without support for built-in variables.') 2967 builtin_vars = None 2968 2969 local_scope = gclient_eval.Exec(contents, options.deps_file, 2970 builtin_vars=builtin_vars) 2971 2972 for var in options.vars: 2973 name, _, value = var.partition('=') 2974 if not name or not value: 2975 parser.error( 2976 'Wrong var format: %s should be of the form name=value.' % var) 2977 if name in local_scope['vars']: 2978 gclient_eval.SetVar(local_scope, name, value) 2979 else: 2980 gclient_eval.AddVar(local_scope, name, value) 2981 2982 for revision in options.setdep_revisions: 2983 name, _, value = revision.partition('@') 2984 if not name or not value: 2985 parser.error( 2986 'Wrong dep format: %s should be of the form dep@rev.' % revision) 2987 if ':' in name: 2988 name, _, package = name.partition(':') 2989 if not name or not package: 2990 parser.error( 2991 'Wrong CIPD format: %s:%s should be of the form path:pkg@version.' 2992 % (name, package)) 2993 gclient_eval.SetCIPD(local_scope, name, package, value) 2994 else: 2995 gclient_eval.SetRevision(local_scope, name, value) 2996 2997 with open(options.deps_file, 'wb') as f: 2998 f.write(gclient_eval.RenderDEPSFile(local_scope).encode('utf-8')) 2999 3000 3001@metrics.collector.collect_metrics('gclient verify') 3002def CMDverify(parser, args): 3003 """Verifies the DEPS file deps are only from allowed_hosts.""" 3004 (options, args) = parser.parse_args(args) 3005 client = GClient.LoadCurrentConfig(options) 3006 if not client: 3007 raise gclient_utils.Error('client not configured; see \'gclient config\'') 3008 client.RunOnDeps(None, []) 3009 # Look at each first-level dependency of this gclient only. 3010 for dep in client.dependencies: 3011 bad_deps = dep.findDepsFromNotAllowedHosts() 3012 if not bad_deps: 3013 continue 3014 print("There are deps from not allowed hosts in file %s" % dep.deps_file) 3015 for bad_dep in bad_deps: 3016 print("\t%s at %s" % (bad_dep.name, bad_dep.url)) 3017 print("allowed_hosts:", ', '.join(dep.allowed_hosts)) 3018 sys.stdout.flush() 3019 raise gclient_utils.Error( 3020 'dependencies from disallowed hosts; check your DEPS file.') 3021 return 0 3022 3023 3024@subcommand.epilog("""For more information on what metrics are we collecting and 3025why, please read metrics.README.md or visit https://bit.ly/2ufRS4p""") 3026@metrics.collector.collect_metrics('gclient metrics') 3027def CMDmetrics(parser, args): 3028 """Reports, and optionally modifies, the status of metric collection.""" 3029 parser.add_option('--opt-in', action='store_true', dest='enable_metrics', 3030 help='Opt-in to metrics collection.', 3031 default=None) 3032 parser.add_option('--opt-out', action='store_false', dest='enable_metrics', 3033 help='Opt-out of metrics collection.') 3034 options, args = parser.parse_args(args) 3035 if args: 3036 parser.error('Unused arguments: "%s"' % '" "'.join(args)) 3037 if not metrics.collector.config.is_googler: 3038 print("You're not a Googler. Metrics collection is disabled for you.") 3039 return 0 3040 3041 if options.enable_metrics is not None: 3042 metrics.collector.config.opted_in = options.enable_metrics 3043 3044 if metrics.collector.config.opted_in is None: 3045 print("You haven't opted in or out of metrics collection.") 3046 elif metrics.collector.config.opted_in: 3047 print("You have opted in. Thanks!") 3048 else: 3049 print("You have opted out. Please consider opting in.") 3050 return 0 3051 3052 3053class OptionParser(optparse.OptionParser): 3054 gclientfile_default = os.environ.get('GCLIENT_FILE', '.gclient') 3055 3056 def __init__(self, **kwargs): 3057 optparse.OptionParser.__init__( 3058 self, version='%prog ' + __version__, **kwargs) 3059 3060 # Some arm boards have issues with parallel sync. 3061 if platform.machine().startswith('arm'): 3062 jobs = 1 3063 else: 3064 jobs = max(8, gclient_utils.NumLocalCpus()) 3065 3066 self.add_option( 3067 '-j', '--jobs', default=jobs, type='int', 3068 help='Specify how many SCM commands can run in parallel; defaults to ' 3069 '%default on this machine') 3070 self.add_option( 3071 '-v', '--verbose', action='count', default=0, 3072 help='Produces additional output for diagnostics. Can be used up to ' 3073 'three times for more logging info.') 3074 self.add_option( 3075 '--gclientfile', dest='config_filename', 3076 help='Specify an alternate %s file' % self.gclientfile_default) 3077 self.add_option( 3078 '--spec', 3079 help='create a gclient file containing the provided string. Due to ' 3080 'Cygwin/Python brokenness, it can\'t contain any newlines.') 3081 self.add_option( 3082 '--no-nag-max', default=False, action='store_true', 3083 help='Ignored for backwards compatibility.') 3084 3085 def parse_args(self, args=None, _values=None): 3086 """Integrates standard options processing.""" 3087 # Create an optparse.Values object that will store only the actual passed 3088 # options, without the defaults. 3089 actual_options = optparse.Values() 3090 _, args = optparse.OptionParser.parse_args(self, args, actual_options) 3091 # Create an optparse.Values object with the default options. 3092 options = optparse.Values(self.get_default_values().__dict__) 3093 # Update it with the options passed by the user. 3094 options._update_careful(actual_options.__dict__) 3095 # Store the options passed by the user in an _actual_options attribute. 3096 # We store only the keys, and not the values, since the values can contain 3097 # arbitrary information, which might be PII. 3098 metrics.collector.add('arguments', list(actual_options.__dict__)) 3099 3100 levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] 3101 logging.basicConfig( 3102 level=levels[min(options.verbose, len(levels) - 1)], 3103 format='%(module)s(%(lineno)d) %(funcName)s:%(message)s') 3104 if options.config_filename and options.spec: 3105 self.error('Cannot specify both --gclientfile and --spec') 3106 if (options.config_filename and 3107 options.config_filename != os.path.basename(options.config_filename)): 3108 self.error('--gclientfile target must be a filename, not a path') 3109 if not options.config_filename: 3110 options.config_filename = self.gclientfile_default 3111 options.entries_filename = options.config_filename + '_entries' 3112 if options.jobs < 1: 3113 self.error('--jobs must be 1 or higher') 3114 3115 # These hacks need to die. 3116 if not hasattr(options, 'revisions'): 3117 # GClient.RunOnDeps expects it even if not applicable. 3118 options.revisions = [] 3119 if not hasattr(options, 'head'): 3120 options.head = None 3121 if not hasattr(options, 'nohooks'): 3122 options.nohooks = True 3123 if not hasattr(options, 'noprehooks'): 3124 options.noprehooks = True 3125 if not hasattr(options, 'deps_os'): 3126 options.deps_os = None 3127 if not hasattr(options, 'force'): 3128 options.force = None 3129 return (options, args) 3130 3131 3132def disable_buffering(): 3133 # Make stdout auto-flush so buildbot doesn't kill us during lengthy 3134 # operations. Python as a strong tendency to buffer sys.stdout. 3135 sys.stdout = gclient_utils.MakeFileAutoFlush(sys.stdout) 3136 # Make stdout annotated with the thread ids. 3137 sys.stdout = gclient_utils.MakeFileAnnotated(sys.stdout) 3138 3139 3140def path_contains_tilde(): 3141 for element in os.environ['PATH'].split(os.pathsep): 3142 if element.startswith('~') and os.path.abspath( 3143 os.path.realpath(os.path.expanduser(element))) == DEPOT_TOOLS_DIR: 3144 return True 3145 return False 3146 3147 3148def can_run_gclient_and_helpers(): 3149 if sys.hexversion < 0x02060000: 3150 print( 3151 '\nYour python version %s is unsupported, please upgrade.\n' % 3152 sys.version.split(' ', 1)[0], 3153 file=sys.stderr) 3154 return False 3155 if not sys.executable: 3156 print( 3157 '\nPython cannot find the location of it\'s own executable.\n', 3158 file=sys.stderr) 3159 return False 3160 if path_contains_tilde(): 3161 print( 3162 '\nYour PATH contains a literal "~", which works in some shells ' + 3163 'but will break when python tries to run subprocesses. ' + 3164 'Replace the "~" with $HOME.\n' + 3165 'See https://crbug.com/952865.\n', 3166 file=sys.stderr) 3167 return False 3168 return True 3169 3170 3171def main(argv): 3172 """Doesn't parse the arguments here, just find the right subcommand to 3173 execute.""" 3174 if not can_run_gclient_and_helpers(): 3175 return 2 3176 fix_encoding.fix_encoding() 3177 disable_buffering() 3178 setup_color.init() 3179 dispatcher = subcommand.CommandDispatcher(__name__) 3180 try: 3181 return dispatcher.execute(OptionParser(), argv) 3182 except KeyboardInterrupt: 3183 gclient_utils.GClientChildren.KillAllRemainingChildren() 3184 raise 3185 except (gclient_utils.Error, subprocess2.CalledProcessError) as e: 3186 print('Error: %s' % str(e), file=sys.stderr) 3187 return 1 3188 finally: 3189 gclient_utils.PrintWarnings() 3190 return 0 3191 3192 3193if '__main__' == __name__: 3194 with metrics.collector.print_notice_and_exit(): 3195 sys.exit(main(sys.argv[1:])) 3196 3197# vim: ts=2:sw=2:tw=80:et: 3198