1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Gclient-specific SCM-specific operations.""" 6 7from __future__ import print_function 8 9import collections 10import contextlib 11import errno 12import json 13import logging 14import os 15import posixpath 16import re 17import sys 18import tempfile 19import threading 20import traceback 21 22try: 23 import urlparse 24except ImportError: # For Py3 compatibility 25 import urllib.parse as urlparse 26 27import gclient_utils 28import git_cache 29import scm 30import shutil 31import subprocess2 32 33 34THIS_FILE_PATH = os.path.abspath(__file__) 35 36GSUTIL_DEFAULT_PATH = os.path.join( 37 os.path.dirname(os.path.abspath(__file__)), 'gsutil.py') 38 39 40class NoUsableRevError(gclient_utils.Error): 41 """Raised if requested revision isn't found in checkout.""" 42 43 44class DiffFiltererWrapper(object): 45 """Simple base class which tracks which file is being diffed and 46 replaces instances of its file name in the original and 47 working copy lines of the git diff output.""" 48 index_string = None 49 original_prefix = "--- " 50 working_prefix = "+++ " 51 52 def __init__(self, relpath, print_func): 53 # Note that we always use '/' as the path separator to be 54 # consistent with cygwin-style output on Windows 55 self._relpath = relpath.replace("\\", "/") 56 self._current_file = None 57 self._print_func = print_func 58 59 def SetCurrentFile(self, current_file): 60 self._current_file = current_file 61 62 @property 63 def _replacement_file(self): 64 return posixpath.join(self._relpath, self._current_file) 65 66 def _Replace(self, line): 67 return line.replace(self._current_file, self._replacement_file) 68 69 def Filter(self, line): 70 if (line.startswith(self.index_string)): 71 self.SetCurrentFile(line[len(self.index_string):]) 72 line = self._Replace(line) 73 else: 74 if (line.startswith(self.original_prefix) or 75 line.startswith(self.working_prefix)): 76 line = self._Replace(line) 77 self._print_func(line) 78 79 80class GitDiffFilterer(DiffFiltererWrapper): 81 index_string = "diff --git " 82 83 def SetCurrentFile(self, current_file): 84 # Get filename by parsing "a/<filename> b/<filename>" 85 self._current_file = current_file[:(len(current_file)/2)][2:] 86 87 def _Replace(self, line): 88 return re.sub("[a|b]/" + self._current_file, self._replacement_file, line) 89 90 91# SCMWrapper base class 92 93class SCMWrapper(object): 94 """Add necessary glue between all the supported SCM. 95 96 This is the abstraction layer to bind to different SCM. 97 """ 98 def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None, 99 out_cb=None, print_outbuf=False): 100 self.url = url 101 self._root_dir = root_dir 102 if self._root_dir: 103 self._root_dir = self._root_dir.replace('/', os.sep) 104 self.relpath = relpath 105 if self.relpath: 106 self.relpath = self.relpath.replace('/', os.sep) 107 if self.relpath and self._root_dir: 108 self.checkout_path = os.path.join(self._root_dir, self.relpath) 109 if out_fh is None: 110 out_fh = sys.stdout 111 self.out_fh = out_fh 112 self.out_cb = out_cb 113 self.print_outbuf = print_outbuf 114 115 def Print(self, *args, **kwargs): 116 kwargs.setdefault('file', self.out_fh) 117 if kwargs.pop('timestamp', True): 118 self.out_fh.write('[%s] ' % gclient_utils.Elapsed()) 119 print(*args, **kwargs) 120 121 def RunCommand(self, command, options, args, file_list=None): 122 commands = ['update', 'updatesingle', 'revert', 123 'revinfo', 'status', 'diff', 'pack', 'runhooks'] 124 125 if not command in commands: 126 raise gclient_utils.Error('Unknown command %s' % command) 127 128 if not command in dir(self): 129 raise gclient_utils.Error('Command %s not implemented in %s wrapper' % ( 130 command, self.__class__.__name__)) 131 132 return getattr(self, command)(options, args, file_list) 133 134 @staticmethod 135 def _get_first_remote_url(checkout_path): 136 log = scm.GIT.Capture( 137 ['config', '--local', '--get-regexp', r'remote.*.url'], 138 cwd=checkout_path) 139 # Get the second token of the first line of the log. 140 return log.splitlines()[0].split(' ', 1)[1] 141 142 def GetCacheMirror(self): 143 if getattr(self, 'cache_dir', None): 144 url, _ = gclient_utils.SplitUrlRevision(self.url) 145 return git_cache.Mirror(url) 146 return None 147 148 def GetActualRemoteURL(self, options): 149 """Attempt to determine the remote URL for this SCMWrapper.""" 150 # Git 151 if os.path.exists(os.path.join(self.checkout_path, '.git')): 152 actual_remote_url = self._get_first_remote_url(self.checkout_path) 153 154 mirror = self.GetCacheMirror() 155 # If the cache is used, obtain the actual remote URL from there. 156 if (mirror and mirror.exists() and 157 mirror.mirror_path.replace('\\', '/') == 158 actual_remote_url.replace('\\', '/')): 159 actual_remote_url = self._get_first_remote_url(mirror.mirror_path) 160 return actual_remote_url 161 return None 162 163 def DoesRemoteURLMatch(self, options): 164 """Determine whether the remote URL of this checkout is the expected URL.""" 165 if not os.path.exists(self.checkout_path): 166 # A checkout which doesn't exist can't be broken. 167 return True 168 169 actual_remote_url = self.GetActualRemoteURL(options) 170 if actual_remote_url: 171 return (gclient_utils.SplitUrlRevision(actual_remote_url)[0].rstrip('/') 172 == gclient_utils.SplitUrlRevision(self.url)[0].rstrip('/')) 173 else: 174 # This may occur if the self.checkout_path exists but does not contain a 175 # valid git checkout. 176 return False 177 178 def _DeleteOrMove(self, force): 179 """Delete the checkout directory or move it out of the way. 180 181 Args: 182 force: bool; if True, delete the directory. Otherwise, just move it. 183 """ 184 if force and os.environ.get('CHROME_HEADLESS') == '1': 185 self.Print('_____ Conflicting directory found in %s. Removing.' 186 % self.checkout_path) 187 gclient_utils.AddWarning('Conflicting directory %s deleted.' 188 % self.checkout_path) 189 gclient_utils.rmtree(self.checkout_path) 190 else: 191 bad_scm_dir = os.path.join(self._root_dir, '_bad_scm', 192 os.path.dirname(self.relpath)) 193 194 try: 195 os.makedirs(bad_scm_dir) 196 except OSError as e: 197 if e.errno != errno.EEXIST: 198 raise 199 200 dest_path = tempfile.mkdtemp( 201 prefix=os.path.basename(self.relpath), 202 dir=bad_scm_dir) 203 self.Print('_____ Conflicting directory found in %s. Moving to %s.' 204 % (self.checkout_path, dest_path)) 205 gclient_utils.AddWarning('Conflicting directory %s moved to %s.' 206 % (self.checkout_path, dest_path)) 207 shutil.move(self.checkout_path, dest_path) 208 209 210class GitWrapper(SCMWrapper): 211 """Wrapper for Git""" 212 name = 'git' 213 remote = 'origin' 214 215 @property 216 def cache_dir(self): 217 try: 218 return git_cache.Mirror.GetCachePath() 219 except RuntimeError: 220 return None 221 222 def __init__(self, url=None, *args, **kwargs): 223 """Removes 'git+' fake prefix from git URL.""" 224 if url and (url.startswith('git+http://') or 225 url.startswith('git+https://')): 226 url = url[4:] 227 SCMWrapper.__init__(self, url, *args, **kwargs) 228 filter_kwargs = { 'time_throttle': 1, 'out_fh': self.out_fh } 229 if self.out_cb: 230 filter_kwargs['predicate'] = self.out_cb 231 self.filter = gclient_utils.GitFilter(**filter_kwargs) 232 self._running_under_rosetta = None 233 234 def GetCheckoutRoot(self): 235 return scm.GIT.GetCheckoutRoot(self.checkout_path) 236 237 def GetRevisionDate(self, _revision): 238 """Returns the given revision's date in ISO-8601 format (which contains the 239 time zone).""" 240 # TODO(floitsch): get the time-stamp of the given revision and not just the 241 # time-stamp of the currently checked out revision. 242 return self._Capture(['log', '-n', '1', '--format=%ai']) 243 244 def _GetDiffFilenames(self, base): 245 """Returns the names of files modified since base.""" 246 return self._Capture( 247 # Filter to remove base if it is None. 248 list(filter(bool, ['-c', 'core.quotePath=false', 'diff', '--name-only', 249 base]) 250 )).split() 251 252 def diff(self, options, _args, _file_list): 253 _, revision = gclient_utils.SplitUrlRevision(self.url) 254 if not revision: 255 revision = 'refs/remotes/%s/master' % self.remote 256 self._Run(['-c', 'core.quotePath=false', 'diff', revision], options) 257 258 def pack(self, _options, _args, _file_list): 259 """Generates a patch file which can be applied to the root of the 260 repository. 261 262 The patch file is generated from a diff of the merge base of HEAD and 263 its upstream branch. 264 """ 265 try: 266 merge_base = [self._Capture(['merge-base', 'HEAD', self.remote])] 267 except subprocess2.CalledProcessError: 268 merge_base = [] 269 gclient_utils.CheckCallAndFilter( 270 ['git', 'diff'] + merge_base, 271 cwd=self.checkout_path, 272 filter_fn=GitDiffFilterer(self.relpath, print_func=self.Print).Filter) 273 274 def _Scrub(self, target, options): 275 """Scrubs out all changes in the local repo, back to the state of target.""" 276 quiet = [] 277 if not options.verbose: 278 quiet = ['--quiet'] 279 self._Run(['reset', '--hard', target] + quiet, options) 280 if options.force and options.delete_unversioned_trees: 281 # where `target` is a commit that contains both upper and lower case 282 # versions of the same file on a case insensitive filesystem, we are 283 # actually in a broken state here. The index will have both 'a' and 'A', 284 # but only one of them will exist on the disk. To progress, we delete 285 # everything that status thinks is modified. 286 output = self._Capture([ 287 '-c', 'core.quotePath=false', 'status', '--porcelain'], strip=False) 288 for line in output.splitlines(): 289 # --porcelain (v1) looks like: 290 # XY filename 291 try: 292 filename = line[3:] 293 self.Print('_____ Deleting residual after reset: %r.' % filename) 294 gclient_utils.rm_file_or_tree( 295 os.path.join(self.checkout_path, filename)) 296 except OSError: 297 pass 298 299 def _FetchAndReset(self, revision, file_list, options): 300 """Equivalent to git fetch; git reset.""" 301 self._SetFetchConfig(options) 302 303 self._Fetch(options, prune=True, quiet=options.verbose) 304 self._Scrub(revision, options) 305 if file_list is not None: 306 files = self._Capture( 307 ['-c', 'core.quotePath=false', 'ls-files']).splitlines() 308 file_list.extend( 309 [os.path.join(self.checkout_path, f) for f in files]) 310 311 def _DisableHooks(self): 312 hook_dir = os.path.join(self.checkout_path, '.git', 'hooks') 313 if not os.path.isdir(hook_dir): 314 return 315 for f in os.listdir(hook_dir): 316 if not f.endswith('.sample') and not f.endswith('.disabled'): 317 disabled_hook_path = os.path.join(hook_dir, f + '.disabled') 318 if os.path.exists(disabled_hook_path): 319 os.remove(disabled_hook_path) 320 os.rename(os.path.join(hook_dir, f), disabled_hook_path) 321 322 def _maybe_break_locks(self, options): 323 """This removes all .lock files from this repo's .git directory, if the 324 user passed the --break_repo_locks command line flag. 325 326 In particular, this will cleanup index.lock files, as well as ref lock 327 files. 328 """ 329 if options.break_repo_locks: 330 git_dir = os.path.join(self.checkout_path, '.git') 331 for path, _, filenames in os.walk(git_dir): 332 for filename in filenames: 333 if filename.endswith('.lock'): 334 to_break = os.path.join(path, filename) 335 self.Print('breaking lock: %s' % (to_break,)) 336 try: 337 os.remove(to_break) 338 except OSError as ex: 339 self.Print('FAILED to break lock: %s: %s' % (to_break, ex)) 340 raise 341 342 def apply_patch_ref(self, patch_repo, patch_rev, target_rev, options, 343 file_list): 344 """Apply a patch on top of the revision we're synced at. 345 346 The patch ref is given by |patch_repo|@|patch_rev|. 347 |target_rev| is usually the branch that the |patch_rev| was uploaded against 348 (e.g. 'refs/heads/master'), but this is not required. 349 350 We cherry-pick all commits reachable from |patch_rev| on top of the curret 351 HEAD, excluding those reachable from |target_rev| 352 (i.e. git cherry-pick target_rev..patch_rev). 353 354 Graphically, it looks like this: 355 356 ... -> o -> [possibly already landed commits] -> target_rev 357 \ 358 -> [possibly not yet landed dependent CLs] -> patch_rev 359 360 The final checkout state is then: 361 362 ... -> HEAD -> [possibly not yet landed dependent CLs] -> patch_rev 363 364 After application, if |options.reset_patch_ref| is specified, we soft reset 365 the cherry-picked changes, keeping them in git index only. 366 367 Args: 368 patch_repo: The patch origin. 369 e.g. 'https://foo.googlesource.com/bar' 370 patch_rev: The revision to patch. 371 e.g. 'refs/changes/1234/34/1'. 372 target_rev: The revision to use when finding the merge base. 373 Typically, the branch that the patch was uploaded against. 374 e.g. 'refs/heads/master' or 'refs/heads/infra/config'. 375 options: The options passed to gclient. 376 file_list: A list where modified files will be appended. 377 """ 378 379 # Abort any cherry-picks in progress. 380 try: 381 self._Capture(['cherry-pick', '--abort']) 382 except subprocess2.CalledProcessError: 383 pass 384 385 base_rev = self._Capture(['rev-parse', 'HEAD']) 386 387 if not target_rev: 388 raise gclient_utils.Error('A target revision for the patch must be given') 389 elif target_rev.startswith(('refs/heads/', 'refs/branch-heads')): 390 # If |target_rev| is in refs/heads/** or refs/branch-heads/**, try first 391 # to find the corresponding remote ref for it, since |target_rev| might 392 # point to a local ref which is not up to date with the corresponding 393 # remote ref. 394 remote_ref = ''.join(scm.GIT.RefToRemoteRef(target_rev, self.remote)) 395 self.Print('Trying the corresponding remote ref for %r: %r\n' % ( 396 target_rev, remote_ref)) 397 if scm.GIT.IsValidRevision(self.checkout_path, remote_ref): 398 target_rev = remote_ref 399 elif not scm.GIT.IsValidRevision(self.checkout_path, target_rev): 400 # Fetch |target_rev| if it's not already available. 401 url, _ = gclient_utils.SplitUrlRevision(self.url) 402 mirror = self._GetMirror(url, options, target_rev) 403 if mirror: 404 rev_type = 'branch' if target_rev.startswith('refs/') else 'hash' 405 self._UpdateMirrorIfNotContains(mirror, options, rev_type, target_rev) 406 self._Fetch(options, refspec=target_rev) 407 408 self.Print('===Applying patch===') 409 self.Print('Revision to patch is %r @ %r.' % (patch_repo, patch_rev)) 410 self.Print('Current dir is %r' % self.checkout_path) 411 self._Capture(['reset', '--hard']) 412 self._Capture(['fetch', '--no-tags', patch_repo, patch_rev]) 413 patch_rev = self._Capture(['rev-parse', 'FETCH_HEAD']) 414 415 if not options.rebase_patch_ref: 416 self._Capture(['checkout', patch_rev]) 417 # Adjust base_rev to be the first parent of our checked out patch ref; 418 # This will allow us to correctly extend `file_list`, and will show the 419 # correct file-list to programs which do `git diff --cached` expecting to 420 # see the patch diff. 421 base_rev = self._Capture(['rev-parse', patch_rev+'~']) 422 423 else: 424 self.Print('Will cherrypick %r .. %r on top of %r.' % ( 425 target_rev, patch_rev, base_rev)) 426 try: 427 if scm.GIT.IsAncestor(self.checkout_path, patch_rev, target_rev): 428 # If |patch_rev| is an ancestor of |target_rev|, check it out. 429 self._Capture(['checkout', patch_rev]) 430 else: 431 # If a change was uploaded on top of another change, which has already 432 # landed, one of the commits in the cherry-pick range will be 433 # redundant, since it has already landed and its changes incorporated 434 # in the tree. 435 # We pass '--keep-redundant-commits' to ignore those changes. 436 self._Capture(['cherry-pick', target_rev + '..' + patch_rev, 437 '--keep-redundant-commits']) 438 439 except subprocess2.CalledProcessError as e: 440 self.Print('Failed to apply patch.') 441 self.Print('Revision to patch was %r @ %r.' % (patch_repo, patch_rev)) 442 self.Print('Tried to cherrypick %r .. %r on top of %r.' % ( 443 target_rev, patch_rev, base_rev)) 444 self.Print('Current dir is %r' % self.checkout_path) 445 self.Print('git returned non-zero exit status %s:\n%s' % ( 446 e.returncode, e.stderr.decode('utf-8'))) 447 # Print the current status so that developers know what changes caused 448 # the patch failure, since git cherry-pick doesn't show that 449 # information. 450 self.Print(self._Capture(['status'])) 451 try: 452 self._Capture(['cherry-pick', '--abort']) 453 except subprocess2.CalledProcessError: 454 pass 455 raise 456 457 if file_list is not None: 458 file_list.extend(self._GetDiffFilenames(base_rev)) 459 460 if options.reset_patch_ref: 461 self._Capture(['reset', '--soft', base_rev]) 462 463 def update(self, options, args, file_list): 464 """Runs git to update or transparently checkout the working copy. 465 466 All updated files will be appended to file_list. 467 468 Raises: 469 Error: if can't get URL for relative path. 470 """ 471 if args: 472 raise gclient_utils.Error("Unsupported argument(s): %s" % ",".join(args)) 473 474 self._CheckMinVersion("1.6.6") 475 476 # If a dependency is not pinned, track the default remote branch. 477 default_rev = 'refs/remotes/%s/master' % self.remote 478 url, deps_revision = gclient_utils.SplitUrlRevision(self.url) 479 revision = deps_revision 480 managed = True 481 if options.revision: 482 # Override the revision number. 483 revision = str(options.revision) 484 if revision == 'unmanaged': 485 # Check again for a revision in case an initial ref was specified 486 # in the url, for example bla.git@refs/heads/custombranch 487 revision = deps_revision 488 managed = False 489 if not revision: 490 revision = default_rev 491 492 if managed: 493 self._DisableHooks() 494 495 printed_path = False 496 verbose = [] 497 if options.verbose: 498 self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False) 499 verbose = ['--verbose'] 500 printed_path = True 501 502 revision_ref = revision 503 if ':' in revision: 504 revision_ref, _, revision = revision.partition(':') 505 506 if revision_ref.startswith('refs/branch-heads'): 507 options.with_branch_heads = True 508 509 mirror = self._GetMirror(url, options, revision_ref) 510 if mirror: 511 url = mirror.mirror_path 512 513 remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) 514 if remote_ref: 515 # Rewrite remote refs to their local equivalents. 516 revision = ''.join(remote_ref) 517 rev_type = "branch" 518 elif revision.startswith('refs/'): 519 # Local branch? We probably don't want to support, since DEPS should 520 # always specify branches as they are in the upstream repo. 521 rev_type = "branch" 522 else: 523 # hash is also a tag, only make a distinction at checkout 524 rev_type = "hash" 525 526 # If we are going to introduce a new project, there is a possibility that 527 # we are syncing back to a state where the project was originally a 528 # sub-project rolled by DEPS (realistic case: crossing the Blink merge point 529 # syncing backwards, when Blink was a DEPS entry and not part of src.git). 530 # In such case, we might have a backup of the former .git folder, which can 531 # be used to avoid re-fetching the entire repo again (useful for bisects). 532 backup_dir = self.GetGitBackupDirPath() 533 target_dir = os.path.join(self.checkout_path, '.git') 534 if os.path.exists(backup_dir) and not os.path.exists(target_dir): 535 gclient_utils.safe_makedirs(self.checkout_path) 536 os.rename(backup_dir, target_dir) 537 # Reset to a clean state 538 self._Scrub('HEAD', options) 539 540 if (not os.path.exists(self.checkout_path) or 541 (os.path.isdir(self.checkout_path) and 542 not os.path.exists(os.path.join(self.checkout_path, '.git')))): 543 if mirror: 544 self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision) 545 try: 546 self._Clone(revision, url, options) 547 except subprocess2.CalledProcessError: 548 self._DeleteOrMove(options.force) 549 self._Clone(revision, url, options) 550 if file_list is not None: 551 files = self._Capture( 552 ['-c', 'core.quotePath=false', 'ls-files']).splitlines() 553 file_list.extend( 554 [os.path.join(self.checkout_path, f) for f in files]) 555 if mirror: 556 self._Capture( 557 ['remote', 'set-url', '--push', 'origin', mirror.url]) 558 if not verbose: 559 # Make the output a little prettier. It's nice to have some whitespace 560 # between projects when cloning. 561 self.Print('') 562 return self._Capture(['rev-parse', '--verify', 'HEAD']) 563 564 if mirror: 565 self._Capture( 566 ['remote', 'set-url', '--push', 'origin', mirror.url]) 567 568 if not managed: 569 self._SetFetchConfig(options) 570 self.Print('________ unmanaged solution; skipping %s' % self.relpath) 571 return self._Capture(['rev-parse', '--verify', 'HEAD']) 572 573 self._maybe_break_locks(options) 574 575 if mirror: 576 self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision) 577 578 # See if the url has changed (the unittests use git://foo for the url, let 579 # that through). 580 current_url = self._Capture(['config', 'remote.%s.url' % self.remote]) 581 return_early = False 582 # TODO(maruel): Delete url != 'git://foo' since it's just to make the 583 # unit test pass. (and update the comment above) 584 # Skip url auto-correction if remote.origin.gclient-auto-fix-url is set. 585 # This allows devs to use experimental repos which have a different url 586 # but whose branch(s) are the same as official repos. 587 if (current_url.rstrip('/') != url.rstrip('/') and url != 'git://foo' and 588 subprocess2.capture( 589 ['git', 'config', 'remote.%s.gclient-auto-fix-url' % self.remote], 590 cwd=self.checkout_path).strip() != 'False'): 591 self.Print('_____ switching %s from %s to new upstream %s' % ( 592 self.relpath, current_url, url)) 593 if not (options.force or options.reset): 594 # Make sure it's clean 595 self._CheckClean(revision) 596 # Switch over to the new upstream 597 self._Run(['remote', 'set-url', self.remote, url], options) 598 if mirror: 599 with open(os.path.join( 600 self.checkout_path, '.git', 'objects', 'info', 'alternates'), 601 'w') as fh: 602 fh.write(os.path.join(url, 'objects')) 603 self._EnsureValidHeadObjectOrCheckout(revision, options, url) 604 self._FetchAndReset(revision, file_list, options) 605 606 return_early = True 607 else: 608 self._EnsureValidHeadObjectOrCheckout(revision, options, url) 609 610 if return_early: 611 return self._Capture(['rev-parse', '--verify', 'HEAD']) 612 613 cur_branch = self._GetCurrentBranch() 614 615 # Cases: 616 # 0) HEAD is detached. Probably from our initial clone. 617 # - make sure HEAD is contained by a named ref, then update. 618 # Cases 1-4. HEAD is a branch. 619 # 1) current branch is not tracking a remote branch 620 # - try to rebase onto the new hash or branch 621 # 2) current branch is tracking a remote branch with local committed 622 # changes, but the DEPS file switched to point to a hash 623 # - rebase those changes on top of the hash 624 # 3) current branch is tracking a remote branch w/or w/out changes, and 625 # no DEPS switch 626 # - see if we can FF, if not, prompt the user for rebase, merge, or stop 627 # 4) current branch is tracking a remote branch, but DEPS switches to a 628 # different remote branch, and 629 # a) current branch has no local changes, and --force: 630 # - checkout new branch 631 # b) current branch has local changes, and --force and --reset: 632 # - checkout new branch 633 # c) otherwise exit 634 635 # GetUpstreamBranch returns something like 'refs/remotes/origin/master' for 636 # a tracking branch 637 # or 'master' if not a tracking branch (it's based on a specific rev/hash) 638 # or it returns None if it couldn't find an upstream 639 if cur_branch is None: 640 upstream_branch = None 641 current_type = "detached" 642 logging.debug("Detached HEAD") 643 else: 644 upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path) 645 if not upstream_branch or not upstream_branch.startswith('refs/remotes'): 646 current_type = "hash" 647 logging.debug("Current branch is not tracking an upstream (remote)" 648 " branch.") 649 elif upstream_branch.startswith('refs/remotes'): 650 current_type = "branch" 651 else: 652 raise gclient_utils.Error('Invalid Upstream: %s' % upstream_branch) 653 654 self._SetFetchConfig(options) 655 656 # Fetch upstream if we don't already have |revision|. 657 if not scm.GIT.IsValidRevision(self.checkout_path, revision, sha_only=True): 658 self._Fetch(options, prune=options.force) 659 660 if not scm.GIT.IsValidRevision(self.checkout_path, revision, 661 sha_only=True): 662 # Update the remotes first so we have all the refs. 663 remote_output = scm.GIT.Capture(['remote'] + verbose + ['update'], 664 cwd=self.checkout_path) 665 if verbose: 666 self.Print(remote_output) 667 668 revision = self._AutoFetchRef(options, revision) 669 670 # This is a big hammer, debatable if it should even be here... 671 if options.force or options.reset: 672 target = 'HEAD' 673 if options.upstream and upstream_branch: 674 target = upstream_branch 675 self._Scrub(target, options) 676 677 if current_type == 'detached': 678 # case 0 679 # We just did a Scrub, this is as clean as it's going to get. In 680 # particular if HEAD is a commit that contains two versions of the same 681 # file on a case-insensitive filesystem (e.g. 'a' and 'A'), there's no way 682 # to actually "Clean" the checkout; that commit is uncheckoutable on this 683 # system. The best we can do is carry forward to the checkout step. 684 if not (options.force or options.reset): 685 self._CheckClean(revision) 686 self._CheckDetachedHead(revision, options) 687 if self._Capture(['rev-list', '-n', '1', 'HEAD']) == revision: 688 self.Print('Up-to-date; skipping checkout.') 689 else: 690 # 'git checkout' may need to overwrite existing untracked files. Allow 691 # it only when nuclear options are enabled. 692 self._Checkout( 693 options, 694 revision, 695 force=(options.force and options.delete_unversioned_trees), 696 quiet=True, 697 ) 698 if not printed_path: 699 self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False) 700 elif current_type == 'hash': 701 # case 1 702 # Can't find a merge-base since we don't know our upstream. That makes 703 # this command VERY likely to produce a rebase failure. For now we 704 # assume origin is our upstream since that's what the old behavior was. 705 upstream_branch = self.remote 706 if options.revision or deps_revision: 707 upstream_branch = revision 708 self._AttemptRebase(upstream_branch, file_list, options, 709 printed_path=printed_path, merge=options.merge) 710 printed_path = True 711 elif rev_type == 'hash': 712 # case 2 713 self._AttemptRebase(upstream_branch, file_list, options, 714 newbase=revision, printed_path=printed_path, 715 merge=options.merge) 716 printed_path = True 717 elif remote_ref and ''.join(remote_ref) != upstream_branch: 718 # case 4 719 new_base = ''.join(remote_ref) 720 if not printed_path: 721 self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False) 722 switch_error = ("Could not switch upstream branch from %s to %s\n" 723 % (upstream_branch, new_base) + 724 "Please use --force or merge or rebase manually:\n" + 725 "cd %s; git rebase %s\n" % (self.checkout_path, new_base) + 726 "OR git checkout -b <some new branch> %s" % new_base) 727 force_switch = False 728 if options.force: 729 try: 730 self._CheckClean(revision) 731 # case 4a 732 force_switch = True 733 except gclient_utils.Error as e: 734 if options.reset: 735 # case 4b 736 force_switch = True 737 else: 738 switch_error = '%s\n%s' % (e.message, switch_error) 739 if force_switch: 740 self.Print("Switching upstream branch from %s to %s" % 741 (upstream_branch, new_base)) 742 switch_branch = 'gclient_' + remote_ref[1] 743 self._Capture(['branch', '-f', switch_branch, new_base]) 744 self._Checkout(options, switch_branch, force=True, quiet=True) 745 else: 746 # case 4c 747 raise gclient_utils.Error(switch_error) 748 else: 749 # case 3 - the default case 750 rebase_files = self._GetDiffFilenames(upstream_branch) 751 if verbose: 752 self.Print('Trying fast-forward merge to branch : %s' % upstream_branch) 753 try: 754 merge_args = ['merge'] 755 if options.merge: 756 merge_args.append('--ff') 757 else: 758 merge_args.append('--ff-only') 759 merge_args.append(upstream_branch) 760 merge_output = self._Capture(merge_args) 761 except subprocess2.CalledProcessError as e: 762 rebase_files = [] 763 if re.match(b'fatal: Not possible to fast-forward, aborting.', 764 e.stderr): 765 if not printed_path: 766 self.Print('_____ %s at %s' % (self.relpath, revision), 767 timestamp=False) 768 printed_path = True 769 while True: 770 if not options.auto_rebase: 771 try: 772 action = self._AskForData( 773 'Cannot %s, attempt to rebase? ' 774 '(y)es / (q)uit / (s)kip : ' % 775 ('merge' if options.merge else 'fast-forward merge'), 776 options) 777 except ValueError: 778 raise gclient_utils.Error('Invalid Character') 779 if options.auto_rebase or re.match(r'yes|y', action, re.I): 780 self._AttemptRebase(upstream_branch, rebase_files, options, 781 printed_path=printed_path, merge=False) 782 printed_path = True 783 break 784 elif re.match(r'quit|q', action, re.I): 785 raise gclient_utils.Error("Can't fast-forward, please merge or " 786 "rebase manually.\n" 787 "cd %s && git " % self.checkout_path 788 + "rebase %s" % upstream_branch) 789 elif re.match(r'skip|s', action, re.I): 790 self.Print('Skipping %s' % self.relpath) 791 return 792 else: 793 self.Print('Input not recognized') 794 elif re.match(b"error: Your local changes to '.*' would be " 795 b"overwritten by merge. Aborting.\nPlease, commit your " 796 b"changes or stash them before you can merge.\n", 797 e.stderr): 798 if not printed_path: 799 self.Print('_____ %s at %s' % (self.relpath, revision), 800 timestamp=False) 801 printed_path = True 802 raise gclient_utils.Error(e.stderr.decode('utf-8')) 803 else: 804 # Some other problem happened with the merge 805 logging.error("Error during fast-forward merge in %s!" % self.relpath) 806 self.Print(e.stderr.decode('utf-8')) 807 raise 808 else: 809 # Fast-forward merge was successful 810 if not re.match('Already up-to-date.', merge_output) or verbose: 811 if not printed_path: 812 self.Print('_____ %s at %s' % (self.relpath, revision), 813 timestamp=False) 814 printed_path = True 815 self.Print(merge_output.strip()) 816 if not verbose: 817 # Make the output a little prettier. It's nice to have some 818 # whitespace between projects when syncing. 819 self.Print('') 820 821 if file_list is not None: 822 file_list.extend( 823 [os.path.join(self.checkout_path, f) for f in rebase_files]) 824 825 # If the rebase generated a conflict, abort and ask user to fix 826 if self._IsRebasing(): 827 raise gclient_utils.Error('\n____ %s at %s\n' 828 '\nConflict while rebasing this branch.\n' 829 'Fix the conflict and run gclient again.\n' 830 'See man git-rebase for details.\n' 831 % (self.relpath, revision)) 832 833 if verbose: 834 self.Print('Checked out revision %s' % self.revinfo(options, (), None), 835 timestamp=False) 836 837 # If --reset and --delete_unversioned_trees are specified, remove any 838 # untracked directories. 839 if options.reset and options.delete_unversioned_trees: 840 # GIT.CaptureStatus() uses 'dit diff' to compare to a specific SHA1 (the 841 # merge-base by default), so doesn't include untracked files. So we use 842 # 'git ls-files --directory --others --exclude-standard' here directly. 843 paths = scm.GIT.Capture( 844 ['-c', 'core.quotePath=false', 'ls-files', 845 '--directory', '--others', '--exclude-standard'], 846 self.checkout_path) 847 for path in (p for p in paths.splitlines() if p.endswith('/')): 848 full_path = os.path.join(self.checkout_path, path) 849 if not os.path.islink(full_path): 850 self.Print('_____ removing unversioned directory %s' % path) 851 gclient_utils.rmtree(full_path) 852 853 return self._Capture(['rev-parse', '--verify', 'HEAD']) 854 855 def revert(self, options, _args, file_list): 856 """Reverts local modifications. 857 858 All reverted files will be appended to file_list. 859 """ 860 if not os.path.isdir(self.checkout_path): 861 # revert won't work if the directory doesn't exist. It needs to 862 # checkout instead. 863 self.Print('_____ %s is missing, syncing instead' % self.relpath) 864 # Don't reuse the args. 865 return self.update(options, [], file_list) 866 867 default_rev = "refs/heads/master" 868 if options.upstream: 869 if self._GetCurrentBranch(): 870 upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path) 871 default_rev = upstream_branch or default_rev 872 _, deps_revision = gclient_utils.SplitUrlRevision(self.url) 873 if not deps_revision: 874 deps_revision = default_rev 875 if deps_revision.startswith('refs/heads/'): 876 deps_revision = deps_revision.replace('refs/heads/', self.remote + '/') 877 try: 878 deps_revision = self.GetUsableRev(deps_revision, options) 879 except NoUsableRevError as e: 880 # If the DEPS entry's url and hash changed, try to update the origin. 881 # See also http://crbug.com/520067. 882 logging.warning( 883 "Couldn't find usable revision, will retrying to update instead: %s", 884 e.message) 885 return self.update(options, [], file_list) 886 887 if file_list is not None: 888 files = self._GetDiffFilenames(deps_revision) 889 890 self._Scrub(deps_revision, options) 891 self._Run(['clean', '-f', '-d'], options) 892 893 if file_list is not None: 894 file_list.extend([os.path.join(self.checkout_path, f) for f in files]) 895 896 def revinfo(self, _options, _args, _file_list): 897 """Returns revision""" 898 return self._Capture(['rev-parse', 'HEAD']) 899 900 def runhooks(self, options, args, file_list): 901 self.status(options, args, file_list) 902 903 def status(self, options, _args, file_list): 904 """Display status information.""" 905 if not os.path.isdir(self.checkout_path): 906 self.Print('________ couldn\'t run status in %s:\n' 907 'The directory does not exist.' % self.checkout_path) 908 else: 909 merge_base = [] 910 if self.url: 911 _, base_rev = gclient_utils.SplitUrlRevision(self.url) 912 if base_rev: 913 merge_base = [base_rev] 914 self._Run( 915 ['-c', 'core.quotePath=false', 'diff', '--name-status'] + merge_base, 916 options, always_show_header=options.verbose) 917 if file_list is not None: 918 files = self._GetDiffFilenames(merge_base[0] if merge_base else None) 919 file_list.extend([os.path.join(self.checkout_path, f) for f in files]) 920 921 def GetUsableRev(self, rev, options): 922 """Finds a useful revision for this repository.""" 923 sha1 = None 924 if not os.path.isdir(self.checkout_path): 925 raise NoUsableRevError( 926 'This is not a git repo, so we cannot get a usable rev.') 927 928 if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev): 929 sha1 = rev 930 else: 931 # May exist in origin, but we don't have it yet, so fetch and look 932 # again. 933 self._Fetch(options) 934 if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev): 935 sha1 = rev 936 937 if not sha1: 938 raise NoUsableRevError( 939 'Hash %s does not appear to be a valid hash in this repo.' % rev) 940 941 return sha1 942 943 def GetGitBackupDirPath(self): 944 """Returns the path where the .git folder for the current project can be 945 staged/restored. Use case: subproject moved from DEPS <-> outer project.""" 946 return os.path.join(self._root_dir, 947 'old_' + self.relpath.replace(os.sep, '_')) + '.git' 948 949 def _GetMirror(self, url, options, revision_ref=None): 950 """Get a git_cache.Mirror object for the argument url.""" 951 if not self.cache_dir: 952 return None 953 mirror_kwargs = { 954 'print_func': self.filter, 955 'refs': [] 956 } 957 if hasattr(options, 'with_branch_heads') and options.with_branch_heads: 958 mirror_kwargs['refs'].append('refs/branch-heads/*') 959 elif revision_ref and revision_ref.startswith('refs/branch-heads/'): 960 mirror_kwargs['refs'].append(revision_ref) 961 if hasattr(options, 'with_tags') and options.with_tags: 962 mirror_kwargs['refs'].append('refs/tags/*') 963 elif revision_ref and revision_ref.startswith('refs/tags/'): 964 mirror_kwargs['refs'].append(revision_ref) 965 return git_cache.Mirror(url, **mirror_kwargs) 966 967 def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision): 968 """Update a git mirror by fetching the latest commits from the remote, 969 unless mirror already contains revision whose type is sha1 hash. 970 """ 971 if rev_type == 'hash' and mirror.contains_revision(revision): 972 if options.verbose: 973 self.Print('skipping mirror update, it has rev=%s already' % revision, 974 timestamp=False) 975 return 976 977 if getattr(options, 'shallow', False): 978 # HACK(hinoka): These repositories should be super shallow. 979 if 'flash' in mirror.url: 980 depth = 10 981 else: 982 depth = 10000 983 else: 984 depth = None 985 mirror.populate(verbose=options.verbose, 986 bootstrap=not getattr(options, 'no_bootstrap', False), 987 depth=depth, 988 lock_timeout=getattr(options, 'lock_timeout', 0)) 989 990 def _Clone(self, revision, url, options): 991 """Clone a git repository from the given URL. 992 993 Once we've cloned the repo, we checkout a working branch if the specified 994 revision is a branch head. If it is a tag or a specific commit, then we 995 leave HEAD detached as it makes future updates simpler -- in this case the 996 user should first create a new branch or switch to an existing branch before 997 making changes in the repo.""" 998 if not options.verbose: 999 # git clone doesn't seem to insert a newline properly before printing 1000 # to stdout 1001 self.Print('') 1002 cfg = gclient_utils.DefaultIndexPackConfig(url) 1003 clone_cmd = cfg + ['clone', '--no-checkout', '--progress'] 1004 if self.cache_dir: 1005 clone_cmd.append('--shared') 1006 if options.verbose: 1007 clone_cmd.append('--verbose') 1008 clone_cmd.append(url) 1009 # If the parent directory does not exist, Git clone on Windows will not 1010 # create it, so we need to do it manually. 1011 parent_dir = os.path.dirname(self.checkout_path) 1012 gclient_utils.safe_makedirs(parent_dir) 1013 1014 template_dir = None 1015 if hasattr(options, 'no_history') and options.no_history: 1016 if gclient_utils.IsGitSha(revision): 1017 # In the case of a subproject, the pinned sha is not necessarily the 1018 # head of the remote branch (so we can't just use --depth=N). Instead, 1019 # we tell git to fetch all the remote objects from SHA..HEAD by means of 1020 # a template git dir which has a 'shallow' file pointing to the sha. 1021 template_dir = tempfile.mkdtemp( 1022 prefix='_gclient_gittmp_%s' % os.path.basename(self.checkout_path), 1023 dir=parent_dir) 1024 self._Run(['init', '--bare', template_dir], options, cwd=self._root_dir) 1025 with open(os.path.join(template_dir, 'shallow'), 'w') as template_file: 1026 template_file.write(revision) 1027 clone_cmd.append('--template=' + template_dir) 1028 else: 1029 # Otherwise, we're just interested in the HEAD. Just use --depth. 1030 clone_cmd.append('--depth=1') 1031 1032 tmp_dir = tempfile.mkdtemp( 1033 prefix='_gclient_%s_' % os.path.basename(self.checkout_path), 1034 dir=parent_dir) 1035 try: 1036 clone_cmd.append(tmp_dir) 1037 if self.print_outbuf: 1038 print_stdout = True 1039 filter_fn = None 1040 else: 1041 print_stdout = False 1042 filter_fn = self.filter 1043 self._Run(clone_cmd, options, cwd=self._root_dir, retry=True, 1044 print_stdout=print_stdout, filter_fn=filter_fn) 1045 gclient_utils.safe_makedirs(self.checkout_path) 1046 gclient_utils.safe_rename(os.path.join(tmp_dir, '.git'), 1047 os.path.join(self.checkout_path, '.git')) 1048 # TODO(https://github.com/git-for-windows/git/issues/2569): Remove once 1049 # fixed. 1050 if sys.platform.startswith('win'): 1051 try: 1052 self._Run(['config', '--unset', 'core.worktree'], options, 1053 cwd=self.checkout_path) 1054 except subprocess2.CalledProcessError: 1055 pass 1056 except: 1057 traceback.print_exc(file=self.out_fh) 1058 raise 1059 finally: 1060 if os.listdir(tmp_dir): 1061 self.Print('_____ removing non-empty tmp dir %s' % tmp_dir) 1062 gclient_utils.rmtree(tmp_dir) 1063 if template_dir: 1064 gclient_utils.rmtree(template_dir) 1065 self._SetFetchConfig(options) 1066 self._Fetch(options, prune=options.force) 1067 revision = self._AutoFetchRef(options, revision) 1068 remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote) 1069 self._Checkout(options, ''.join(remote_ref or revision), quiet=True) 1070 if self._GetCurrentBranch() is None: 1071 # Squelch git's very verbose detached HEAD warning and use our own 1072 self.Print( 1073 ('Checked out %s to a detached HEAD. Before making any commits\n' 1074 'in this repo, you should use \'git checkout <branch>\' to switch to\n' 1075 'an existing branch or use \'git checkout %s -b <branch>\' to\n' 1076 'create a new branch for your work.') % (revision, self.remote)) 1077 1078 def _AskForData(self, prompt, options): 1079 if options.jobs > 1: 1080 self.Print(prompt) 1081 raise gclient_utils.Error("Background task requires input. Rerun " 1082 "gclient with --jobs=1 so that\n" 1083 "interaction is possible.") 1084 return gclient_utils.AskForData(prompt) 1085 1086 1087 def _AttemptRebase(self, upstream, files, options, newbase=None, 1088 branch=None, printed_path=False, merge=False): 1089 """Attempt to rebase onto either upstream or, if specified, newbase.""" 1090 if files is not None: 1091 files.extend(self._GetDiffFilenames(upstream)) 1092 revision = upstream 1093 if newbase: 1094 revision = newbase 1095 action = 'merge' if merge else 'rebase' 1096 if not printed_path: 1097 self.Print('_____ %s : Attempting %s onto %s...' % ( 1098 self.relpath, action, revision)) 1099 printed_path = True 1100 else: 1101 self.Print('Attempting %s onto %s...' % (action, revision)) 1102 1103 if merge: 1104 merge_output = self._Capture(['merge', revision]) 1105 if options.verbose: 1106 self.Print(merge_output) 1107 return 1108 1109 # Build the rebase command here using the args 1110 # git rebase [options] [--onto <newbase>] <upstream> [<branch>] 1111 rebase_cmd = ['rebase'] 1112 if options.verbose: 1113 rebase_cmd.append('--verbose') 1114 if newbase: 1115 rebase_cmd.extend(['--onto', newbase]) 1116 rebase_cmd.append(upstream) 1117 if branch: 1118 rebase_cmd.append(branch) 1119 1120 try: 1121 rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path) 1122 except subprocess2.CalledProcessError as e: 1123 if (re.match(br'cannot rebase: you have unstaged changes', e.stderr) or 1124 re.match(br'cannot rebase: your index contains uncommitted changes', 1125 e.stderr)): 1126 while True: 1127 rebase_action = self._AskForData( 1128 'Cannot rebase because of unstaged changes.\n' 1129 '\'git reset --hard HEAD\' ?\n' 1130 'WARNING: destroys any uncommitted work in your current branch!' 1131 ' (y)es / (q)uit / (s)how : ', options) 1132 if re.match(r'yes|y', rebase_action, re.I): 1133 self._Scrub('HEAD', options) 1134 # Should this be recursive? 1135 rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path) 1136 break 1137 elif re.match(r'quit|q', rebase_action, re.I): 1138 raise gclient_utils.Error("Please merge or rebase manually\n" 1139 "cd %s && git " % self.checkout_path 1140 + "%s" % ' '.join(rebase_cmd)) 1141 elif re.match(r'show|s', rebase_action, re.I): 1142 self.Print('%s' % e.stderr.decode('utf-8').strip()) 1143 continue 1144 else: 1145 gclient_utils.Error("Input not recognized") 1146 continue 1147 elif re.search(br'^CONFLICT', e.stdout, re.M): 1148 raise gclient_utils.Error("Conflict while rebasing this branch.\n" 1149 "Fix the conflict and run gclient again.\n" 1150 "See 'man git-rebase' for details.\n") 1151 else: 1152 self.Print(e.stdout.decode('utf-8').strip()) 1153 self.Print('Rebase produced error output:\n%s' % 1154 e.stderr.decode('utf-8').strip()) 1155 raise gclient_utils.Error("Unrecognized error, please merge or rebase " 1156 "manually.\ncd %s && git " % 1157 self.checkout_path 1158 + "%s" % ' '.join(rebase_cmd)) 1159 1160 self.Print(rebase_output.strip()) 1161 if not options.verbose: 1162 # Make the output a little prettier. It's nice to have some 1163 # whitespace between projects when syncing. 1164 self.Print('') 1165 1166 @staticmethod 1167 def _CheckMinVersion(min_version): 1168 (ok, current_version) = scm.GIT.AssertVersion(min_version) 1169 if not ok: 1170 raise gclient_utils.Error('git version %s < minimum required %s' % 1171 (current_version, min_version)) 1172 1173 def _EnsureValidHeadObjectOrCheckout(self, revision, options, url): 1174 # Special case handling if all 3 conditions are met: 1175 # * the mirros have recently changed, but deps destination remains same, 1176 # * the git histories of mirrors are conflicting. 1177 # * git cache is used 1178 # This manifests itself in current checkout having invalid HEAD commit on 1179 # most git operations. Since git cache is used, just deleted the .git 1180 # folder, and re-create it by cloning. 1181 try: 1182 self._Capture(['rev-list', '-n', '1', 'HEAD']) 1183 except subprocess2.CalledProcessError as e: 1184 if (b'fatal: bad object HEAD' in e.stderr 1185 and self.cache_dir and self.cache_dir in url): 1186 self.Print(( 1187 'Likely due to DEPS change with git cache_dir, ' 1188 'the current commit points to no longer existing object.\n' 1189 '%s' % e) 1190 ) 1191 self._DeleteOrMove(options.force) 1192 self._Clone(revision, url, options) 1193 else: 1194 raise 1195 1196 def _IsRebasing(self): 1197 # Check for any of REBASE-i/REBASE-m/REBASE/AM. Unfortunately git doesn't 1198 # have a plumbing command to determine whether a rebase is in progress, so 1199 # for now emualate (more-or-less) git-rebase.sh / git-completion.bash 1200 g = os.path.join(self.checkout_path, '.git') 1201 return ( 1202 os.path.isdir(os.path.join(g, "rebase-merge")) or 1203 os.path.isdir(os.path.join(g, "rebase-apply"))) 1204 1205 def _CheckClean(self, revision, fixup=False): 1206 lockfile = os.path.join(self.checkout_path, ".git", "index.lock") 1207 if os.path.exists(lockfile): 1208 raise gclient_utils.Error( 1209 '\n____ %s at %s\n' 1210 '\tYour repo is locked, possibly due to a concurrent git process.\n' 1211 '\tIf no git executable is running, then clean up %r and try again.\n' 1212 % (self.relpath, revision, lockfile)) 1213 1214 # Make sure the tree is clean; see git-rebase.sh for reference 1215 try: 1216 scm.GIT.Capture(['update-index', '--ignore-submodules', '--refresh'], 1217 cwd=self.checkout_path) 1218 except subprocess2.CalledProcessError: 1219 raise gclient_utils.Error('\n____ %s at %s\n' 1220 '\tYou have unstaged changes.\n' 1221 '\tPlease commit, stash, or reset.\n' 1222 % (self.relpath, revision)) 1223 try: 1224 scm.GIT.Capture(['diff-index', '--cached', '--name-status', '-r', 1225 '--ignore-submodules', 'HEAD', '--'], 1226 cwd=self.checkout_path) 1227 except subprocess2.CalledProcessError: 1228 raise gclient_utils.Error('\n____ %s at %s\n' 1229 '\tYour index contains uncommitted changes\n' 1230 '\tPlease commit, stash, or reset.\n' 1231 % (self.relpath, revision)) 1232 1233 def _CheckDetachedHead(self, revision, _options): 1234 # HEAD is detached. Make sure it is safe to move away from (i.e., it is 1235 # reference by a commit). If not, error out -- most likely a rebase is 1236 # in progress, try to detect so we can give a better error. 1237 try: 1238 scm.GIT.Capture(['name-rev', '--no-undefined', 'HEAD'], 1239 cwd=self.checkout_path) 1240 except subprocess2.CalledProcessError: 1241 # Commit is not contained by any rev. See if the user is rebasing: 1242 if self._IsRebasing(): 1243 # Punt to the user 1244 raise gclient_utils.Error('\n____ %s at %s\n' 1245 '\tAlready in a conflict, i.e. (no branch).\n' 1246 '\tFix the conflict and run gclient again.\n' 1247 '\tOr to abort run:\n\t\tgit-rebase --abort\n' 1248 '\tSee man git-rebase for details.\n' 1249 % (self.relpath, revision)) 1250 # Let's just save off the commit so we can proceed. 1251 name = ('saved-by-gclient-' + 1252 self._Capture(['rev-parse', '--short', 'HEAD'])) 1253 self._Capture(['branch', '-f', name]) 1254 self.Print('_____ found an unreferenced commit and saved it as \'%s\'' % 1255 name) 1256 1257 def _GetCurrentBranch(self): 1258 # Returns name of current branch or None for detached HEAD 1259 branch = self._Capture(['rev-parse', '--abbrev-ref=strict', 'HEAD']) 1260 if branch == 'HEAD': 1261 return None 1262 return branch 1263 1264 def _Capture(self, args, **kwargs): 1265 set_git_dir = 'cwd' not in kwargs 1266 kwargs.setdefault('cwd', self.checkout_path) 1267 kwargs.setdefault('stderr', subprocess2.PIPE) 1268 strip = kwargs.pop('strip', True) 1269 env = scm.GIT.ApplyEnvVars(kwargs) 1270 # If an explicit cwd isn't set, then default to the .git/ subdir so we get 1271 # stricter behavior. This can be useful in cases of slight corruption -- 1272 # we don't accidentally go corrupting parent git checks too. See 1273 # https://crbug.com/1000825 for an example. 1274 if set_git_dir: 1275 git_dir = os.path.abspath(os.path.join(self.checkout_path, '.git')) 1276 # Depending on how the .gclient file was defined, self.checkout_path 1277 # might be set to a unicode string, not a regular string; on Windows 1278 # Python2, we can't set env vars to be unicode strings, so we 1279 # forcibly cast the value to a string before setting it. 1280 env.setdefault('GIT_DIR', str(git_dir)) 1281 ret = subprocess2.check_output( 1282 ['git'] + args, env=env, **kwargs).decode('utf-8') 1283 if strip: 1284 ret = ret.strip() 1285 self.Print('Finished running: %s %s' % ('git', ' '.join(args))) 1286 return ret 1287 1288 def _Checkout(self, options, ref, force=False, quiet=None): 1289 """Performs a 'git-checkout' operation. 1290 1291 Args: 1292 options: The configured option set 1293 ref: (str) The branch/commit to checkout 1294 quiet: (bool/None) Whether or not the checkout should pass '--quiet'; if 1295 'None', the behavior is inferred from 'options.verbose'. 1296 Returns: (str) The output of the checkout operation 1297 """ 1298 if quiet is None: 1299 quiet = (not options.verbose) 1300 checkout_args = ['checkout'] 1301 if force: 1302 checkout_args.append('--force') 1303 if quiet: 1304 checkout_args.append('--quiet') 1305 checkout_args.append(ref) 1306 return self._Capture(checkout_args) 1307 1308 def _Fetch(self, options, remote=None, prune=False, quiet=False, 1309 refspec=None): 1310 cfg = gclient_utils.DefaultIndexPackConfig(self.url) 1311 # When updating, the ref is modified to be a remote ref . 1312 # (e.g. refs/heads/NAME becomes refs/remotes/REMOTE/NAME). 1313 # Try to reverse that mapping. 1314 original_ref = scm.GIT.RemoteRefToRef(refspec, self.remote) 1315 if original_ref: 1316 refspec = original_ref + ':' + refspec 1317 # When a mirror is configured, it only fetches 1318 # refs/{heads,branch-heads,tags}/*. 1319 # If asked to fetch other refs, we must fetch those directly from the 1320 # repository, and not from the mirror. 1321 if not original_ref.startswith( 1322 ('refs/heads/', 'refs/branch-heads/', 'refs/tags/')): 1323 remote, _ = gclient_utils.SplitUrlRevision(self.url) 1324 fetch_cmd = cfg + [ 1325 'fetch', 1326 remote or self.remote, 1327 ] 1328 if refspec: 1329 fetch_cmd.append(refspec) 1330 1331 if prune: 1332 fetch_cmd.append('--prune') 1333 if options.verbose: 1334 fetch_cmd.append('--verbose') 1335 if not hasattr(options, 'with_tags') or not options.with_tags: 1336 fetch_cmd.append('--no-tags') 1337 elif quiet: 1338 fetch_cmd.append('--quiet') 1339 self._Run(fetch_cmd, options, show_header=options.verbose, retry=True) 1340 1341 def _SetFetchConfig(self, options): 1342 """Adds, and optionally fetches, "branch-heads" and "tags" refspecs 1343 if requested.""" 1344 if options.force or options.reset: 1345 try: 1346 self._Run(['config', '--unset-all', 'remote.%s.fetch' % self.remote], 1347 options) 1348 self._Run(['config', 'remote.%s.fetch' % self.remote, 1349 '+refs/heads/*:refs/remotes/%s/*' % self.remote], options) 1350 except subprocess2.CalledProcessError as e: 1351 # If exit code was 5, it means we attempted to unset a config that 1352 # didn't exist. Ignore it. 1353 if e.returncode != 5: 1354 raise 1355 if hasattr(options, 'with_branch_heads') and options.with_branch_heads: 1356 config_cmd = ['config', 'remote.%s.fetch' % self.remote, 1357 '+refs/branch-heads/*:refs/remotes/branch-heads/*', 1358 '^\\+refs/branch-heads/\\*:.*$'] 1359 self._Run(config_cmd, options) 1360 if hasattr(options, 'with_tags') and options.with_tags: 1361 config_cmd = ['config', 'remote.%s.fetch' % self.remote, 1362 '+refs/tags/*:refs/tags/*', 1363 '^\\+refs/tags/\\*:.*$'] 1364 self._Run(config_cmd, options) 1365 1366 def _AutoFetchRef(self, options, revision): 1367 """Attempts to fetch |revision| if not available in local repo. 1368 1369 Returns possibly updated revision.""" 1370 if not scm.GIT.IsValidRevision(self.checkout_path, revision): 1371 self._Fetch(options, refspec=revision) 1372 revision = self._Capture(['rev-parse', 'FETCH_HEAD']) 1373 return revision 1374 1375 def _IsRunningUnderRosetta(self): 1376 if sys.platform != 'darwin': 1377 return False 1378 if self._running_under_rosetta is None: 1379 # If we are running under Rosetta, platform.machine() is 1380 # 'x86_64'; we need to use a sysctl to see if we're being 1381 # translated. 1382 import ctypes 1383 libSystem = ctypes.CDLL("libSystem.dylib") 1384 ret = ctypes.c_int(0) 1385 size = ctypes.c_size_t(4) 1386 e = libSystem.sysctlbyname(ctypes.c_char_p(b'sysctl.proc_translated'), 1387 ctypes.byref(ret), ctypes.byref(size), None, 0) 1388 self._running_under_rosetta = e == 0 and ret.value == 1 1389 return self._running_under_rosetta 1390 1391 def _Run(self, args, options, **kwargs): 1392 # Disable 'unused options' warning | pylint: disable=unused-argument 1393 kwargs.setdefault('cwd', self.checkout_path) 1394 kwargs.setdefault('filter_fn', self.filter) 1395 kwargs.setdefault('show_header', True) 1396 env = scm.GIT.ApplyEnvVars(kwargs) 1397 1398 cmd = ['git'] + args 1399 1400 if self._IsRunningUnderRosetta(): 1401 # We currently only ship an Intel Python binary in depot_tools. 1402 # Intel binaries run under Rosetta on ARM Macs, and by default 1403 # prefer to run their subprocesses as Intel under Rosetta too. 1404 # Intel git running under Rosetta has a bug where it fails to 1405 # clone src.git (rdar://7868319), so until we ship a native 1406 # ARM python3 binary, explicitly use `arch` to let git run 1407 # the native ARM slice instead of the Intel slice. 1408 # TODO(thakis): Remove this again once we ship an arm64 python3 1409 # binary. 1410 cmd = ['arch', '-arch', 'arm64e', '-arch', 'arm64'] + cmd 1411 gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs) 1412 1413 1414class CipdPackage(object): 1415 """A representation of a single CIPD package.""" 1416 1417 def __init__(self, name, version, authority_for_subdir): 1418 self._authority_for_subdir = authority_for_subdir 1419 self._name = name 1420 self._version = version 1421 1422 @property 1423 def authority_for_subdir(self): 1424 """Whether this package has authority to act on behalf of its subdir. 1425 1426 Some operations should only be performed once per subdirectory. A package 1427 that has authority for its subdirectory is the only package that should 1428 perform such operations. 1429 1430 Returns: 1431 bool; whether this package has subdir authority. 1432 """ 1433 return self._authority_for_subdir 1434 1435 @property 1436 def name(self): 1437 return self._name 1438 1439 @property 1440 def version(self): 1441 return self._version 1442 1443 1444class CipdRoot(object): 1445 """A representation of a single CIPD root.""" 1446 def __init__(self, root_dir, service_url): 1447 self._all_packages = set() 1448 self._mutator_lock = threading.Lock() 1449 self._packages_by_subdir = collections.defaultdict(list) 1450 self._root_dir = root_dir 1451 self._service_url = service_url 1452 1453 def add_package(self, subdir, package, version): 1454 """Adds a package to this CIPD root. 1455 1456 As far as clients are concerned, this grants both root and subdir authority 1457 to packages arbitrarily. (The implementation grants root authority to the 1458 first package added and subdir authority to the first package added for that 1459 subdir, but clients should not depend on or expect that behavior.) 1460 1461 Args: 1462 subdir: str; relative path to where the package should be installed from 1463 the cipd root directory. 1464 package: str; the cipd package name. 1465 version: str; the cipd package version. 1466 Returns: 1467 CipdPackage; the package that was created and added to this root. 1468 """ 1469 with self._mutator_lock: 1470 cipd_package = CipdPackage( 1471 package, version, 1472 not self._packages_by_subdir[subdir]) 1473 self._all_packages.add(cipd_package) 1474 self._packages_by_subdir[subdir].append(cipd_package) 1475 return cipd_package 1476 1477 def packages(self, subdir): 1478 """Get the list of configured packages for the given subdir.""" 1479 return list(self._packages_by_subdir[subdir]) 1480 1481 def clobber(self): 1482 """Remove the .cipd directory. 1483 1484 This is useful for forcing ensure to redownload and reinitialize all 1485 packages. 1486 """ 1487 with self._mutator_lock: 1488 cipd_cache_dir = os.path.join(self.root_dir, '.cipd') 1489 try: 1490 gclient_utils.rmtree(os.path.join(cipd_cache_dir)) 1491 except OSError: 1492 if os.path.exists(cipd_cache_dir): 1493 raise 1494 1495 @contextlib.contextmanager 1496 def _create_ensure_file(self): 1497 try: 1498 contents = '$ParanoidMode CheckPresence\n\n' 1499 for subdir, packages in sorted(self._packages_by_subdir.items()): 1500 contents += '@Subdir %s\n' % subdir 1501 for package in sorted(packages, key=lambda p: p.name): 1502 contents += '%s %s\n' % (package.name, package.version) 1503 contents += '\n' 1504 ensure_file = None 1505 with tempfile.NamedTemporaryFile( 1506 suffix='.ensure', delete=False, mode='wb') as ensure_file: 1507 ensure_file.write(contents.encode('utf-8', 'replace')) 1508 yield ensure_file.name 1509 finally: 1510 if ensure_file is not None and os.path.exists(ensure_file.name): 1511 os.remove(ensure_file.name) 1512 1513 def ensure(self): 1514 """Run `cipd ensure`.""" 1515 with self._mutator_lock: 1516 with self._create_ensure_file() as ensure_file: 1517 cmd = [ 1518 'cipd', 'ensure', 1519 '-log-level', 'error', 1520 '-root', self.root_dir, 1521 '-ensure-file', ensure_file, 1522 ] 1523 gclient_utils.CheckCallAndFilter( 1524 cmd, print_stdout=True, show_header=True) 1525 1526 def run(self, command): 1527 if command == 'update': 1528 self.ensure() 1529 elif command == 'revert': 1530 self.clobber() 1531 self.ensure() 1532 1533 def created_package(self, package): 1534 """Checks whether this root created the given package. 1535 1536 Args: 1537 package: CipdPackage; the package to check. 1538 Returns: 1539 bool; whether this root created the given package. 1540 """ 1541 return package in self._all_packages 1542 1543 @property 1544 def root_dir(self): 1545 return self._root_dir 1546 1547 @property 1548 def service_url(self): 1549 return self._service_url 1550 1551 1552class CipdWrapper(SCMWrapper): 1553 """Wrapper for CIPD. 1554 1555 Currently only supports chrome-infra-packages.appspot.com. 1556 """ 1557 name = 'cipd' 1558 1559 def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None, 1560 out_cb=None, root=None, package=None): 1561 super(CipdWrapper, self).__init__( 1562 url=url, root_dir=root_dir, relpath=relpath, out_fh=out_fh, 1563 out_cb=out_cb) 1564 assert root.created_package(package) 1565 self._package = package 1566 self._root = root 1567 1568 #override 1569 def GetCacheMirror(self): 1570 return None 1571 1572 #override 1573 def GetActualRemoteURL(self, options): 1574 return self._root.service_url 1575 1576 #override 1577 def DoesRemoteURLMatch(self, options): 1578 del options 1579 return True 1580 1581 def revert(self, options, args, file_list): 1582 """Does nothing. 1583 1584 CIPD packages should be reverted at the root by running 1585 `CipdRoot.run('revert')`. 1586 """ 1587 pass 1588 1589 def diff(self, options, args, file_list): 1590 """CIPD has no notion of diffing.""" 1591 pass 1592 1593 def pack(self, options, args, file_list): 1594 """CIPD has no notion of diffing.""" 1595 pass 1596 1597 def revinfo(self, options, args, file_list): 1598 """Grab the instance ID.""" 1599 try: 1600 tmpdir = tempfile.mkdtemp() 1601 describe_json_path = os.path.join(tmpdir, 'describe.json') 1602 cmd = [ 1603 'cipd', 'describe', 1604 self._package.name, 1605 '-log-level', 'error', 1606 '-version', self._package.version, 1607 '-json-output', describe_json_path 1608 ] 1609 gclient_utils.CheckCallAndFilter(cmd) 1610 with open(describe_json_path) as f: 1611 describe_json = json.load(f) 1612 return describe_json.get('result', {}).get('pin', {}).get('instance_id') 1613 finally: 1614 gclient_utils.rmtree(tmpdir) 1615 1616 def status(self, options, args, file_list): 1617 pass 1618 1619 def update(self, options, args, file_list): 1620 """Does nothing. 1621 1622 CIPD packages should be updated at the root by running 1623 `CipdRoot.run('update')`. 1624 """ 1625 pass 1626