1#!/usr/bin/env python 2# Copyright 2015 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Rolls DEPS controlled dependency. 7 8Works only with git checkout and git dependencies. Currently this 9script will always roll to the tip of to origin/master. 10""" 11 12from __future__ import print_function 13 14import argparse 15import os 16import re 17import subprocess2 18import sys 19 20NEED_SHELL = sys.platform.startswith('win') 21GCLIENT_PATH = os.path.join( 22 os.path.dirname(os.path.abspath(__file__)), 'gclient.py') 23 24 25# Commit subject that will be considered a roll. In the format generated by the 26# git log used, so it's "<year>-<month>-<day> <author> <subject>" 27_ROLL_SUBJECT = re.compile( 28 # Date 29 r'^\d\d\d\d-\d\d-\d\d ' 30 # Author 31 r'[^ ]+ ' 32 # Subject 33 r'(' 34 # Generated by 35 # https://skia.googlesource.com/buildbot/+/master/autoroll/go/repo_manager/deps_repo_manager.go 36 r'Roll [^ ]+ [a-f0-9]+\.\.[a-f0-9]+ \(\d+ commits\)' 37 r'|' 38 # Generated by 39 # https://chromium.googlesource.com/infra/infra/+/master/recipes/recipe_modules/recipe_autoroller/api.py 40 r'Roll recipe dependencies \(trivial\)\.' 41 r')$') 42 43 44class Error(Exception): 45 pass 46 47 48class AlreadyRolledError(Error): 49 pass 50 51 52def check_output(*args, **kwargs): 53 """subprocess.check_output() passing shell=True on Windows for git.""" 54 kwargs.setdefault('shell', NEED_SHELL) 55 return subprocess2.check_output(*args, **kwargs).decode('utf-8') 56 57 58def check_call(*args, **kwargs): 59 """subprocess.check_call() passing shell=True on Windows for git.""" 60 kwargs.setdefault('shell', NEED_SHELL) 61 subprocess2.check_call(*args, **kwargs) 62 63 64def return_code(*args, **kwargs): 65 """subprocess.call() passing shell=True on Windows for git and 66 subprocess2.VOID for stdout and stderr.""" 67 kwargs.setdefault('shell', NEED_SHELL) 68 kwargs.setdefault('stdout', subprocess2.VOID) 69 kwargs.setdefault('stderr', subprocess2.VOID) 70 return subprocess2.call(*args, **kwargs) 71 72 73def is_pristine(root): 74 """Returns True if a git checkout is pristine.""" 75 # Check both origin/master and origin/main since many projects are 76 # transitioning to origin/main. 77 for branch in ('origin/main', 'origin/master'): 78 # `git rev-parse --verify` has a non-zero return code if the revision 79 # doesn't exist. 80 rev_cmd = ['git', 'rev-parse', '--verify', '--quiet', 81 'refs/remotes/' + branch] 82 if return_code(rev_cmd, cwd=root) != 0: 83 continue 84 85 diff_cmd = ['git', 'diff', '--ignore-submodules', branch] 86 return (not check_output(diff_cmd, cwd=root).strip() and 87 not check_output(diff_cmd + ['--cached'], cwd=root).strip()) 88 89 90 raise Error('Couldn\'t find any of origin/main or origin/master') 91 92def get_log_url(upstream_url, head, master): 93 """Returns an URL to read logs via a Web UI if applicable.""" 94 if re.match(r'https://[^/]*\.googlesource\.com/', upstream_url): 95 # gitiles 96 return '%s/+log/%s..%s' % (upstream_url, head[:12], master[:12]) 97 if upstream_url.startswith('https://github.com/'): 98 upstream_url = upstream_url.rstrip('/') 99 if upstream_url.endswith('.git'): 100 upstream_url = upstream_url[:-len('.git')] 101 return '%s/compare/%s...%s' % (upstream_url, head[:12], master[:12]) 102 return None 103 104 105def should_show_log(upstream_url): 106 """Returns True if a short log should be included in the tree.""" 107 # Skip logs for very active projects. 108 if upstream_url.endswith('/v8/v8.git'): 109 return False 110 if 'webrtc' in upstream_url: 111 return False 112 return True 113 114 115def gclient(args): 116 """Executes gclient with the given args and returns the stdout.""" 117 return check_output([sys.executable, GCLIENT_PATH] + args).strip() 118 119 120def generate_commit_message( 121 full_dir, dependency, head, roll_to, no_log, log_limit): 122 """Creates the commit message for this specific roll.""" 123 commit_range = '%s..%s' % (head, roll_to) 124 commit_range_for_header = '%s..%s' % (head[:9], roll_to[:9]) 125 upstream_url = check_output( 126 ['git', 'config', 'remote.origin.url'], cwd=full_dir).strip() 127 log_url = get_log_url(upstream_url, head, roll_to) 128 cmd = ['git', 'log', commit_range, '--date=short', '--no-merges'] 129 logs = check_output( 130 # Args with '=' are automatically quoted. 131 cmd + ['--format=%ad %ae %s', '--'], 132 cwd=full_dir).rstrip() 133 logs = re.sub(r'(?m)^(\d\d\d\d-\d\d-\d\d [^@]+)@[^ ]+( .*)$', r'\1\2', logs) 134 lines = logs.splitlines() 135 cleaned_lines = [l for l in lines if not _ROLL_SUBJECT.match(l)] 136 logs = '\n'.join(cleaned_lines) + '\n' 137 138 nb_commits = len(lines) 139 rolls = nb_commits - len(cleaned_lines) 140 header = 'Roll %s/ %s (%d commit%s%s)\n\n' % ( 141 dependency, 142 commit_range_for_header, 143 nb_commits, 144 's' if nb_commits > 1 else '', 145 ('; %s trivial rolls' % rolls) if rolls else '') 146 log_section = '' 147 if log_url: 148 log_section = log_url + '\n\n' 149 log_section += '$ %s ' % ' '.join(cmd) 150 log_section += '--format=\'%ad %ae %s\'\n' 151 log_section = log_section.replace(commit_range, commit_range_for_header) 152 # It is important that --no-log continues to work, as it is used by 153 # internal -> external rollers. Please do not remove or break it. 154 if not no_log and should_show_log(upstream_url): 155 if len(cleaned_lines) > log_limit: 156 # Keep the first N/2 log entries and last N/2 entries. 157 lines = logs.splitlines(True) 158 lines = lines[:log_limit//2] + ['(...)\n'] + lines[-log_limit//2:] 159 logs = ''.join(lines) 160 log_section += logs 161 return header + log_section 162 163 164def calculate_roll(full_dir, dependency, roll_to): 165 """Calculates the roll for a dependency by processing gclient_dict, and 166 fetching the dependency via git. 167 """ 168 head = gclient(['getdep', '-r', dependency]) 169 if not head: 170 raise Error('%s is unpinned.' % dependency) 171 check_call(['git', 'fetch', 'origin', '--quiet'], cwd=full_dir) 172 roll_to = check_output(['git', 'rev-parse', roll_to], cwd=full_dir).strip() 173 return head, roll_to 174 175 176def gen_commit_msg(logs, cmdline, reviewers, bug): 177 """Returns the final commit message.""" 178 commit_msg = '' 179 if len(logs) > 1: 180 commit_msg = 'Rolling %d dependencies\n\n' % len(logs) 181 commit_msg += '\n\n'.join(logs) 182 commit_msg += '\nCreated with:\n ' + cmdline + '\n' 183 commit_msg += 'R=%s\n' % ','.join(reviewers) if reviewers else '' 184 commit_msg += '\nBug: %s\n' % bug if bug else '' 185 return commit_msg 186 187 188def finalize(commit_msg, current_dir, rolls): 189 """Commits changes to the DEPS file, then uploads a CL.""" 190 print('Commit message:') 191 print('\n'.join(' ' + i for i in commit_msg.splitlines())) 192 193 check_call(['git', 'add', 'DEPS'], cwd=current_dir) 194 check_call(['git', 'commit', '--quiet', '-m', commit_msg], cwd=current_dir) 195 196 # Pull the dependency to the right revision. This is surprising to users 197 # otherwise. 198 for _head, roll_to, full_dir in sorted(rolls.values()): 199 check_call(['git', 'checkout', '--quiet', roll_to], cwd=full_dir) 200 201 202def main(): 203 parser = argparse.ArgumentParser(description=__doc__) 204 parser.add_argument( 205 '--ignore-dirty-tree', action='store_true', 206 help='Roll anyways, even if there is a diff.') 207 parser.add_argument( 208 '-r', '--reviewer', 209 help='To specify multiple reviewers, use comma separated list, e.g. ' 210 '-r joe,jane,john. Defaults to @chromium.org') 211 parser.add_argument('-b', '--bug', help='Associate a bug number to the roll') 212 # It is important that --no-log continues to work, as it is used by 213 # internal -> external rollers. Please do not remove or break it. 214 parser.add_argument( 215 '--no-log', action='store_true', 216 help='Do not include the short log in the commit message') 217 parser.add_argument( 218 '--log-limit', type=int, default=100, 219 help='Trim log after N commits (default: %(default)s)') 220 parser.add_argument( 221 '--roll-to', default='origin/master', 222 help='Specify the new commit to roll to (default: %(default)s)') 223 parser.add_argument( 224 '--key', action='append', default=[], 225 help='Regex(es) for dependency in DEPS file') 226 parser.add_argument('dep_path', nargs='+', help='Path(s) to dependency') 227 args = parser.parse_args() 228 229 if len(args.dep_path) > 1: 230 if args.roll_to != 'origin/master': 231 parser.error( 232 'Can\'t use multiple paths to roll simultaneously and --roll-to') 233 if args.key: 234 parser.error( 235 'Can\'t use multiple paths to roll simultaneously and --key') 236 reviewers = None 237 if args.reviewer: 238 reviewers = args.reviewer.split(',') 239 for i, r in enumerate(reviewers): 240 if not '@' in r: 241 reviewers[i] = r + '@chromium.org' 242 243 gclient_root = gclient(['root']) 244 current_dir = os.getcwd() 245 dependencies = sorted(d.replace('\\', '/').rstrip('/') for d in args.dep_path) 246 cmdline = 'roll-dep ' + ' '.join(dependencies) + ''.join( 247 ' --key ' + k for k in args.key) 248 try: 249 if not args.ignore_dirty_tree and not is_pristine(current_dir): 250 raise Error( 251 'Ensure %s is clean first (no non-merged commits).' % current_dir) 252 # First gather all the information without modifying anything, except for a 253 # git fetch. 254 rolls = {} 255 for dependency in dependencies: 256 full_dir = os.path.normpath(os.path.join(gclient_root, dependency)) 257 if not os.path.isdir(full_dir): 258 print('Dependency %s not found at %s' % (dependency, full_dir)) 259 full_dir = os.path.normpath(os.path.join(current_dir, dependency)) 260 print('Will look for relative dependency at %s' % full_dir) 261 if not os.path.isdir(full_dir): 262 raise Error('Directory not found: %s (%s)' % (dependency, full_dir)) 263 264 head, roll_to = calculate_roll(full_dir, dependency, args.roll_to) 265 if roll_to == head: 266 if len(dependencies) == 1: 267 raise AlreadyRolledError('No revision to roll!') 268 print('%s: Already at latest commit %s' % (dependency, roll_to)) 269 else: 270 print( 271 '%s: Rolling from %s to %s' % (dependency, head[:10], roll_to[:10])) 272 rolls[dependency] = (head, roll_to, full_dir) 273 274 logs = [] 275 setdep_args = [] 276 for dependency, (head, roll_to, full_dir) in sorted(rolls.items()): 277 log = generate_commit_message( 278 full_dir, dependency, head, roll_to, args.no_log, args.log_limit) 279 logs.append(log) 280 setdep_args.extend(['-r', '{}@{}'.format(dependency, roll_to)]) 281 282 gclient(['setdep'] + setdep_args) 283 284 commit_msg = gen_commit_msg(logs, cmdline, reviewers, args.bug) 285 finalize(commit_msg, current_dir, rolls) 286 except Error as e: 287 sys.stderr.write('error: %s\n' % e) 288 return 2 if isinstance(e, AlreadyRolledError) else 1 289 except subprocess.CalledProcessError: 290 return 1 291 292 print('') 293 if not reviewers: 294 print('You forgot to pass -r, make sure to insert a R=foo@example.com line') 295 print('to the commit description before emailing.') 296 print('') 297 print('Run:') 298 print(' git cl upload --send-mail') 299 return 0 300 301 302if __name__ == '__main__': 303 sys.exit(main()) 304