1#!/usr/bin/env python
2# Copyright 2015 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Rolls DEPS controlled dependency.
7
8Works only with git checkout and git dependencies.  Currently this
9script will always roll to the tip of to origin/master.
10"""
11
12from __future__ import print_function
13
14import argparse
15import os
16import re
17import subprocess2
18import sys
19
20NEED_SHELL = sys.platform.startswith('win')
21GCLIENT_PATH = os.path.join(
22    os.path.dirname(os.path.abspath(__file__)), 'gclient.py')
23
24
25# Commit subject that will be considered a roll. In the format generated by the
26# git log used, so it's "<year>-<month>-<day> <author> <subject>"
27_ROLL_SUBJECT = re.compile(
28    # Date
29    r'^\d\d\d\d-\d\d-\d\d '
30    # Author
31    r'[^ ]+ '
32    # Subject
33    r'('
34      # Generated by
35      # https://skia.googlesource.com/buildbot/+/master/autoroll/go/repo_manager/deps_repo_manager.go
36      r'Roll [^ ]+ [a-f0-9]+\.\.[a-f0-9]+ \(\d+ commits\)'
37      r'|'
38      # Generated by
39      # https://chromium.googlesource.com/infra/infra/+/master/recipes/recipe_modules/recipe_autoroller/api.py
40      r'Roll recipe dependencies \(trivial\)\.'
41    r')$')
42
43
44class Error(Exception):
45  pass
46
47
48class AlreadyRolledError(Error):
49  pass
50
51
52def check_output(*args, **kwargs):
53  """subprocess.check_output() passing shell=True on Windows for git."""
54  kwargs.setdefault('shell', NEED_SHELL)
55  return subprocess2.check_output(*args, **kwargs).decode('utf-8')
56
57
58def check_call(*args, **kwargs):
59  """subprocess.check_call() passing shell=True on Windows for git."""
60  kwargs.setdefault('shell', NEED_SHELL)
61  subprocess2.check_call(*args, **kwargs)
62
63
64def return_code(*args, **kwargs):
65  """subprocess.call() passing shell=True on Windows for git and
66  subprocess2.VOID for stdout and stderr."""
67  kwargs.setdefault('shell', NEED_SHELL)
68  kwargs.setdefault('stdout', subprocess2.VOID)
69  kwargs.setdefault('stderr', subprocess2.VOID)
70  return subprocess2.call(*args, **kwargs)
71
72
73def is_pristine(root):
74  """Returns True if a git checkout is pristine."""
75  # Check both origin/master and origin/main since many projects are
76  # transitioning to origin/main.
77  for branch in ('origin/main', 'origin/master'):
78    # `git rev-parse --verify` has a non-zero return code if the revision
79    # doesn't exist.
80    rev_cmd = ['git', 'rev-parse', '--verify', '--quiet',
81               'refs/remotes/' + branch]
82    if return_code(rev_cmd, cwd=root) != 0:
83      continue
84
85    diff_cmd = ['git', 'diff', '--ignore-submodules', branch]
86    return (not check_output(diff_cmd, cwd=root).strip() and
87            not check_output(diff_cmd + ['--cached'], cwd=root).strip())
88
89
90  raise Error('Couldn\'t find any of origin/main or origin/master')
91
92def get_log_url(upstream_url, head, master):
93  """Returns an URL to read logs via a Web UI if applicable."""
94  if re.match(r'https://[^/]*\.googlesource\.com/', upstream_url):
95    # gitiles
96    return '%s/+log/%s..%s' % (upstream_url, head[:12], master[:12])
97  if upstream_url.startswith('https://github.com/'):
98    upstream_url = upstream_url.rstrip('/')
99    if upstream_url.endswith('.git'):
100      upstream_url = upstream_url[:-len('.git')]
101    return '%s/compare/%s...%s' % (upstream_url, head[:12], master[:12])
102  return None
103
104
105def should_show_log(upstream_url):
106  """Returns True if a short log should be included in the tree."""
107  # Skip logs for very active projects.
108  if upstream_url.endswith('/v8/v8.git'):
109    return False
110  if 'webrtc' in upstream_url:
111    return False
112  return True
113
114
115def gclient(args):
116  """Executes gclient with the given args and returns the stdout."""
117  return check_output([sys.executable, GCLIENT_PATH] + args).strip()
118
119
120def generate_commit_message(
121    full_dir, dependency, head, roll_to, no_log, log_limit):
122  """Creates the commit message for this specific roll."""
123  commit_range = '%s..%s' % (head, roll_to)
124  commit_range_for_header = '%s..%s' % (head[:9], roll_to[:9])
125  upstream_url = check_output(
126      ['git', 'config', 'remote.origin.url'], cwd=full_dir).strip()
127  log_url = get_log_url(upstream_url, head, roll_to)
128  cmd = ['git', 'log', commit_range, '--date=short', '--no-merges']
129  logs = check_output(
130      # Args with '=' are automatically quoted.
131      cmd + ['--format=%ad %ae %s', '--'],
132      cwd=full_dir).rstrip()
133  logs = re.sub(r'(?m)^(\d\d\d\d-\d\d-\d\d [^@]+)@[^ ]+( .*)$', r'\1\2', logs)
134  lines = logs.splitlines()
135  cleaned_lines = [l for l in lines if not _ROLL_SUBJECT.match(l)]
136  logs = '\n'.join(cleaned_lines) + '\n'
137
138  nb_commits = len(lines)
139  rolls = nb_commits - len(cleaned_lines)
140  header = 'Roll %s/ %s (%d commit%s%s)\n\n' % (
141      dependency,
142      commit_range_for_header,
143      nb_commits,
144      's' if nb_commits > 1 else '',
145      ('; %s trivial rolls' % rolls) if rolls else '')
146  log_section = ''
147  if log_url:
148    log_section = log_url + '\n\n'
149  log_section += '$ %s ' % ' '.join(cmd)
150  log_section += '--format=\'%ad %ae %s\'\n'
151  log_section = log_section.replace(commit_range, commit_range_for_header)
152  # It is important that --no-log continues to work, as it is used by
153  # internal -> external rollers. Please do not remove or break it.
154  if not no_log and should_show_log(upstream_url):
155    if len(cleaned_lines) > log_limit:
156      # Keep the first N/2 log entries and last N/2 entries.
157      lines = logs.splitlines(True)
158      lines = lines[:log_limit//2] + ['(...)\n'] + lines[-log_limit//2:]
159      logs = ''.join(lines)
160    log_section += logs
161  return header + log_section
162
163
164def calculate_roll(full_dir, dependency, roll_to):
165  """Calculates the roll for a dependency by processing gclient_dict, and
166  fetching the dependency via git.
167  """
168  head = gclient(['getdep', '-r', dependency])
169  if not head:
170    raise Error('%s is unpinned.' % dependency)
171  check_call(['git', 'fetch', 'origin', '--quiet'], cwd=full_dir)
172  roll_to = check_output(['git', 'rev-parse', roll_to], cwd=full_dir).strip()
173  return head, roll_to
174
175
176def gen_commit_msg(logs, cmdline, reviewers, bug):
177  """Returns the final commit message."""
178  commit_msg = ''
179  if len(logs) > 1:
180    commit_msg = 'Rolling %d dependencies\n\n' % len(logs)
181  commit_msg += '\n\n'.join(logs)
182  commit_msg += '\nCreated with:\n  ' + cmdline + '\n'
183  commit_msg += 'R=%s\n' % ','.join(reviewers) if reviewers else ''
184  commit_msg += '\nBug: %s\n' % bug if bug else ''
185  return commit_msg
186
187
188def finalize(commit_msg, current_dir, rolls):
189  """Commits changes to the DEPS file, then uploads a CL."""
190  print('Commit message:')
191  print('\n'.join('    ' + i for i in commit_msg.splitlines()))
192
193  check_call(['git', 'add', 'DEPS'], cwd=current_dir)
194  check_call(['git', 'commit', '--quiet', '-m', commit_msg], cwd=current_dir)
195
196  # Pull the dependency to the right revision. This is surprising to users
197  # otherwise.
198  for _head, roll_to, full_dir in sorted(rolls.values()):
199    check_call(['git', 'checkout', '--quiet', roll_to], cwd=full_dir)
200
201
202def main():
203  parser = argparse.ArgumentParser(description=__doc__)
204  parser.add_argument(
205      '--ignore-dirty-tree', action='store_true',
206      help='Roll anyways, even if there is a diff.')
207  parser.add_argument(
208      '-r', '--reviewer',
209      help='To specify multiple reviewers, use comma separated list, e.g. '
210           '-r joe,jane,john. Defaults to @chromium.org')
211  parser.add_argument('-b', '--bug', help='Associate a bug number to the roll')
212  # It is important that --no-log continues to work, as it is used by
213  # internal -> external rollers. Please do not remove or break it.
214  parser.add_argument(
215      '--no-log', action='store_true',
216      help='Do not include the short log in the commit message')
217  parser.add_argument(
218      '--log-limit', type=int, default=100,
219      help='Trim log after N commits (default: %(default)s)')
220  parser.add_argument(
221      '--roll-to', default='origin/master',
222      help='Specify the new commit to roll to (default: %(default)s)')
223  parser.add_argument(
224      '--key', action='append', default=[],
225      help='Regex(es) for dependency in DEPS file')
226  parser.add_argument('dep_path', nargs='+', help='Path(s) to dependency')
227  args = parser.parse_args()
228
229  if len(args.dep_path) > 1:
230    if args.roll_to != 'origin/master':
231      parser.error(
232          'Can\'t use multiple paths to roll simultaneously and --roll-to')
233    if args.key:
234      parser.error(
235          'Can\'t use multiple paths to roll simultaneously and --key')
236  reviewers = None
237  if args.reviewer:
238    reviewers = args.reviewer.split(',')
239    for i, r in enumerate(reviewers):
240      if not '@' in r:
241        reviewers[i] = r + '@chromium.org'
242
243  gclient_root = gclient(['root'])
244  current_dir = os.getcwd()
245  dependencies = sorted(d.replace('\\', '/').rstrip('/') for d in args.dep_path)
246  cmdline = 'roll-dep ' + ' '.join(dependencies) + ''.join(
247      ' --key ' + k for k in args.key)
248  try:
249    if not args.ignore_dirty_tree and not is_pristine(current_dir):
250      raise Error(
251          'Ensure %s is clean first (no non-merged commits).' % current_dir)
252    # First gather all the information without modifying anything, except for a
253    # git fetch.
254    rolls = {}
255    for dependency in dependencies:
256      full_dir = os.path.normpath(os.path.join(gclient_root, dependency))
257      if not os.path.isdir(full_dir):
258        print('Dependency %s not found at %s' % (dependency, full_dir))
259        full_dir = os.path.normpath(os.path.join(current_dir, dependency))
260        print('Will look for relative dependency at %s' % full_dir)
261        if not os.path.isdir(full_dir):
262          raise Error('Directory not found: %s (%s)' % (dependency, full_dir))
263
264      head, roll_to = calculate_roll(full_dir, dependency, args.roll_to)
265      if roll_to == head:
266        if len(dependencies) == 1:
267          raise AlreadyRolledError('No revision to roll!')
268        print('%s: Already at latest commit %s' % (dependency, roll_to))
269      else:
270        print(
271            '%s: Rolling from %s to %s' % (dependency, head[:10], roll_to[:10]))
272        rolls[dependency] = (head, roll_to, full_dir)
273
274    logs = []
275    setdep_args = []
276    for dependency, (head, roll_to, full_dir) in sorted(rolls.items()):
277      log = generate_commit_message(
278          full_dir, dependency, head, roll_to, args.no_log, args.log_limit)
279      logs.append(log)
280      setdep_args.extend(['-r', '{}@{}'.format(dependency, roll_to)])
281
282    gclient(['setdep'] + setdep_args)
283
284    commit_msg = gen_commit_msg(logs, cmdline, reviewers, args.bug)
285    finalize(commit_msg, current_dir, rolls)
286  except Error as e:
287    sys.stderr.write('error: %s\n' % e)
288    return 2 if isinstance(e, AlreadyRolledError) else 1
289  except subprocess.CalledProcessError:
290    return 1
291
292  print('')
293  if not reviewers:
294    print('You forgot to pass -r, make sure to insert a R=foo@example.com line')
295    print('to the commit description before emailing.')
296    print('')
297  print('Run:')
298  print('  git cl upload --send-mail')
299  return 0
300
301
302if __name__ == '__main__':
303  sys.exit(main())
304