1# Copyright (c) 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6"""Top-level presubmit script for Skia.
7
8See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
9for more details about the presubmit API built into gcl.
10"""
11
12import collections
13import csv
14import fnmatch
15import os
16import re
17import subprocess
18import sys
19import traceback
20
21
22REVERT_CL_SUBJECT_PREFIX = 'Revert '
23
24# Please add the complete email address here (and not just 'xyz@' or 'xyz').
25PUBLIC_API_OWNERS = (
26    'mtklein@google.com',
27    'reed@chromium.org',
28    'reed@google.com',
29    'bsalomon@chromium.org',
30    'bsalomon@google.com',
31    'djsollen@chromium.org',
32    'djsollen@google.com',
33    'hcm@chromium.org',
34    'hcm@google.com',
35)
36
37AUTHORS_FILE_NAME = 'AUTHORS'
38RELEASE_NOTES_FILE_NAME = 'RELEASE_NOTES.txt'
39
40DOCS_PREVIEW_URL = 'https://skia.org/?cl={issue}'
41GOLD_TRYBOT_URL = 'https://gold.skia.org/search?issue='
42
43SERVICE_ACCOUNT_SUFFIX = [
44    '@%s.iam.gserviceaccount.com' % project for project in [
45        'skia-buildbots.google.com', 'skia-swarming-bots', 'skia-public',
46        'skia-corp.google.com', 'chops-service-accounts']]
47
48
49def _CheckChangeHasEol(input_api, output_api, source_file_filter=None):
50  """Checks that files end with at least one \n (LF)."""
51  eof_files = []
52  for f in input_api.AffectedSourceFiles(source_file_filter):
53    contents = input_api.ReadFile(f, 'rb')
54    # Check that the file ends in at least one newline character.
55    if len(contents) > 1 and contents[-1:] != '\n':
56      eof_files.append(f.LocalPath())
57
58  if eof_files:
59    return [output_api.PresubmitPromptWarning(
60      'These files should end in a newline character:',
61      items=eof_files)]
62  return []
63
64
65def _JsonChecks(input_api, output_api):
66  """Run checks on any modified json files."""
67  failing_files = []
68  for affected_file in input_api.AffectedFiles(None):
69    affected_file_path = affected_file.LocalPath()
70    is_json = affected_file_path.endswith('.json')
71    is_metadata = (affected_file_path.startswith('site/') and
72                   affected_file_path.endswith('/METADATA'))
73    if is_json or is_metadata:
74      try:
75        input_api.json.load(open(affected_file_path, 'r'))
76      except ValueError:
77        failing_files.append(affected_file_path)
78
79  results = []
80  if failing_files:
81    results.append(
82        output_api.PresubmitError(
83            'The following files contain invalid json:\n%s\n\n' %
84                '\n'.join(failing_files)))
85  return results
86
87
88def _IfDefChecks(input_api, output_api):
89  """Ensures if/ifdef are not before includes. See skbug/3362 for details."""
90  comment_block_start_pattern = re.compile('^\s*\/\*.*$')
91  comment_block_middle_pattern = re.compile('^\s+\*.*')
92  comment_block_end_pattern = re.compile('^\s+\*\/.*$')
93  single_line_comment_pattern = re.compile('^\s*//.*$')
94  def is_comment(line):
95    return (comment_block_start_pattern.match(line) or
96            comment_block_middle_pattern.match(line) or
97            comment_block_end_pattern.match(line) or
98            single_line_comment_pattern.match(line))
99
100  empty_line_pattern = re.compile('^\s*$')
101  def is_empty_line(line):
102    return empty_line_pattern.match(line)
103
104  failing_files = []
105  for affected_file in input_api.AffectedSourceFiles(None):
106    affected_file_path = affected_file.LocalPath()
107    if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'):
108      f = open(affected_file_path)
109      for line in f.xreadlines():
110        if is_comment(line) or is_empty_line(line):
111          continue
112        # The below will be the first real line after comments and newlines.
113        if line.startswith('#if 0 '):
114          pass
115        elif line.startswith('#if ') or line.startswith('#ifdef '):
116          failing_files.append(affected_file_path)
117        break
118
119  results = []
120  if failing_files:
121    results.append(
122        output_api.PresubmitError(
123            'The following files have #if or #ifdef before includes:\n%s\n\n'
124            'See https://bug.skia.org/3362 for why this should be fixed.' %
125                '\n'.join(failing_files)))
126  return results
127
128
129def _CopyrightChecks(input_api, output_api, source_file_filter=None):
130  results = []
131  year_pattern = r'\d{4}'
132  year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern)
133  years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern)
134  copyright_pattern = (
135      r'Copyright (\([cC]\) )?%s \w+' % years_pattern)
136
137  for affected_file in input_api.AffectedSourceFiles(source_file_filter):
138    if 'third_party' in affected_file.LocalPath():
139      continue
140    contents = input_api.ReadFile(affected_file, 'rb')
141    if not re.search(copyright_pattern, contents):
142      results.append(output_api.PresubmitError(
143          '%s is missing a correct copyright header.' % affected_file))
144  return results
145
146
147def _InfraTests(input_api, output_api):
148  """Run the infra tests."""
149  results = []
150  if not any(f.LocalPath().startswith('infra')
151             for f in input_api.AffectedFiles()):
152    return results
153
154  cmd = ['python', os.path.join('infra', 'bots', 'infra_tests.py')]
155  try:
156    subprocess.check_output(cmd)
157  except subprocess.CalledProcessError as e:
158    results.append(output_api.PresubmitError(
159        '`%s` failed:\n%s' % (' '.join(cmd), e.output)))
160  return results
161
162
163def _CheckGNFormatted(input_api, output_api):
164  """Make sure any .gn files we're changing have been formatted."""
165  files = []
166  for f in input_api.AffectedFiles():
167    if (f.LocalPath().endswith('.gn') or
168        f.LocalPath().endswith('.gni')):
169      files.append(f)
170  if not files:
171    return []
172
173  cmd = ['python', os.path.join('bin', 'fetch-gn')]
174  try:
175    subprocess.check_output(cmd)
176  except subprocess.CalledProcessError as e:
177    return [output_api.PresubmitError(
178        '`%s` failed:\n%s' % (' '.join(cmd), e.output))]
179
180  results = []
181  for f in files:
182    gn = 'gn.exe' if 'win32' in sys.platform else 'gn'
183    gn = os.path.join(input_api.PresubmitLocalPath(), 'bin', gn)
184    cmd = [gn, 'format', '--dry-run', f.LocalPath()]
185    try:
186      subprocess.check_output(cmd)
187    except subprocess.CalledProcessError:
188      fix = 'bin/gn format ' + f.LocalPath()
189      results.append(output_api.PresubmitError(
190          '`%s` failed, try\n\t%s' % (' '.join(cmd), fix)))
191  return results
192
193def _CheckIncludesFormatted(input_api, output_api):
194  """Make sure #includes in files we're changing have been formatted."""
195  files = [str(f) for f in input_api.AffectedFiles() if f.Action() != 'D']
196  cmd = ['python',
197         'tools/rewrite_includes.py',
198         '--dry-run'] + files
199  if 0 != subprocess.call(cmd):
200    return [output_api.PresubmitError('`%s` failed' % ' '.join(cmd))]
201  return []
202
203def _CheckCompileIsolate(input_api, output_api):
204  """Ensure that gen_compile_isolate.py does not change compile.isolate."""
205  # Only run the check if files were added or removed.
206  results = []
207  script = os.path.join('infra', 'bots', 'gen_compile_isolate.py')
208  isolate = os.path.join('infra', 'bots', 'compile.isolated')
209  for f in input_api.AffectedFiles():
210    if f.Action() in ('A', 'D', 'R'):
211      break
212    if f.LocalPath() in (script, isolate):
213      break
214  else:
215    return results
216
217  cmd = ['python', script, 'test']
218  try:
219    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
220  except subprocess.CalledProcessError as e:
221    results.append(output_api.PresubmitError(e.output))
222  return results
223
224
225class _WarningsAsErrors():
226  def __init__(self, output_api):
227    self.output_api = output_api
228    self.old_warning = None
229  def __enter__(self):
230    self.old_warning = self.output_api.PresubmitPromptWarning
231    self.output_api.PresubmitPromptWarning = self.output_api.PresubmitError
232    return self.output_api
233  def __exit__(self, ex_type, ex_value, ex_traceback):
234    self.output_api.PresubmitPromptWarning = self.old_warning
235
236
237def _CheckDEPSValid(input_api, output_api):
238  """Ensure that DEPS contains valid entries."""
239  results = []
240  script = os.path.join('infra', 'bots', 'check_deps.py')
241  relevant_files = ('DEPS', script)
242  for f in input_api.AffectedFiles():
243    if f.LocalPath() in relevant_files:
244      break
245  else:
246    return results
247  cmd = ['python', script]
248  try:
249    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
250  except subprocess.CalledProcessError as e:
251    results.append(output_api.PresubmitError(e.output))
252  return results
253
254
255def _CommonChecks(input_api, output_api):
256  """Presubmit checks common to upload and commit."""
257  results = []
258  sources = lambda x: (x.LocalPath().endswith('.h') or
259                       x.LocalPath().endswith('.py') or
260                       x.LocalPath().endswith('.sh') or
261                       x.LocalPath().endswith('.m') or
262                       x.LocalPath().endswith('.mm') or
263                       x.LocalPath().endswith('.go') or
264                       x.LocalPath().endswith('.c') or
265                       x.LocalPath().endswith('.cc') or
266                       x.LocalPath().endswith('.cpp'))
267  results.extend(_CheckChangeHasEol(
268      input_api, output_api, source_file_filter=sources))
269  with _WarningsAsErrors(output_api):
270    results.extend(input_api.canned_checks.CheckChangeHasNoCR(
271        input_api, output_api, source_file_filter=sources))
272    results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
273        input_api, output_api, source_file_filter=sources))
274  results.extend(_JsonChecks(input_api, output_api))
275  results.extend(_IfDefChecks(input_api, output_api))
276  results.extend(_CopyrightChecks(input_api, output_api,
277                                  source_file_filter=sources))
278  results.extend(_CheckCompileIsolate(input_api, output_api))
279  results.extend(_CheckDEPSValid(input_api, output_api))
280  results.extend(_CheckIncludesFormatted(input_api, output_api))
281  return results
282
283
284def CheckChangeOnUpload(input_api, output_api):
285  """Presubmit checks for the change on upload."""
286  results = []
287  results.extend(_CommonChecks(input_api, output_api))
288  # Run on upload, not commit, since the presubmit bot apparently doesn't have
289  # coverage or Go installed.
290  results.extend(_InfraTests(input_api, output_api))
291
292  results.extend(_CheckGNFormatted(input_api, output_api))
293  results.extend(_CheckReleaseNotesForPublicAPI(input_api, output_api))
294  return results
295
296
297class CodeReview(object):
298  """Abstracts which codereview tool is used for the specified issue."""
299
300  def __init__(self, input_api):
301    self._issue = input_api.change.issue
302    self._gerrit = input_api.gerrit
303
304  def GetOwnerEmail(self):
305    return self._gerrit.GetChangeOwner(self._issue)
306
307  def GetSubject(self):
308    return self._gerrit.GetChangeInfo(self._issue)['subject']
309
310  def GetDescription(self):
311    return self._gerrit.GetChangeDescription(self._issue)
312
313  def GetReviewers(self):
314    code_review_label = (
315        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
316    return [r['email'] for r in code_review_label.get('all', [])]
317
318  def GetApprovers(self):
319    approvers = []
320    code_review_label = (
321        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
322    for m in code_review_label.get('all', []):
323      if m.get("value") == 1:
324        approvers.append(m["email"])
325    return approvers
326
327
328def _CheckOwnerIsInAuthorsFile(input_api, output_api):
329  results = []
330  if input_api.change.issue:
331    cr = CodeReview(input_api)
332
333    owner_email = cr.GetOwnerEmail()
334
335    # Service accounts don't need to be in AUTHORS.
336    for suffix in SERVICE_ACCOUNT_SUFFIX:
337      if owner_email.endswith(suffix):
338        return results
339
340    try:
341      authors_content = ''
342      for line in open(AUTHORS_FILE_NAME):
343        if not line.startswith('#'):
344          authors_content += line
345      email_fnmatches = re.findall('<(.*)>', authors_content)
346      for email_fnmatch in email_fnmatches:
347        if fnmatch.fnmatch(owner_email, email_fnmatch):
348          # Found a match, the user is in the AUTHORS file break out of the loop
349          break
350      else:
351        results.append(
352          output_api.PresubmitError(
353            'The email %s is not in Skia\'s AUTHORS file.\n'
354            'Issue owner, this CL must include an addition to the Skia AUTHORS '
355            'file.'
356            % owner_email))
357    except IOError:
358      # Do not fail if authors file cannot be found.
359      traceback.print_exc()
360      input_api.logging.error('AUTHORS file not found!')
361
362  return results
363
364
365def _CheckReleaseNotesForPublicAPI(input_api, output_api):
366  """Checks to see if release notes file is updated with public API changes."""
367  results = []
368  public_api_changed = False
369  release_file_changed = False
370  for affected_file in input_api.AffectedFiles():
371    affected_file_path = affected_file.LocalPath()
372    file_path, file_ext = os.path.splitext(affected_file_path)
373    # We only care about files that end in .h and are under the top-level
374    # include dir, but not include/private.
375    if (file_ext == '.h' and
376        file_path.split(os.path.sep)[0] == 'include' and
377        'private' not in file_path):
378      public_api_changed = True
379    elif affected_file_path == RELEASE_NOTES_FILE_NAME:
380      release_file_changed = True
381
382  if public_api_changed and not release_file_changed:
383    results.append(output_api.PresubmitPromptWarning(
384        'If this change affects a client API, please add a summary line '
385        'to the %s file.' % RELEASE_NOTES_FILE_NAME))
386  return results
387
388
389
390def _CheckLGTMsForPublicAPI(input_api, output_api):
391  """Check LGTMs for public API changes.
392
393  For public API files make sure there is an LGTM from the list of owners in
394  PUBLIC_API_OWNERS.
395  """
396  results = []
397  requires_owner_check = False
398  for affected_file in input_api.AffectedFiles():
399    affected_file_path = affected_file.LocalPath()
400    file_path, file_ext = os.path.splitext(affected_file_path)
401    # We only care about files that end in .h and are under the top-level
402    # include dir, but not include/private.
403    if (file_ext == '.h' and
404        'include' == file_path.split(os.path.sep)[0] and
405        'private' not in file_path):
406      requires_owner_check = True
407
408  if not requires_owner_check:
409    return results
410
411  lgtm_from_owner = False
412  if input_api.change.issue:
413    cr = CodeReview(input_api)
414
415    if re.match(REVERT_CL_SUBJECT_PREFIX, cr.GetSubject(), re.I):
416      # It is a revert CL, ignore the public api owners check.
417      return results
418
419    if input_api.gerrit:
420      for reviewer in cr.GetReviewers():
421        if reviewer in PUBLIC_API_OWNERS:
422          # If an owner is specified as an reviewer in Gerrit then ignore the
423          # public api owners check.
424          return results
425    else:
426      match = re.search(r'^TBR=(.*)$', cr.GetDescription(), re.M)
427      if match:
428        tbr_section = match.group(1).strip().split(' ')[0]
429        tbr_entries = tbr_section.split(',')
430        for owner in PUBLIC_API_OWNERS:
431          if owner in tbr_entries or owner.split('@')[0] in tbr_entries:
432            # If an owner is specified in the TBR= line then ignore the public
433            # api owners check.
434            return results
435
436    if cr.GetOwnerEmail() in PUBLIC_API_OWNERS:
437      # An owner created the CL that is an automatic LGTM.
438      lgtm_from_owner = True
439
440    for approver in cr.GetApprovers():
441      if approver in PUBLIC_API_OWNERS:
442        # Found an lgtm in a message from an owner.
443        lgtm_from_owner = True
444        break
445
446  if not lgtm_from_owner:
447    results.append(
448        output_api.PresubmitError(
449            "If this CL adds to or changes Skia's public API, you need an LGTM "
450            "from any of %s.  If this CL only removes from or doesn't change "
451            "Skia's public API, please add a short note to the CL saying so. "
452            "Add one of the owners as a reviewer to your CL as well as to the "
453            "TBR= line.  If you don't know if this CL affects Skia's public "
454            "API, treat it like it does." % str(PUBLIC_API_OWNERS)))
455  return results
456
457
458def PostUploadHook(gerrit, change, output_api):
459  """git cl upload will call this hook after the issue is created/modified.
460
461  This hook does the following:
462  * Adds a link to preview docs changes if there are any docs changes in the CL.
463  * Adds 'No-Try: true' if the CL contains only docs changes.
464  """
465  if not change.issue:
466    return []
467
468  # Skip PostUploadHooks for all auto-commit service account bots. New
469  # patchsets (caused due to PostUploadHooks) invalidates the CQ+2 vote from
470  # the "--use-commit-queue" flag to "git cl upload".
471  for suffix in SERVICE_ACCOUNT_SUFFIX:
472    if change.author_email.endswith(suffix):
473      return []
474
475  results = []
476  at_least_one_docs_change = False
477  all_docs_changes = True
478  for affected_file in change.AffectedFiles():
479    affected_file_path = affected_file.LocalPath()
480    file_path, _ = os.path.splitext(affected_file_path)
481    if 'site' == file_path.split(os.path.sep)[0]:
482      at_least_one_docs_change = True
483    else:
484      all_docs_changes = False
485    if at_least_one_docs_change and not all_docs_changes:
486      break
487
488  footers = change.GitFootersFromDescription()
489  description_changed = False
490
491  # If the change includes only doc changes then add No-Try: true in the
492  # CL's description if it does not exist yet.
493  if all_docs_changes and 'true' not in footers.get('No-Try', []):
494    description_changed = True
495    change.AddDescriptionFooter('No-Try', 'true')
496    results.append(
497        output_api.PresubmitNotifyResult(
498            'This change has only doc changes. Automatically added '
499            '\'No-Try: true\' to the CL\'s description'))
500
501  # If there is at least one docs change then add preview link in the CL's
502  # description if it does not already exist there.
503  docs_preview_link = DOCS_PREVIEW_URL.format(issue=change.issue)
504  if (at_least_one_docs_change
505      and docs_preview_link not in footers.get('Docs-Preview', [])):
506    # Automatically add a link to where the docs can be previewed.
507    description_changed = True
508    change.AddDescriptionFooter('Docs-Preview', docs_preview_link)
509    results.append(
510        output_api.PresubmitNotifyResult(
511            'Automatically added a link to preview the docs changes to the '
512            'CL\'s description'))
513
514  # If the description has changed update it.
515  if description_changed:
516    gerrit.UpdateDescription(
517        change.FullDescriptionText(), change.issue)
518
519  return results
520
521
522def CheckChangeOnCommit(input_api, output_api):
523  """Presubmit checks for the change on commit."""
524  results = []
525  results.extend(_CommonChecks(input_api, output_api))
526  results.extend(_CheckLGTMsForPublicAPI(input_api, output_api))
527  results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api))
528  # Checks for the presence of 'DO NOT''SUBMIT' in CL description and in
529  # content of files.
530  results.extend(
531      input_api.canned_checks.CheckDoNotSubmit(input_api, output_api))
532  return results
533