1# Copyright (c) 2013 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6"""Top-level presubmit script for Skia. 7 8See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts 9for more details about the presubmit API built into gcl. 10""" 11 12import collections 13import csv 14import fnmatch 15import os 16import re 17import subprocess 18import sys 19import traceback 20 21 22REVERT_CL_SUBJECT_PREFIX = 'Revert ' 23 24# Please add the complete email address here (and not just 'xyz@' or 'xyz'). 25PUBLIC_API_OWNERS = ( 26 'mtklein@google.com', 27 'reed@chromium.org', 28 'reed@google.com', 29 'bsalomon@chromium.org', 30 'bsalomon@google.com', 31 'djsollen@chromium.org', 32 'djsollen@google.com', 33 'hcm@chromium.org', 34 'hcm@google.com', 35) 36 37AUTHORS_FILE_NAME = 'AUTHORS' 38RELEASE_NOTES_FILE_NAME = 'RELEASE_NOTES.txt' 39 40DOCS_PREVIEW_URL = 'https://skia.org/?cl={issue}' 41GOLD_TRYBOT_URL = 'https://gold.skia.org/search?issue=' 42 43SERVICE_ACCOUNT_SUFFIX = [ 44 '@%s.iam.gserviceaccount.com' % project for project in [ 45 'skia-buildbots.google.com', 'skia-swarming-bots', 'skia-public', 46 'skia-corp.google.com', 'chops-service-accounts']] 47 48 49def _CheckChangeHasEol(input_api, output_api, source_file_filter=None): 50 """Checks that files end with at least one \n (LF).""" 51 eof_files = [] 52 for f in input_api.AffectedSourceFiles(source_file_filter): 53 contents = input_api.ReadFile(f, 'rb') 54 # Check that the file ends in at least one newline character. 55 if len(contents) > 1 and contents[-1:] != '\n': 56 eof_files.append(f.LocalPath()) 57 58 if eof_files: 59 return [output_api.PresubmitPromptWarning( 60 'These files should end in a newline character:', 61 items=eof_files)] 62 return [] 63 64 65def _JsonChecks(input_api, output_api): 66 """Run checks on any modified json files.""" 67 failing_files = [] 68 for affected_file in input_api.AffectedFiles(None): 69 affected_file_path = affected_file.LocalPath() 70 is_json = affected_file_path.endswith('.json') 71 is_metadata = (affected_file_path.startswith('site/') and 72 affected_file_path.endswith('/METADATA')) 73 if is_json or is_metadata: 74 try: 75 input_api.json.load(open(affected_file_path, 'r')) 76 except ValueError: 77 failing_files.append(affected_file_path) 78 79 results = [] 80 if failing_files: 81 results.append( 82 output_api.PresubmitError( 83 'The following files contain invalid json:\n%s\n\n' % 84 '\n'.join(failing_files))) 85 return results 86 87 88def _IfDefChecks(input_api, output_api): 89 """Ensures if/ifdef are not before includes. See skbug/3362 for details.""" 90 comment_block_start_pattern = re.compile('^\s*\/\*.*$') 91 comment_block_middle_pattern = re.compile('^\s+\*.*') 92 comment_block_end_pattern = re.compile('^\s+\*\/.*$') 93 single_line_comment_pattern = re.compile('^\s*//.*$') 94 def is_comment(line): 95 return (comment_block_start_pattern.match(line) or 96 comment_block_middle_pattern.match(line) or 97 comment_block_end_pattern.match(line) or 98 single_line_comment_pattern.match(line)) 99 100 empty_line_pattern = re.compile('^\s*$') 101 def is_empty_line(line): 102 return empty_line_pattern.match(line) 103 104 failing_files = [] 105 for affected_file in input_api.AffectedSourceFiles(None): 106 affected_file_path = affected_file.LocalPath() 107 if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'): 108 f = open(affected_file_path) 109 for line in f.xreadlines(): 110 if is_comment(line) or is_empty_line(line): 111 continue 112 # The below will be the first real line after comments and newlines. 113 if line.startswith('#if 0 '): 114 pass 115 elif line.startswith('#if ') or line.startswith('#ifdef '): 116 failing_files.append(affected_file_path) 117 break 118 119 results = [] 120 if failing_files: 121 results.append( 122 output_api.PresubmitError( 123 'The following files have #if or #ifdef before includes:\n%s\n\n' 124 'See https://bug.skia.org/3362 for why this should be fixed.' % 125 '\n'.join(failing_files))) 126 return results 127 128 129def _CopyrightChecks(input_api, output_api, source_file_filter=None): 130 results = [] 131 year_pattern = r'\d{4}' 132 year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern) 133 years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern) 134 copyright_pattern = ( 135 r'Copyright (\([cC]\) )?%s \w+' % years_pattern) 136 137 for affected_file in input_api.AffectedSourceFiles(source_file_filter): 138 if 'third_party' in affected_file.LocalPath(): 139 continue 140 contents = input_api.ReadFile(affected_file, 'rb') 141 if not re.search(copyright_pattern, contents): 142 results.append(output_api.PresubmitError( 143 '%s is missing a correct copyright header.' % affected_file)) 144 return results 145 146 147def _InfraTests(input_api, output_api): 148 """Run the infra tests.""" 149 results = [] 150 if not any(f.LocalPath().startswith('infra') 151 for f in input_api.AffectedFiles()): 152 return results 153 154 cmd = ['python', os.path.join('infra', 'bots', 'infra_tests.py')] 155 try: 156 subprocess.check_output(cmd) 157 except subprocess.CalledProcessError as e: 158 results.append(output_api.PresubmitError( 159 '`%s` failed:\n%s' % (' '.join(cmd), e.output))) 160 return results 161 162 163def _CheckGNFormatted(input_api, output_api): 164 """Make sure any .gn files we're changing have been formatted.""" 165 files = [] 166 for f in input_api.AffectedFiles(): 167 if (f.LocalPath().endswith('.gn') or 168 f.LocalPath().endswith('.gni')): 169 files.append(f) 170 if not files: 171 return [] 172 173 cmd = ['python', os.path.join('bin', 'fetch-gn')] 174 try: 175 subprocess.check_output(cmd) 176 except subprocess.CalledProcessError as e: 177 return [output_api.PresubmitError( 178 '`%s` failed:\n%s' % (' '.join(cmd), e.output))] 179 180 results = [] 181 for f in files: 182 gn = 'gn.exe' if 'win32' in sys.platform else 'gn' 183 gn = os.path.join(input_api.PresubmitLocalPath(), 'bin', gn) 184 cmd = [gn, 'format', '--dry-run', f.LocalPath()] 185 try: 186 subprocess.check_output(cmd) 187 except subprocess.CalledProcessError: 188 fix = 'bin/gn format ' + f.LocalPath() 189 results.append(output_api.PresubmitError( 190 '`%s` failed, try\n\t%s' % (' '.join(cmd), fix))) 191 return results 192 193def _CheckIncludesFormatted(input_api, output_api): 194 """Make sure #includes in files we're changing have been formatted.""" 195 files = [str(f) for f in input_api.AffectedFiles() if f.Action() != 'D'] 196 cmd = ['python', 197 'tools/rewrite_includes.py', 198 '--dry-run'] + files 199 if 0 != subprocess.call(cmd): 200 return [output_api.PresubmitError('`%s` failed' % ' '.join(cmd))] 201 return [] 202 203def _CheckCompileIsolate(input_api, output_api): 204 """Ensure that gen_compile_isolate.py does not change compile.isolate.""" 205 # Only run the check if files were added or removed. 206 results = [] 207 script = os.path.join('infra', 'bots', 'gen_compile_isolate.py') 208 isolate = os.path.join('infra', 'bots', 'compile.isolated') 209 for f in input_api.AffectedFiles(): 210 if f.Action() in ('A', 'D', 'R'): 211 break 212 if f.LocalPath() in (script, isolate): 213 break 214 else: 215 return results 216 217 cmd = ['python', script, 'test'] 218 try: 219 subprocess.check_output(cmd, stderr=subprocess.STDOUT) 220 except subprocess.CalledProcessError as e: 221 results.append(output_api.PresubmitError(e.output)) 222 return results 223 224 225class _WarningsAsErrors(): 226 def __init__(self, output_api): 227 self.output_api = output_api 228 self.old_warning = None 229 def __enter__(self): 230 self.old_warning = self.output_api.PresubmitPromptWarning 231 self.output_api.PresubmitPromptWarning = self.output_api.PresubmitError 232 return self.output_api 233 def __exit__(self, ex_type, ex_value, ex_traceback): 234 self.output_api.PresubmitPromptWarning = self.old_warning 235 236 237def _CheckDEPSValid(input_api, output_api): 238 """Ensure that DEPS contains valid entries.""" 239 results = [] 240 script = os.path.join('infra', 'bots', 'check_deps.py') 241 relevant_files = ('DEPS', script) 242 for f in input_api.AffectedFiles(): 243 if f.LocalPath() in relevant_files: 244 break 245 else: 246 return results 247 cmd = ['python', script] 248 try: 249 subprocess.check_output(cmd, stderr=subprocess.STDOUT) 250 except subprocess.CalledProcessError as e: 251 results.append(output_api.PresubmitError(e.output)) 252 return results 253 254 255def _CommonChecks(input_api, output_api): 256 """Presubmit checks common to upload and commit.""" 257 results = [] 258 sources = lambda x: (x.LocalPath().endswith('.h') or 259 x.LocalPath().endswith('.py') or 260 x.LocalPath().endswith('.sh') or 261 x.LocalPath().endswith('.m') or 262 x.LocalPath().endswith('.mm') or 263 x.LocalPath().endswith('.go') or 264 x.LocalPath().endswith('.c') or 265 x.LocalPath().endswith('.cc') or 266 x.LocalPath().endswith('.cpp')) 267 results.extend(_CheckChangeHasEol( 268 input_api, output_api, source_file_filter=sources)) 269 with _WarningsAsErrors(output_api): 270 results.extend(input_api.canned_checks.CheckChangeHasNoCR( 271 input_api, output_api, source_file_filter=sources)) 272 results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace( 273 input_api, output_api, source_file_filter=sources)) 274 results.extend(_JsonChecks(input_api, output_api)) 275 results.extend(_IfDefChecks(input_api, output_api)) 276 results.extend(_CopyrightChecks(input_api, output_api, 277 source_file_filter=sources)) 278 results.extend(_CheckCompileIsolate(input_api, output_api)) 279 results.extend(_CheckDEPSValid(input_api, output_api)) 280 results.extend(_CheckIncludesFormatted(input_api, output_api)) 281 return results 282 283 284def CheckChangeOnUpload(input_api, output_api): 285 """Presubmit checks for the change on upload.""" 286 results = [] 287 results.extend(_CommonChecks(input_api, output_api)) 288 # Run on upload, not commit, since the presubmit bot apparently doesn't have 289 # coverage or Go installed. 290 results.extend(_InfraTests(input_api, output_api)) 291 292 results.extend(_CheckGNFormatted(input_api, output_api)) 293 results.extend(_CheckReleaseNotesForPublicAPI(input_api, output_api)) 294 return results 295 296 297class CodeReview(object): 298 """Abstracts which codereview tool is used for the specified issue.""" 299 300 def __init__(self, input_api): 301 self._issue = input_api.change.issue 302 self._gerrit = input_api.gerrit 303 304 def GetOwnerEmail(self): 305 return self._gerrit.GetChangeOwner(self._issue) 306 307 def GetSubject(self): 308 return self._gerrit.GetChangeInfo(self._issue)['subject'] 309 310 def GetDescription(self): 311 return self._gerrit.GetChangeDescription(self._issue) 312 313 def GetReviewers(self): 314 code_review_label = ( 315 self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review']) 316 return [r['email'] for r in code_review_label.get('all', [])] 317 318 def GetApprovers(self): 319 approvers = [] 320 code_review_label = ( 321 self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review']) 322 for m in code_review_label.get('all', []): 323 if m.get("value") == 1: 324 approvers.append(m["email"]) 325 return approvers 326 327 328def _CheckOwnerIsInAuthorsFile(input_api, output_api): 329 results = [] 330 if input_api.change.issue: 331 cr = CodeReview(input_api) 332 333 owner_email = cr.GetOwnerEmail() 334 335 # Service accounts don't need to be in AUTHORS. 336 for suffix in SERVICE_ACCOUNT_SUFFIX: 337 if owner_email.endswith(suffix): 338 return results 339 340 try: 341 authors_content = '' 342 for line in open(AUTHORS_FILE_NAME): 343 if not line.startswith('#'): 344 authors_content += line 345 email_fnmatches = re.findall('<(.*)>', authors_content) 346 for email_fnmatch in email_fnmatches: 347 if fnmatch.fnmatch(owner_email, email_fnmatch): 348 # Found a match, the user is in the AUTHORS file break out of the loop 349 break 350 else: 351 results.append( 352 output_api.PresubmitError( 353 'The email %s is not in Skia\'s AUTHORS file.\n' 354 'Issue owner, this CL must include an addition to the Skia AUTHORS ' 355 'file.' 356 % owner_email)) 357 except IOError: 358 # Do not fail if authors file cannot be found. 359 traceback.print_exc() 360 input_api.logging.error('AUTHORS file not found!') 361 362 return results 363 364 365def _CheckReleaseNotesForPublicAPI(input_api, output_api): 366 """Checks to see if release notes file is updated with public API changes.""" 367 results = [] 368 public_api_changed = False 369 release_file_changed = False 370 for affected_file in input_api.AffectedFiles(): 371 affected_file_path = affected_file.LocalPath() 372 file_path, file_ext = os.path.splitext(affected_file_path) 373 # We only care about files that end in .h and are under the top-level 374 # include dir, but not include/private. 375 if (file_ext == '.h' and 376 file_path.split(os.path.sep)[0] == 'include' and 377 'private' not in file_path): 378 public_api_changed = True 379 elif affected_file_path == RELEASE_NOTES_FILE_NAME: 380 release_file_changed = True 381 382 if public_api_changed and not release_file_changed: 383 results.append(output_api.PresubmitPromptWarning( 384 'If this change affects a client API, please add a summary line ' 385 'to the %s file.' % RELEASE_NOTES_FILE_NAME)) 386 return results 387 388 389 390def _CheckLGTMsForPublicAPI(input_api, output_api): 391 """Check LGTMs for public API changes. 392 393 For public API files make sure there is an LGTM from the list of owners in 394 PUBLIC_API_OWNERS. 395 """ 396 results = [] 397 requires_owner_check = False 398 for affected_file in input_api.AffectedFiles(): 399 affected_file_path = affected_file.LocalPath() 400 file_path, file_ext = os.path.splitext(affected_file_path) 401 # We only care about files that end in .h and are under the top-level 402 # include dir, but not include/private. 403 if (file_ext == '.h' and 404 'include' == file_path.split(os.path.sep)[0] and 405 'private' not in file_path): 406 requires_owner_check = True 407 408 if not requires_owner_check: 409 return results 410 411 lgtm_from_owner = False 412 if input_api.change.issue: 413 cr = CodeReview(input_api) 414 415 if re.match(REVERT_CL_SUBJECT_PREFIX, cr.GetSubject(), re.I): 416 # It is a revert CL, ignore the public api owners check. 417 return results 418 419 if input_api.gerrit: 420 for reviewer in cr.GetReviewers(): 421 if reviewer in PUBLIC_API_OWNERS: 422 # If an owner is specified as an reviewer in Gerrit then ignore the 423 # public api owners check. 424 return results 425 else: 426 match = re.search(r'^TBR=(.*)$', cr.GetDescription(), re.M) 427 if match: 428 tbr_section = match.group(1).strip().split(' ')[0] 429 tbr_entries = tbr_section.split(',') 430 for owner in PUBLIC_API_OWNERS: 431 if owner in tbr_entries or owner.split('@')[0] in tbr_entries: 432 # If an owner is specified in the TBR= line then ignore the public 433 # api owners check. 434 return results 435 436 if cr.GetOwnerEmail() in PUBLIC_API_OWNERS: 437 # An owner created the CL that is an automatic LGTM. 438 lgtm_from_owner = True 439 440 for approver in cr.GetApprovers(): 441 if approver in PUBLIC_API_OWNERS: 442 # Found an lgtm in a message from an owner. 443 lgtm_from_owner = True 444 break 445 446 if not lgtm_from_owner: 447 results.append( 448 output_api.PresubmitError( 449 "If this CL adds to or changes Skia's public API, you need an LGTM " 450 "from any of %s. If this CL only removes from or doesn't change " 451 "Skia's public API, please add a short note to the CL saying so. " 452 "Add one of the owners as a reviewer to your CL as well as to the " 453 "TBR= line. If you don't know if this CL affects Skia's public " 454 "API, treat it like it does." % str(PUBLIC_API_OWNERS))) 455 return results 456 457 458def PostUploadHook(gerrit, change, output_api): 459 """git cl upload will call this hook after the issue is created/modified. 460 461 This hook does the following: 462 * Adds a link to preview docs changes if there are any docs changes in the CL. 463 * Adds 'No-Try: true' if the CL contains only docs changes. 464 """ 465 if not change.issue: 466 return [] 467 468 # Skip PostUploadHooks for all auto-commit service account bots. New 469 # patchsets (caused due to PostUploadHooks) invalidates the CQ+2 vote from 470 # the "--use-commit-queue" flag to "git cl upload". 471 for suffix in SERVICE_ACCOUNT_SUFFIX: 472 if change.author_email.endswith(suffix): 473 return [] 474 475 results = [] 476 at_least_one_docs_change = False 477 all_docs_changes = True 478 for affected_file in change.AffectedFiles(): 479 affected_file_path = affected_file.LocalPath() 480 file_path, _ = os.path.splitext(affected_file_path) 481 if 'site' == file_path.split(os.path.sep)[0]: 482 at_least_one_docs_change = True 483 else: 484 all_docs_changes = False 485 if at_least_one_docs_change and not all_docs_changes: 486 break 487 488 footers = change.GitFootersFromDescription() 489 description_changed = False 490 491 # If the change includes only doc changes then add No-Try: true in the 492 # CL's description if it does not exist yet. 493 if all_docs_changes and 'true' not in footers.get('No-Try', []): 494 description_changed = True 495 change.AddDescriptionFooter('No-Try', 'true') 496 results.append( 497 output_api.PresubmitNotifyResult( 498 'This change has only doc changes. Automatically added ' 499 '\'No-Try: true\' to the CL\'s description')) 500 501 # If there is at least one docs change then add preview link in the CL's 502 # description if it does not already exist there. 503 docs_preview_link = DOCS_PREVIEW_URL.format(issue=change.issue) 504 if (at_least_one_docs_change 505 and docs_preview_link not in footers.get('Docs-Preview', [])): 506 # Automatically add a link to where the docs can be previewed. 507 description_changed = True 508 change.AddDescriptionFooter('Docs-Preview', docs_preview_link) 509 results.append( 510 output_api.PresubmitNotifyResult( 511 'Automatically added a link to preview the docs changes to the ' 512 'CL\'s description')) 513 514 # If the description has changed update it. 515 if description_changed: 516 gerrit.UpdateDescription( 517 change.FullDescriptionText(), change.issue) 518 519 return results 520 521 522def CheckChangeOnCommit(input_api, output_api): 523 """Presubmit checks for the change on commit.""" 524 results = [] 525 results.extend(_CommonChecks(input_api, output_api)) 526 results.extend(_CheckLGTMsForPublicAPI(input_api, output_api)) 527 results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api)) 528 # Checks for the presence of 'DO NOT''SUBMIT' in CL description and in 529 # content of files. 530 results.extend( 531 input_api.canned_checks.CheckDoNotSubmit(input_api, output_api)) 532 return results 533