1#!/usr/bin/env python3 2"""Check proposed changes for common issues.""" 3import re 4import sys 5import shutil 6import os.path 7import subprocess 8import sysconfig 9 10import reindent 11import untabify 12 13 14# Excluded directories which are copies of external libraries: 15# don't check their coding style 16EXCLUDE_DIRS = [os.path.join('Modules', '_ctypes', 'libffi_osx'), 17 os.path.join('Modules', '_ctypes', 'libffi_msvc'), 18 os.path.join('Modules', '_decimal', 'libmpdec'), 19 os.path.join('Modules', 'expat'), 20 os.path.join('Modules', 'zlib')] 21SRCDIR = sysconfig.get_config_var('srcdir') 22 23 24def n_files_str(count): 25 """Return 'N file(s)' with the proper plurality on 'file'.""" 26 return "{} file{}".format(count, "s" if count != 1 else "") 27 28 29def status(message, modal=False, info=None): 30 """Decorator to output status info to stdout.""" 31 def decorated_fxn(fxn): 32 def call_fxn(*args, **kwargs): 33 sys.stdout.write(message + ' ... ') 34 sys.stdout.flush() 35 result = fxn(*args, **kwargs) 36 if not modal and not info: 37 print("done") 38 elif info: 39 print(info(result)) 40 else: 41 print("yes" if result else "NO") 42 return result 43 return call_fxn 44 return decorated_fxn 45 46 47def get_git_branch(): 48 """Get the symbolic name for the current git branch""" 49 cmd = "git rev-parse --abbrev-ref HEAD".split() 50 try: 51 return subprocess.check_output(cmd, 52 stderr=subprocess.DEVNULL, 53 cwd=SRCDIR) 54 except subprocess.CalledProcessError: 55 return None 56 57 58def get_git_upstream_remote(): 59 """Get the remote name to use for upstream branches 60 61 Uses "upstream" if it exists, "origin" otherwise 62 """ 63 cmd = "git remote get-url upstream".split() 64 try: 65 subprocess.check_output(cmd, 66 stderr=subprocess.DEVNULL, 67 cwd=SRCDIR) 68 except subprocess.CalledProcessError: 69 return "origin" 70 return "upstream" 71 72 73@status("Getting base branch for PR", 74 info=lambda x: x if x is not None else "not a PR branch") 75def get_base_branch(): 76 if not os.path.exists(os.path.join(SRCDIR, '.git')): 77 # Not a git checkout, so there's no base branch 78 return None 79 version = sys.version_info 80 if version.releaselevel == 'alpha': 81 base_branch = "master" 82 else: 83 base_branch = "{0.major}.{0.minor}".format(version) 84 this_branch = get_git_branch() 85 if this_branch is None or this_branch == base_branch: 86 # Not on a git PR branch, so there's no base branch 87 return None 88 upstream_remote = get_git_upstream_remote() 89 return upstream_remote + "/" + base_branch 90 91 92@status("Getting the list of files that have been added/changed", 93 info=lambda x: n_files_str(len(x))) 94def changed_files(base_branch=None): 95 """Get the list of changed or added files from git.""" 96 if os.path.exists(os.path.join(SRCDIR, '.git')): 97 # We just use an existence check here as: 98 # directory = normal git checkout/clone 99 # file = git worktree directory 100 if base_branch: 101 cmd = 'git diff --name-status ' + base_branch 102 else: 103 cmd = 'git status --porcelain' 104 filenames = [] 105 with subprocess.Popen(cmd.split(), 106 stdout=subprocess.PIPE, 107 cwd=SRCDIR) as st: 108 for line in st.stdout: 109 line = line.decode().rstrip() 110 status_text, filename = line.split(maxsplit=1) 111 status = set(status_text) 112 # modified, added or unmerged files 113 if not status.intersection('MAU'): 114 continue 115 if ' -> ' in filename: 116 # file is renamed 117 filename = filename.split(' -> ', 2)[1].strip() 118 filenames.append(filename) 119 else: 120 sys.exit('need a git checkout to get modified files') 121 122 filenames2 = [] 123 for filename in filenames: 124 # Normalize the path to be able to match using .startswith() 125 filename = os.path.normpath(filename) 126 if any(filename.startswith(path) for path in EXCLUDE_DIRS): 127 # Exclude the file 128 continue 129 filenames2.append(filename) 130 131 return filenames2 132 133 134def report_modified_files(file_paths): 135 count = len(file_paths) 136 if count == 0: 137 return n_files_str(count) 138 else: 139 lines = ["{}:".format(n_files_str(count))] 140 for path in file_paths: 141 lines.append(" {}".format(path)) 142 return "\n".join(lines) 143 144 145@status("Fixing Python file whitespace", info=report_modified_files) 146def normalize_whitespace(file_paths): 147 """Make sure that the whitespace for .py files have been normalized.""" 148 reindent.makebackup = False # No need to create backups. 149 fixed = [path for path in file_paths if path.endswith('.py') and 150 reindent.check(os.path.join(SRCDIR, path))] 151 return fixed 152 153 154@status("Fixing C file whitespace", info=report_modified_files) 155def normalize_c_whitespace(file_paths): 156 """Report if any C files """ 157 fixed = [] 158 for path in file_paths: 159 abspath = os.path.join(SRCDIR, path) 160 with open(abspath, 'r') as f: 161 if '\t' not in f.read(): 162 continue 163 untabify.process(abspath, 8, verbose=False) 164 fixed.append(path) 165 return fixed 166 167 168ws_re = re.compile(br'\s+(\r?\n)$') 169 170@status("Fixing docs whitespace", info=report_modified_files) 171def normalize_docs_whitespace(file_paths): 172 fixed = [] 173 for path in file_paths: 174 abspath = os.path.join(SRCDIR, path) 175 try: 176 with open(abspath, 'rb') as f: 177 lines = f.readlines() 178 new_lines = [ws_re.sub(br'\1', line) for line in lines] 179 if new_lines != lines: 180 shutil.copyfile(abspath, abspath + '.bak') 181 with open(abspath, 'wb') as f: 182 f.writelines(new_lines) 183 fixed.append(path) 184 except Exception as err: 185 print('Cannot fix %s: %s' % (path, err)) 186 return fixed 187 188 189@status("Docs modified", modal=True) 190def docs_modified(file_paths): 191 """Report if any file in the Doc directory has been changed.""" 192 return bool(file_paths) 193 194 195@status("Misc/ACKS updated", modal=True) 196def credit_given(file_paths): 197 """Check if Misc/ACKS has been changed.""" 198 return os.path.join('Misc', 'ACKS') in file_paths 199 200 201@status("Misc/NEWS.d updated with `blurb`", modal=True) 202def reported_news(file_paths): 203 """Check if Misc/NEWS.d has been changed.""" 204 return any(p.startswith(os.path.join('Misc', 'NEWS.d', 'next')) 205 for p in file_paths) 206 207@status("configure regenerated", modal=True, info=str) 208def regenerated_configure(file_paths): 209 """Check if configure has been regenerated.""" 210 if 'configure.ac' in file_paths: 211 return "yes" if 'configure' in file_paths else "no" 212 else: 213 return "not needed" 214 215@status("pyconfig.h.in regenerated", modal=True, info=str) 216def regenerated_pyconfig_h_in(file_paths): 217 """Check if pyconfig.h.in has been regenerated.""" 218 if 'configure.ac' in file_paths: 219 return "yes" if 'pyconfig.h.in' in file_paths else "no" 220 else: 221 return "not needed" 222 223def travis(pull_request): 224 if pull_request == 'false': 225 print('Not a pull request; skipping') 226 return 227 base_branch = get_base_branch() 228 file_paths = changed_files(base_branch) 229 python_files = [fn for fn in file_paths if fn.endswith('.py')] 230 c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] 231 doc_files = [fn for fn in file_paths if fn.startswith('Doc') and 232 fn.endswith(('.rst', '.inc'))] 233 fixed = [] 234 fixed.extend(normalize_whitespace(python_files)) 235 fixed.extend(normalize_c_whitespace(c_files)) 236 fixed.extend(normalize_docs_whitespace(doc_files)) 237 if not fixed: 238 print('No whitespace issues found') 239 else: 240 print(f'Please fix the {len(fixed)} file(s) with whitespace issues') 241 print('(on UNIX you can run `make patchcheck` to make the fixes)') 242 sys.exit(1) 243 244def main(): 245 base_branch = get_base_branch() 246 file_paths = changed_files(base_branch) 247 python_files = [fn for fn in file_paths if fn.endswith('.py')] 248 c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] 249 doc_files = [fn for fn in file_paths if fn.startswith('Doc') and 250 fn.endswith(('.rst', '.inc'))] 251 misc_files = {p for p in file_paths if p.startswith('Misc')} 252 # PEP 8 whitespace rules enforcement. 253 normalize_whitespace(python_files) 254 # C rules enforcement. 255 normalize_c_whitespace(c_files) 256 # Doc whitespace enforcement. 257 normalize_docs_whitespace(doc_files) 258 # Docs updated. 259 docs_modified(doc_files) 260 # Misc/ACKS changed. 261 credit_given(misc_files) 262 # Misc/NEWS changed. 263 reported_news(misc_files) 264 # Regenerated configure, if necessary. 265 regenerated_configure(file_paths) 266 # Regenerated pyconfig.h.in, if necessary. 267 regenerated_pyconfig_h_in(file_paths) 268 269 # Test suite run and passed. 270 if python_files or c_files: 271 end = " and check for refleaks?" if c_files else "?" 272 print() 273 print("Did you run the test suite" + end) 274 275 276if __name__ == '__main__': 277 import argparse 278 parser = argparse.ArgumentParser(description=__doc__) 279 parser.add_argument('--travis', 280 help='Perform pass/fail checks') 281 args = parser.parse_args() 282 if args.travis: 283 travis(args.travis) 284 else: 285 main() 286