1#!/usr/bin/env python3
2"""Check proposed changes for common issues."""
3import re
4import sys
5import shutil
6import os.path
7import subprocess
8import sysconfig
9
10import reindent
11import untabify
12
13
14# Excluded directories which are copies of external libraries:
15# don't check their coding style
16EXCLUDE_DIRS = [os.path.join('Modules', '_ctypes', 'libffi_osx'),
17                os.path.join('Modules', '_ctypes', 'libffi_msvc'),
18                os.path.join('Modules', '_decimal', 'libmpdec'),
19                os.path.join('Modules', 'expat'),
20                os.path.join('Modules', 'zlib')]
21SRCDIR = sysconfig.get_config_var('srcdir')
22
23
24def n_files_str(count):
25    """Return 'N file(s)' with the proper plurality on 'file'."""
26    return "{} file{}".format(count, "s" if count != 1 else "")
27
28
29def status(message, modal=False, info=None):
30    """Decorator to output status info to stdout."""
31    def decorated_fxn(fxn):
32        def call_fxn(*args, **kwargs):
33            sys.stdout.write(message + ' ... ')
34            sys.stdout.flush()
35            result = fxn(*args, **kwargs)
36            if not modal and not info:
37                print("done")
38            elif info:
39                print(info(result))
40            else:
41                print("yes" if result else "NO")
42            return result
43        return call_fxn
44    return decorated_fxn
45
46
47def get_git_branch():
48    """Get the symbolic name for the current git branch"""
49    cmd = "git rev-parse --abbrev-ref HEAD".split()
50    try:
51        return subprocess.check_output(cmd,
52                                       stderr=subprocess.DEVNULL,
53                                       cwd=SRCDIR)
54    except subprocess.CalledProcessError:
55        return None
56
57
58def get_git_upstream_remote():
59    """Get the remote name to use for upstream branches
60
61    Uses "upstream" if it exists, "origin" otherwise
62    """
63    cmd = "git remote get-url upstream".split()
64    try:
65        subprocess.check_output(cmd,
66                                stderr=subprocess.DEVNULL,
67                                cwd=SRCDIR)
68    except subprocess.CalledProcessError:
69        return "origin"
70    return "upstream"
71
72
73@status("Getting base branch for PR",
74        info=lambda x: x if x is not None else "not a PR branch")
75def get_base_branch():
76    if not os.path.exists(os.path.join(SRCDIR, '.git')):
77        # Not a git checkout, so there's no base branch
78        return None
79    version = sys.version_info
80    if version.releaselevel == 'alpha':
81        base_branch = "master"
82    else:
83        base_branch = "{0.major}.{0.minor}".format(version)
84    this_branch = get_git_branch()
85    if this_branch is None or this_branch == base_branch:
86        # Not on a git PR branch, so there's no base branch
87        return None
88    upstream_remote = get_git_upstream_remote()
89    return upstream_remote + "/" + base_branch
90
91
92@status("Getting the list of files that have been added/changed",
93        info=lambda x: n_files_str(len(x)))
94def changed_files(base_branch=None):
95    """Get the list of changed or added files from git."""
96    if os.path.exists(os.path.join(SRCDIR, '.git')):
97        # We just use an existence check here as:
98        #  directory = normal git checkout/clone
99        #  file = git worktree directory
100        if base_branch:
101            cmd = 'git diff --name-status ' + base_branch
102        else:
103            cmd = 'git status --porcelain'
104        filenames = []
105        with subprocess.Popen(cmd.split(),
106                              stdout=subprocess.PIPE,
107                              cwd=SRCDIR) as st:
108            for line in st.stdout:
109                line = line.decode().rstrip()
110                status_text, filename = line.split(maxsplit=1)
111                status = set(status_text)
112                # modified, added or unmerged files
113                if not status.intersection('MAU'):
114                    continue
115                if ' -> ' in filename:
116                    # file is renamed
117                    filename = filename.split(' -> ', 2)[1].strip()
118                filenames.append(filename)
119    else:
120        sys.exit('need a git checkout to get modified files')
121
122    filenames2 = []
123    for filename in filenames:
124        # Normalize the path to be able to match using .startswith()
125        filename = os.path.normpath(filename)
126        if any(filename.startswith(path) for path in EXCLUDE_DIRS):
127            # Exclude the file
128            continue
129        filenames2.append(filename)
130
131    return filenames2
132
133
134def report_modified_files(file_paths):
135    count = len(file_paths)
136    if count == 0:
137        return n_files_str(count)
138    else:
139        lines = ["{}:".format(n_files_str(count))]
140        for path in file_paths:
141            lines.append("  {}".format(path))
142        return "\n".join(lines)
143
144
145@status("Fixing Python file whitespace", info=report_modified_files)
146def normalize_whitespace(file_paths):
147    """Make sure that the whitespace for .py files have been normalized."""
148    reindent.makebackup = False  # No need to create backups.
149    fixed = [path for path in file_paths if path.endswith('.py') and
150             reindent.check(os.path.join(SRCDIR, path))]
151    return fixed
152
153
154@status("Fixing C file whitespace", info=report_modified_files)
155def normalize_c_whitespace(file_paths):
156    """Report if any C files """
157    fixed = []
158    for path in file_paths:
159        abspath = os.path.join(SRCDIR, path)
160        with open(abspath, 'r') as f:
161            if '\t' not in f.read():
162                continue
163        untabify.process(abspath, 8, verbose=False)
164        fixed.append(path)
165    return fixed
166
167
168ws_re = re.compile(br'\s+(\r?\n)$')
169
170@status("Fixing docs whitespace", info=report_modified_files)
171def normalize_docs_whitespace(file_paths):
172    fixed = []
173    for path in file_paths:
174        abspath = os.path.join(SRCDIR, path)
175        try:
176            with open(abspath, 'rb') as f:
177                lines = f.readlines()
178            new_lines = [ws_re.sub(br'\1', line) for line in lines]
179            if new_lines != lines:
180                shutil.copyfile(abspath, abspath + '.bak')
181                with open(abspath, 'wb') as f:
182                    f.writelines(new_lines)
183                fixed.append(path)
184        except Exception as err:
185            print('Cannot fix %s: %s' % (path, err))
186    return fixed
187
188
189@status("Docs modified", modal=True)
190def docs_modified(file_paths):
191    """Report if any file in the Doc directory has been changed."""
192    return bool(file_paths)
193
194
195@status("Misc/ACKS updated", modal=True)
196def credit_given(file_paths):
197    """Check if Misc/ACKS has been changed."""
198    return os.path.join('Misc', 'ACKS') in file_paths
199
200
201@status("Misc/NEWS.d updated with `blurb`", modal=True)
202def reported_news(file_paths):
203    """Check if Misc/NEWS.d has been changed."""
204    return any(p.startswith(os.path.join('Misc', 'NEWS.d', 'next'))
205               for p in file_paths)
206
207@status("configure regenerated", modal=True, info=str)
208def regenerated_configure(file_paths):
209    """Check if configure has been regenerated."""
210    if 'configure.ac' in file_paths:
211        return "yes" if 'configure' in file_paths else "no"
212    else:
213        return "not needed"
214
215@status("pyconfig.h.in regenerated", modal=True, info=str)
216def regenerated_pyconfig_h_in(file_paths):
217    """Check if pyconfig.h.in has been regenerated."""
218    if 'configure.ac' in file_paths:
219        return "yes" if 'pyconfig.h.in' in file_paths else "no"
220    else:
221        return "not needed"
222
223def travis(pull_request):
224    if pull_request == 'false':
225        print('Not a pull request; skipping')
226        return
227    base_branch = get_base_branch()
228    file_paths = changed_files(base_branch)
229    python_files = [fn for fn in file_paths if fn.endswith('.py')]
230    c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))]
231    doc_files = [fn for fn in file_paths if fn.startswith('Doc') and
232                 fn.endswith(('.rst', '.inc'))]
233    fixed = []
234    fixed.extend(normalize_whitespace(python_files))
235    fixed.extend(normalize_c_whitespace(c_files))
236    fixed.extend(normalize_docs_whitespace(doc_files))
237    if not fixed:
238        print('No whitespace issues found')
239    else:
240        print(f'Please fix the {len(fixed)} file(s) with whitespace issues')
241        print('(on UNIX you can run `make patchcheck` to make the fixes)')
242        sys.exit(1)
243
244def main():
245    base_branch = get_base_branch()
246    file_paths = changed_files(base_branch)
247    python_files = [fn for fn in file_paths if fn.endswith('.py')]
248    c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))]
249    doc_files = [fn for fn in file_paths if fn.startswith('Doc') and
250                 fn.endswith(('.rst', '.inc'))]
251    misc_files = {p for p in file_paths if p.startswith('Misc')}
252    # PEP 8 whitespace rules enforcement.
253    normalize_whitespace(python_files)
254    # C rules enforcement.
255    normalize_c_whitespace(c_files)
256    # Doc whitespace enforcement.
257    normalize_docs_whitespace(doc_files)
258    # Docs updated.
259    docs_modified(doc_files)
260    # Misc/ACKS changed.
261    credit_given(misc_files)
262    # Misc/NEWS changed.
263    reported_news(misc_files)
264    # Regenerated configure, if necessary.
265    regenerated_configure(file_paths)
266    # Regenerated pyconfig.h.in, if necessary.
267    regenerated_pyconfig_h_in(file_paths)
268
269    # Test suite run and passed.
270    if python_files or c_files:
271        end = " and check for refleaks?" if c_files else "?"
272        print()
273        print("Did you run the test suite" + end)
274
275
276if __name__ == '__main__':
277    import argparse
278    parser = argparse.ArgumentParser(description=__doc__)
279    parser.add_argument('--travis',
280                        help='Perform pass/fail checks')
281    args = parser.parse_args()
282    if args.travis:
283        travis(args.travis)
284    else:
285        main()
286