1"""Automation using nox.
2"""
3
4# The following comment should be removed at some point in the future.
5# mypy: disallow-untyped-defs=False
6
7import glob
8import os
9import shutil
10import sys
11from pathlib import Path
12
13import nox
14
15sys.path.append(".")
16from tools.automation import release  # isort:skip  # noqa
17sys.path.pop()
18
19nox.options.reuse_existing_virtualenvs = True
20nox.options.sessions = ["lint"]
21
22LOCATIONS = {
23    "common-wheels": "tests/data/common_wheels",
24    "protected-pip": "tools/tox_pip.py",
25}
26REQUIREMENTS = {
27    "docs": "tools/requirements/docs.txt",
28    "tests": "tools/requirements/tests.txt",
29    "common-wheels": "tools/requirements/tests-common_wheels.txt",
30}
31
32AUTHORS_FILE = "AUTHORS.txt"
33VERSION_FILE = "src/pip/__init__.py"
34
35
36def run_with_protected_pip(session, *arguments):
37    """Do a session.run("pip", *arguments), using a "protected" pip.
38
39    This invokes a wrapper script, that forwards calls to original virtualenv
40    (stable) version, and not the code being tested. This ensures pip being
41    used is not the code being tested.
42    """
43    env = {"VIRTUAL_ENV": session.virtualenv.location}
44
45    command = ("python", LOCATIONS["protected-pip"]) + arguments
46    kwargs = {"env": env, "silent": True}
47    session.run(*command, **kwargs)
48
49
50def should_update_common_wheels():
51    # If the cache hasn't been created, create it.
52    if not os.path.exists(LOCATIONS["common-wheels"]):
53        return True
54
55    # If the requirements was updated after cache, we'll repopulate it.
56    cache_last_populated_at = os.path.getmtime(LOCATIONS["common-wheels"])
57    requirements_updated_at = os.path.getmtime(REQUIREMENTS["common-wheels"])
58    need_to_repopulate = requirements_updated_at > cache_last_populated_at
59
60    # Clear the stale cache.
61    if need_to_repopulate:
62        shutil.rmtree(LOCATIONS["common-wheels"], ignore_errors=True)
63
64    return need_to_repopulate
65
66
67# -----------------------------------------------------------------------------
68# Development Commands
69#   These are currently prototypes to evaluate whether we want to switch over
70#   completely to nox for all our automation. Contributors should prefer using
71#   `tox -e ...` until this note is removed.
72# -----------------------------------------------------------------------------
73@nox.session(python=["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "pypy", "pypy3"])
74def test(session):
75    # Get the common wheels.
76    if should_update_common_wheels():
77        run_with_protected_pip(
78            session,
79            "wheel",
80            "-w", LOCATIONS["common-wheels"],
81            "-r", REQUIREMENTS["common-wheels"],
82        )
83    else:
84        msg = (
85            "Re-using existing common-wheels at {}."
86            .format(LOCATIONS["common-wheels"])
87        )
88        session.log(msg)
89
90    # Build source distribution
91    sdist_dir = os.path.join(session.virtualenv.location, "sdist")
92    if os.path.exists(sdist_dir):
93        shutil.rmtree(sdist_dir, ignore_errors=True)
94    session.run(
95        "python", "setup.py", "sdist",
96        "--formats=zip", "--dist-dir", sdist_dir,
97        silent=True,
98    )
99    generated_files = os.listdir(sdist_dir)
100    assert len(generated_files) == 1
101    generated_sdist = os.path.join(sdist_dir, generated_files[0])
102
103    # Install source distribution
104    run_with_protected_pip(session, "install", generated_sdist)
105
106    # Install test dependencies
107    run_with_protected_pip(session, "install", "-r", REQUIREMENTS["tests"])
108
109    # Parallelize tests as much as possible, by default.
110    arguments = session.posargs or ["-n", "auto"]
111
112    # Run the tests
113    #   LC_CTYPE is set to get UTF-8 output inside of the subprocesses that our
114    #   tests use.
115    session.run("pytest", *arguments, env={"LC_CTYPE": "en_US.UTF-8"})
116
117
118@nox.session
119def docs(session):
120    session.install("-e", ".")
121    session.install("-r", REQUIREMENTS["docs"])
122
123    def get_sphinx_build_command(kind):
124        # Having the conf.py in the docs/html is weird but needed because we
125        # can not use a different configuration directory vs source directory
126        # on RTD currently. So, we'll pass "-c docs/html" here.
127        # See https://github.com/rtfd/readthedocs.org/issues/1543.
128        return [
129            "sphinx-build",
130            "-W",
131            "-c", "docs/html",  # see note above
132            "-d", "docs/build/doctrees/" + kind,
133            "-b", kind,
134            "docs/" + kind,
135            "docs/build/" + kind,
136        ]
137
138    session.run(*get_sphinx_build_command("html"))
139    session.run(*get_sphinx_build_command("man"))
140
141
142@nox.session
143def lint(session):
144    session.install("pre-commit")
145
146    if session.posargs:
147        args = session.posargs + ["--all-files"]
148    else:
149        args = ["--all-files", "--show-diff-on-failure"]
150
151    session.run("pre-commit", "run", *args)
152
153
154@nox.session
155def vendoring(session):
156    session.install("vendoring>=0.3.0")
157
158    if "--upgrade" not in session.posargs:
159        session.run("vendoring", "sync", ".", "-v")
160        return
161
162    def pinned_requirements(path):
163        for line in path.read_text().splitlines():
164            one, two = line.split("==", 1)
165            name = one.strip()
166            version = two.split("#")[0].strip()
167            yield name, version
168
169    vendor_txt = Path("src/pip/_vendor/vendor.txt")
170    for name, old_version in pinned_requirements(vendor_txt):
171        if name == "setuptools":
172            continue
173
174        # update requirements.txt
175        session.run("vendoring", "update", ".", name)
176
177        # get the updated version
178        new_version = old_version
179        for inner_name, inner_version in pinned_requirements(vendor_txt):
180            if inner_name == name:
181                # this is a dedicated assignment, to make flake8 happy
182                new_version = inner_version
183                break
184        else:
185            session.error(f"Could not find {name} in {vendor_txt}")
186
187        # check if the version changed.
188        if new_version == old_version:
189            continue  # no change, nothing more to do here.
190
191        # synchronize the contents
192        session.run("vendoring", "sync", ".")
193
194        # Determine the correct message
195        message = f"Upgrade {name} to {new_version}"
196
197        # Write our news fragment
198        news_file = Path("news") / (name + ".vendor.rst")
199        news_file.write_text(message + "\n")  # "\n" appeases end-of-line-fixer
200
201        # Commit the changes
202        release.commit_file(session, ".", message=message)
203
204
205# -----------------------------------------------------------------------------
206# Release Commands
207# -----------------------------------------------------------------------------
208@nox.session(name="prepare-release")
209def prepare_release(session):
210    version = release.get_version_from_arguments(session)
211    if not version:
212        session.error("Usage: nox -s prepare-release -- <version>")
213
214    session.log("# Ensure nothing is staged")
215    if release.modified_files_in_git("--staged"):
216        session.error("There are files staged in git")
217
218    session.log(f"# Updating {AUTHORS_FILE}")
219    release.generate_authors(AUTHORS_FILE)
220    if release.modified_files_in_git():
221        release.commit_file(
222            session, AUTHORS_FILE, message=f"Update {AUTHORS_FILE}",
223        )
224    else:
225        session.log(f"# No changes to {AUTHORS_FILE}")
226
227    session.log("# Generating NEWS")
228    release.generate_news(session, version)
229
230    session.log(f"# Bumping for release {version}")
231    release.update_version_file(version, VERSION_FILE)
232    release.commit_file(session, VERSION_FILE, message="Bump for release")
233
234    session.log("# Tagging release")
235    release.create_git_tag(session, version, message=f"Release {version}")
236
237    session.log("# Bumping for development")
238    next_dev_version = release.get_next_development_version(version)
239    release.update_version_file(next_dev_version, VERSION_FILE)
240    release.commit_file(session, VERSION_FILE, message="Bump for development")
241
242
243@nox.session(name="build-release")
244def build_release(session):
245    version = release.get_version_from_arguments(session)
246    if not version:
247        session.error("Usage: nox -s build-release -- YY.N[.P]")
248
249    session.log("# Ensure no files in dist/")
250    if release.have_files_in_folder("dist"):
251        session.error(
252            "There are files in dist/. Remove them and try again. "
253            "You can use `git clean -fxdi -- dist` command to do this"
254        )
255
256    session.log("# Install dependencies")
257    session.install("setuptools", "wheel", "twine")
258
259    with release.isolated_temporary_checkout(session, version) as build_dir:
260        session.log(
261            "# Start the build in an isolated, "
262            f"temporary Git checkout at {build_dir!s}",
263        )
264        with release.workdir(session, build_dir):
265            tmp_dists = build_dists(session)
266
267        tmp_dist_paths = (build_dir / p for p in tmp_dists)
268        session.log(f"# Copying dists from {build_dir}")
269        os.makedirs('dist', exist_ok=True)
270        for dist, final in zip(tmp_dist_paths, tmp_dists):
271            session.log(f"# Copying {dist} to {final}")
272            shutil.copy(dist, final)
273
274
275def build_dists(session):
276    """Return dists with valid metadata."""
277    session.log(
278        "# Check if there's any Git-untracked files before building the wheel",
279    )
280
281    has_forbidden_git_untracked_files = any(
282        # Don't report the environment this session is running in
283        not untracked_file.startswith('.nox/build-release/')
284        for untracked_file in release.get_git_untracked_files()
285    )
286    if has_forbidden_git_untracked_files:
287        session.error(
288            "There are untracked files in the working directory. "
289            "Remove them and try again",
290        )
291
292    session.log("# Build distributions")
293    session.run("python", "setup.py", "sdist", "bdist_wheel", silent=True)
294    produced_dists = glob.glob("dist/*")
295
296    session.log(f"# Verify distributions: {', '.join(produced_dists)}")
297    session.run("twine", "check", *produced_dists, silent=True)
298
299    return produced_dists
300
301
302@nox.session(name="upload-release")
303def upload_release(session):
304    version = release.get_version_from_arguments(session)
305    if not version:
306        session.error("Usage: nox -s upload-release -- YY.N[.P]")
307
308    session.log("# Install dependencies")
309    session.install("twine")
310
311    distribution_files = glob.glob("dist/*")
312    session.log(f"# Distribution files: {distribution_files}")
313
314    # Sanity check: Make sure there's 2 distribution files.
315    count = len(distribution_files)
316    if count != 2:
317        session.error(
318            f"Expected 2 distribution files for upload, got {count}. "
319            f"Remove dist/ and run 'nox -s build-release -- {version}'"
320        )
321    # Sanity check: Make sure the files are correctly named.
322    distfile_names = map(os.path.basename, distribution_files)
323    expected_distribution_files = [
324        f"pip-{version}-py2.py3-none-any.whl",
325        f"pip-{version}.tar.gz",
326    ]
327    if sorted(distfile_names) != sorted(expected_distribution_files):
328        session.error(
329            f"Distribution files do not seem to be for {version} release."
330        )
331
332    session.log("# Upload distributions")
333    session.run("twine", "upload", *distribution_files)
334