1"""Automation using nox. 2""" 3 4# The following comment should be removed at some point in the future. 5# mypy: disallow-untyped-defs=False 6 7import glob 8import os 9import shutil 10import sys 11from pathlib import Path 12 13import nox 14 15sys.path.append(".") 16from tools.automation import release # isort:skip # noqa 17sys.path.pop() 18 19nox.options.reuse_existing_virtualenvs = True 20nox.options.sessions = ["lint"] 21 22LOCATIONS = { 23 "common-wheels": "tests/data/common_wheels", 24 "protected-pip": "tools/tox_pip.py", 25} 26REQUIREMENTS = { 27 "docs": "tools/requirements/docs.txt", 28 "tests": "tools/requirements/tests.txt", 29 "common-wheels": "tools/requirements/tests-common_wheels.txt", 30} 31 32AUTHORS_FILE = "AUTHORS.txt" 33VERSION_FILE = "src/pip/__init__.py" 34 35 36def run_with_protected_pip(session, *arguments): 37 """Do a session.run("pip", *arguments), using a "protected" pip. 38 39 This invokes a wrapper script, that forwards calls to original virtualenv 40 (stable) version, and not the code being tested. This ensures pip being 41 used is not the code being tested. 42 """ 43 env = {"VIRTUAL_ENV": session.virtualenv.location} 44 45 command = ("python", LOCATIONS["protected-pip"]) + arguments 46 kwargs = {"env": env, "silent": True} 47 session.run(*command, **kwargs) 48 49 50def should_update_common_wheels(): 51 # If the cache hasn't been created, create it. 52 if not os.path.exists(LOCATIONS["common-wheels"]): 53 return True 54 55 # If the requirements was updated after cache, we'll repopulate it. 56 cache_last_populated_at = os.path.getmtime(LOCATIONS["common-wheels"]) 57 requirements_updated_at = os.path.getmtime(REQUIREMENTS["common-wheels"]) 58 need_to_repopulate = requirements_updated_at > cache_last_populated_at 59 60 # Clear the stale cache. 61 if need_to_repopulate: 62 shutil.rmtree(LOCATIONS["common-wheels"], ignore_errors=True) 63 64 return need_to_repopulate 65 66 67# ----------------------------------------------------------------------------- 68# Development Commands 69# These are currently prototypes to evaluate whether we want to switch over 70# completely to nox for all our automation. Contributors should prefer using 71# `tox -e ...` until this note is removed. 72# ----------------------------------------------------------------------------- 73@nox.session(python=["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "pypy", "pypy3"]) 74def test(session): 75 # Get the common wheels. 76 if should_update_common_wheels(): 77 run_with_protected_pip( 78 session, 79 "wheel", 80 "-w", LOCATIONS["common-wheels"], 81 "-r", REQUIREMENTS["common-wheels"], 82 ) 83 else: 84 msg = ( 85 "Re-using existing common-wheels at {}." 86 .format(LOCATIONS["common-wheels"]) 87 ) 88 session.log(msg) 89 90 # Build source distribution 91 sdist_dir = os.path.join(session.virtualenv.location, "sdist") 92 if os.path.exists(sdist_dir): 93 shutil.rmtree(sdist_dir, ignore_errors=True) 94 session.run( 95 "python", "setup.py", "sdist", 96 "--formats=zip", "--dist-dir", sdist_dir, 97 silent=True, 98 ) 99 generated_files = os.listdir(sdist_dir) 100 assert len(generated_files) == 1 101 generated_sdist = os.path.join(sdist_dir, generated_files[0]) 102 103 # Install source distribution 104 run_with_protected_pip(session, "install", generated_sdist) 105 106 # Install test dependencies 107 run_with_protected_pip(session, "install", "-r", REQUIREMENTS["tests"]) 108 109 # Parallelize tests as much as possible, by default. 110 arguments = session.posargs or ["-n", "auto"] 111 112 # Run the tests 113 # LC_CTYPE is set to get UTF-8 output inside of the subprocesses that our 114 # tests use. 115 session.run("pytest", *arguments, env={"LC_CTYPE": "en_US.UTF-8"}) 116 117 118@nox.session 119def docs(session): 120 session.install("-e", ".") 121 session.install("-r", REQUIREMENTS["docs"]) 122 123 def get_sphinx_build_command(kind): 124 # Having the conf.py in the docs/html is weird but needed because we 125 # can not use a different configuration directory vs source directory 126 # on RTD currently. So, we'll pass "-c docs/html" here. 127 # See https://github.com/rtfd/readthedocs.org/issues/1543. 128 return [ 129 "sphinx-build", 130 "-W", 131 "-c", "docs/html", # see note above 132 "-d", "docs/build/doctrees/" + kind, 133 "-b", kind, 134 "docs/" + kind, 135 "docs/build/" + kind, 136 ] 137 138 session.run(*get_sphinx_build_command("html")) 139 session.run(*get_sphinx_build_command("man")) 140 141 142@nox.session 143def lint(session): 144 session.install("pre-commit") 145 146 if session.posargs: 147 args = session.posargs + ["--all-files"] 148 else: 149 args = ["--all-files", "--show-diff-on-failure"] 150 151 session.run("pre-commit", "run", *args) 152 153 154@nox.session 155def vendoring(session): 156 session.install("vendoring>=0.3.0") 157 158 if "--upgrade" not in session.posargs: 159 session.run("vendoring", "sync", ".", "-v") 160 return 161 162 def pinned_requirements(path): 163 for line in path.read_text().splitlines(): 164 one, two = line.split("==", 1) 165 name = one.strip() 166 version = two.split("#")[0].strip() 167 yield name, version 168 169 vendor_txt = Path("src/pip/_vendor/vendor.txt") 170 for name, old_version in pinned_requirements(vendor_txt): 171 if name == "setuptools": 172 continue 173 174 # update requirements.txt 175 session.run("vendoring", "update", ".", name) 176 177 # get the updated version 178 new_version = old_version 179 for inner_name, inner_version in pinned_requirements(vendor_txt): 180 if inner_name == name: 181 # this is a dedicated assignment, to make flake8 happy 182 new_version = inner_version 183 break 184 else: 185 session.error(f"Could not find {name} in {vendor_txt}") 186 187 # check if the version changed. 188 if new_version == old_version: 189 continue # no change, nothing more to do here. 190 191 # synchronize the contents 192 session.run("vendoring", "sync", ".") 193 194 # Determine the correct message 195 message = f"Upgrade {name} to {new_version}" 196 197 # Write our news fragment 198 news_file = Path("news") / (name + ".vendor.rst") 199 news_file.write_text(message + "\n") # "\n" appeases end-of-line-fixer 200 201 # Commit the changes 202 release.commit_file(session, ".", message=message) 203 204 205# ----------------------------------------------------------------------------- 206# Release Commands 207# ----------------------------------------------------------------------------- 208@nox.session(name="prepare-release") 209def prepare_release(session): 210 version = release.get_version_from_arguments(session) 211 if not version: 212 session.error("Usage: nox -s prepare-release -- <version>") 213 214 session.log("# Ensure nothing is staged") 215 if release.modified_files_in_git("--staged"): 216 session.error("There are files staged in git") 217 218 session.log(f"# Updating {AUTHORS_FILE}") 219 release.generate_authors(AUTHORS_FILE) 220 if release.modified_files_in_git(): 221 release.commit_file( 222 session, AUTHORS_FILE, message=f"Update {AUTHORS_FILE}", 223 ) 224 else: 225 session.log(f"# No changes to {AUTHORS_FILE}") 226 227 session.log("# Generating NEWS") 228 release.generate_news(session, version) 229 230 session.log(f"# Bumping for release {version}") 231 release.update_version_file(version, VERSION_FILE) 232 release.commit_file(session, VERSION_FILE, message="Bump for release") 233 234 session.log("# Tagging release") 235 release.create_git_tag(session, version, message=f"Release {version}") 236 237 session.log("# Bumping for development") 238 next_dev_version = release.get_next_development_version(version) 239 release.update_version_file(next_dev_version, VERSION_FILE) 240 release.commit_file(session, VERSION_FILE, message="Bump for development") 241 242 243@nox.session(name="build-release") 244def build_release(session): 245 version = release.get_version_from_arguments(session) 246 if not version: 247 session.error("Usage: nox -s build-release -- YY.N[.P]") 248 249 session.log("# Ensure no files in dist/") 250 if release.have_files_in_folder("dist"): 251 session.error( 252 "There are files in dist/. Remove them and try again. " 253 "You can use `git clean -fxdi -- dist` command to do this" 254 ) 255 256 session.log("# Install dependencies") 257 session.install("setuptools", "wheel", "twine") 258 259 with release.isolated_temporary_checkout(session, version) as build_dir: 260 session.log( 261 "# Start the build in an isolated, " 262 f"temporary Git checkout at {build_dir!s}", 263 ) 264 with release.workdir(session, build_dir): 265 tmp_dists = build_dists(session) 266 267 tmp_dist_paths = (build_dir / p for p in tmp_dists) 268 session.log(f"# Copying dists from {build_dir}") 269 os.makedirs('dist', exist_ok=True) 270 for dist, final in zip(tmp_dist_paths, tmp_dists): 271 session.log(f"# Copying {dist} to {final}") 272 shutil.copy(dist, final) 273 274 275def build_dists(session): 276 """Return dists with valid metadata.""" 277 session.log( 278 "# Check if there's any Git-untracked files before building the wheel", 279 ) 280 281 has_forbidden_git_untracked_files = any( 282 # Don't report the environment this session is running in 283 not untracked_file.startswith('.nox/build-release/') 284 for untracked_file in release.get_git_untracked_files() 285 ) 286 if has_forbidden_git_untracked_files: 287 session.error( 288 "There are untracked files in the working directory. " 289 "Remove them and try again", 290 ) 291 292 session.log("# Build distributions") 293 session.run("python", "setup.py", "sdist", "bdist_wheel", silent=True) 294 produced_dists = glob.glob("dist/*") 295 296 session.log(f"# Verify distributions: {', '.join(produced_dists)}") 297 session.run("twine", "check", *produced_dists, silent=True) 298 299 return produced_dists 300 301 302@nox.session(name="upload-release") 303def upload_release(session): 304 version = release.get_version_from_arguments(session) 305 if not version: 306 session.error("Usage: nox -s upload-release -- YY.N[.P]") 307 308 session.log("# Install dependencies") 309 session.install("twine") 310 311 distribution_files = glob.glob("dist/*") 312 session.log(f"# Distribution files: {distribution_files}") 313 314 # Sanity check: Make sure there's 2 distribution files. 315 count = len(distribution_files) 316 if count != 2: 317 session.error( 318 f"Expected 2 distribution files for upload, got {count}. " 319 f"Remove dist/ and run 'nox -s build-release -- {version}'" 320 ) 321 # Sanity check: Make sure the files are correctly named. 322 distfile_names = map(os.path.basename, distribution_files) 323 expected_distribution_files = [ 324 f"pip-{version}-py2.py3-none-any.whl", 325 f"pip-{version}.tar.gz", 326 ] 327 if sorted(distfile_names) != sorted(expected_distribution_files): 328 session.error( 329 f"Distribution files do not seem to be for {version} release." 330 ) 331 332 session.log("# Upload distributions") 333 session.run("twine", "upload", *distribution_files) 334