1# coding: utf-8 2from __future__ import absolute_import, division, print_function, unicode_literals 3 4import os 5import shlex 6import sys 7import tempfile 8 9from click.utils import safecall 10from ._compat import InstallCommand 11from ._compat import install_req_from_line 12 13from .. import click 14from .._compat import parse_requirements 15from ..cache import DependencyCache 16from ..exceptions import PipToolsError 17from ..locations import CACHE_DIR 18from ..logging import log 19from ..repositories import LocalRequirementsRepository, PyPIRepository 20from ..resolver import Resolver 21from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_ireq 22from ..writer import OutputWriter 23 24DEFAULT_REQUIREMENTS_FILE = "requirements.in" 25DEFAULT_REQUIREMENTS_OUTPUT_FILE = "requirements.txt" 26 27# Get default values of the pip's options (including options from pipenv.patched.notpip.conf). 28install_command = InstallComand() 29pip_defaults = install_command.parser.get_default_values() 30 31 32@click.command() 33@click.version_option() 34@click.pass_context 35@click.option("-v", "--verbose", count=True, help="Show more output") 36@click.option("-q", "--quiet", count=True, help="Give less output") 37@click.option( 38 "-n", 39 "--dry-run", 40 is_flag=True, 41 help="Only show what would happen, don't change anything", 42) 43@click.option( 44 "-p", 45 "--pre", 46 is_flag=True, 47 default=None, 48 help="Allow resolving to prereleases (default is not)", 49) 50@click.option( 51 "-r", 52 "--rebuild", 53 is_flag=True, 54 help="Clear any caches upfront, rebuild from scratch", 55) 56@click.option( 57 "-f", 58 "--find-links", 59 multiple=True, 60 help="Look for archives in this directory or on this HTML page", 61 envvar="PIP_FIND_LINKS", 62) 63@click.option( 64 "-i", 65 "--index-url", 66 help="Change index URL (defaults to {})".format(pip_defaults.index_url), 67 envvar="PIP_INDEX_URL", 68) 69@click.option( 70 "--extra-index-url", 71 multiple=True, 72 help="Add additional index URL to search", 73 envvar="PIP_EXTRA_INDEX_URL", 74) 75@click.option("--cert", help="Path to alternate CA bundle.") 76@click.option( 77 "--client-cert", 78 help="Path to SSL client certificate, a single file containing " 79 "the private key and the certificate in PEM format.", 80) 81@click.option( 82 "--trusted-host", 83 multiple=True, 84 envvar="PIP_TRUSTED_HOST", 85 help="Mark this host as trusted, even though it does not have " 86 "valid or any HTTPS.", 87) 88@click.option( 89 "--header/--no-header", 90 is_flag=True, 91 default=True, 92 help="Add header to generated file", 93) 94@click.option( 95 "--index/--no-index", 96 is_flag=True, 97 default=True, 98 help="Add index URL to generated file", 99) 100@click.option( 101 "--emit-trusted-host/--no-emit-trusted-host", 102 is_flag=True, 103 default=True, 104 help="Add trusted host option to generated file", 105) 106@click.option( 107 "--annotate/--no-annotate", 108 is_flag=True, 109 default=True, 110 help="Annotate results, indicating where dependencies come from", 111) 112@click.option( 113 "-U", 114 "--upgrade", 115 is_flag=True, 116 default=False, 117 help="Try to upgrade all dependencies to their latest versions", 118) 119@click.option( 120 "-P", 121 "--upgrade-package", 122 "upgrade_packages", 123 nargs=1, 124 multiple=True, 125 help="Specify particular packages to upgrade.", 126) 127@click.option( 128 "-o", 129 "--output-file", 130 nargs=1, 131 default=None, 132 type=click.File("w+b", atomic=True, lazy=True), 133 help=( 134 "Output file name. Required if more than one input file is given. " 135 "Will be derived from input file otherwise." 136 ), 137) 138@click.option( 139 "--allow-unsafe", 140 is_flag=True, 141 default=False, 142 help="Pin packages considered unsafe: {}".format( 143 ", ".join(sorted(UNSAFE_PACKAGES)) 144 ), 145) 146@click.option( 147 "--generate-hashes", 148 is_flag=True, 149 default=False, 150 help="Generate pip 8 style hashes in the resulting requirements file.", 151) 152@click.option( 153 "--max-rounds", 154 default=10, 155 help="Maximum number of rounds before resolving the requirements aborts.", 156) 157@click.argument("src_files", nargs=-1, type=click.Path(exists=True, allow_dash=True)) 158@click.option( 159 "--build-isolation/--no-build-isolation", 160 is_flag=True, 161 default=True, 162 help="Enable isolation when building a modern source distribution. " 163 "Build dependencies specified by PEP 518 must be already installed " 164 "if build isolation is disabled.", 165) 166@click.option( 167 "--emit-find-links/--no-emit-find-links", 168 is_flag=True, 169 default=True, 170 help="Add the find-links option to generated file", 171) 172@click.option( 173 "--cache-dir", 174 help="Store the cache data in DIRECTORY.", 175 default=CACHE_DIR, 176 envvar="PIP_TOOLS_CACHE_DIR", 177 show_default=True, 178 show_envvar=True, 179 type=click.Path(file_okay=False, writable=True), 180) 181@click.option("--pip-args", help="Arguments to pass directly to the pip command.") 182def cli( 183 ctx, 184 verbose, 185 quiet, 186 dry_run, 187 pre, 188 rebuild, 189 find_links, 190 index_url, 191 extra_index_url, 192 cert, 193 client_cert, 194 trusted_host, 195 header, 196 index, 197 emit_trusted_host, 198 annotate, 199 upgrade, 200 upgrade_packages, 201 output_file, 202 allow_unsafe, 203 generate_hashes, 204 src_files, 205 max_rounds, 206 build_isolation, 207 emit_find_links, 208 cache_dir, 209 pip_args, 210): 211 """Compiles requirements.txt from requirements.in specs.""" 212 log.verbosity = verbose - quiet 213 214 if len(src_files) == 0: 215 if os.path.exists(DEFAULT_REQUIREMENTS_FILE): 216 src_files = (DEFAULT_REQUIREMENTS_FILE,) 217 elif os.path.exists("setup.py"): 218 src_files = ("setup.py",) 219 else: 220 raise click.BadParameter( 221 ( 222 "If you do not specify an input file, " 223 "the default is {} or setup.py" 224 ).format(DEFAULT_REQUIREMENTS_FILE) 225 ) 226 227 if not output_file: 228 # An output file must be provided for stdin 229 if src_files == ("-",): 230 raise click.BadParameter("--output-file is required if input is from stdin") 231 # Use default requirements output file if there is a setup.py the source file 232 elif src_files == ("setup.py",): 233 file_name = DEFAULT_REQUIREMENTS_OUTPUT_FILE 234 # An output file must be provided if there are multiple source files 235 elif len(src_files) > 1: 236 raise click.BadParameter( 237 "--output-file is required if two or more input files are given." 238 ) 239 # Otherwise derive the output file from the source file 240 else: 241 base_name = src_files[0].rsplit(".", 1)[0] 242 file_name = base_name + ".txt" 243 244 output_file = click.open_file(file_name, "w+b", atomic=True, lazy=True) 245 246 # Close the file at the end of the context execution 247 ctx.call_on_close(safecall(output_file.close_intelligently)) 248 249 ### 250 # Setup 251 ### 252 253 right_args = shlex.split(pip_args or "") 254 pip_args = [] 255 if find_links: 256 for link in find_links: 257 pip_args.extend(["-f", link]) 258 if index_url: 259 pip_args.extend(["-i", index_url]) 260 if extra_index_url: 261 for extra_index in extra_index_url: 262 pip_args.extend(["--extra-index-url", extra_index]) 263 if cert: 264 pip_args.extend(["--cert", cert]) 265 if client_cert: 266 pip_args.extend(["--client-cert", client_cert]) 267 if pre: 268 pip_args.extend(["--pre"]) 269 if trusted_host: 270 for host in trusted_host: 271 pip_args.extend(["--trusted-host", host]) 272 273 if not build_isolation: 274 pip_args.append("--no-build-isolation") 275 pip_args.extend(right_args) 276 277 repository = PyPIRepository(pip_args, cache_dir=cache_dir) 278 279 # Parse all constraints coming from --upgrade-package/-P 280 upgrade_reqs_gen = (install_req_from_line(pkg) for pkg in upgrade_packages) 281 upgrade_install_reqs = { 282 key_from_ireq(install_req): install_req for install_req in upgrade_reqs_gen 283 } 284 285 existing_pins_to_upgrade = set() 286 287 # Proxy with a LocalRequirementsRepository if --upgrade is not specified 288 # (= default invocation) 289 if not upgrade and os.path.exists(output_file.name): 290 # Use a temporary repository to ensure outdated(removed) options from 291 # existing requirements.txt wouldn't get into the current repository. 292 tmp_repository = PyPIRepository(pip_args, cache_dir=cache_dir) 293 ireqs = parse_requirements( 294 output_file.name, 295 finder=tmp_repository.finder, 296 session=tmp_repository.session, 297 options=tmp_repository.options, 298 ) 299 300 # Exclude packages from --upgrade-package/-P from the existing 301 # constraints, and separately gather pins to be upgraded 302 existing_pins = {} 303 for ireq in filter(is_pinned_requirement, ireqs): 304 key = key_from_ireq(ireq) 305 if key in upgrade_install_reqs: 306 existing_pins_to_upgrade.add(key) 307 else: 308 existing_pins[key] = ireq 309 repository = LocalRequirementsRepository(existing_pins, repository) 310 311 ### 312 # Parsing/collecting initial requirements 313 ### 314 315 constraints = [] 316 for src_file in src_files: 317 is_setup_file = os.path.basename(src_file) == "setup.py" 318 if is_setup_file or src_file == "-": 319 # pip requires filenames and not files. Since we want to support 320 # piping from stdin, we need to briefly save the input from stdin 321 # to a temporary file and have pip read that. also used for 322 # reading requirements from install_requires in setup.py. 323 tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False) 324 if is_setup_file: 325 from distutils.core import run_setup 326 327 dist = run_setup(src_file) 328 tmpfile.write("\n".join(dist.install_requires)) 329 comes_from = "{name} ({filename})".format( 330 name=dist.get_name(), filename=src_file 331 ) 332 else: 333 tmpfile.write(sys.stdin.read()) 334 comes_from = "-r -" 335 tmpfile.flush() 336 reqs = list( 337 parse_requirements( 338 tmpfile.name, 339 finder=repository.finder, 340 session=repository.session, 341 options=repository.options, 342 ) 343 ) 344 for req in reqs: 345 req.comes_from = comes_from 346 constraints.extend(reqs) 347 else: 348 constraints.extend( 349 parse_requirements( 350 src_file, 351 finder=repository.finder, 352 session=repository.session, 353 options=repository.options, 354 ) 355 ) 356 357 primary_packages = { 358 key_from_ireq(ireq) for ireq in constraints if not ireq.constraint 359 } 360 361 allowed_upgrades = primary_packages | existing_pins_to_upgrade 362 constraints.extend( 363 ireq for key, ireq in upgrade_install_reqs.items() if key in allowed_upgrades 364 ) 365 366 # Filter out pip environment markers which do not match (PEP496) 367 constraints = [ 368 req for req in constraints if req.markers is None or req.markers.evaluate() 369 ] 370 371 log.debug("Using indexes:") 372 for index_url in dedup(repository.finder.index_urls): 373 log.debug(" {}".format(index_url)) 374 375 if repository.finder.find_links: 376 log.debug("") 377 log.debug("Configuration:") 378 for find_link in dedup(repository.finder.find_links): 379 log.debug(" -f {}".format(find_link)) 380 381 try: 382 resolver = Resolver( 383 constraints, 384 repository, 385 prereleases=repository.finder.allow_all_prereleases or pre, 386 cache=DependencyCache(cache_dir), 387 clear_caches=rebuild, 388 allow_unsafe=allow_unsafe, 389 ) 390 results = resolver.resolve(max_rounds=max_rounds) 391 if generate_hashes: 392 hashes = resolver.resolve_hashes(results) 393 else: 394 hashes = None 395 except PipToolsError as e: 396 log.error(str(e)) 397 sys.exit(2) 398 399 log.debug("") 400 401 ## 402 # Output 403 ## 404 405 writer = OutputWriter( 406 src_files, 407 output_file, 408 click_ctx=ctx, 409 dry_run=dry_run, 410 emit_header=header, 411 emit_index=index, 412 emit_trusted_host=emit_trusted_host, 413 annotate=annotate, 414 generate_hashes=generate_hashes, 415 default_index_url=repository.DEFAULT_INDEX_URL, 416 index_urls=repository.finder.index_urls, 417 trusted_hosts=repository.finder.trusted_hosts, 418 format_control=repository.finder.format_control, 419 allow_unsafe=allow_unsafe, 420 find_links=repository.finder.find_links, 421 emit_find_links=emit_find_links, 422 ) 423 writer.write( 424 results=results, 425 unsafe_requirements=resolver.unsafe_constraints, 426 markers={ 427 key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers 428 }, 429 hashes=hashes, 430 ) 431 432 if dry_run: 433 log.info("Dry-run, so nothing updated.") 434