1# coding: utf-8 2from __future__ import absolute_import, division, print_function, unicode_literals 3 4import os 5import shlex 6import sys 7import tempfile 8import warnings 9 10from click import Command 11from click.utils import safecall 12from pip._internal.commands import create_command 13from pip._internal.req.constructors import install_req_from_line 14from pip._internal.utils.misc import redact_auth_from_url 15 16from .. import click 17from .._compat import parse_requirements 18from ..cache import DependencyCache 19from ..exceptions import PipToolsError 20from ..locations import CACHE_DIR 21from ..logging import log 22from ..repositories import LocalRequirementsRepository, PyPIRepository 23from ..resolver import Resolver 24from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_ireq 25from ..writer import OutputWriter 26 27DEFAULT_REQUIREMENTS_FILE = "requirements.in" 28DEFAULT_REQUIREMENTS_OUTPUT_FILE = "requirements.txt" 29 30 31def _get_default_option(option_name): 32 """ 33 Get default value of the pip's option (including option from pip.conf) 34 by a given option name. 35 """ 36 install_command = create_command("install") 37 default_values = install_command.parser.get_default_values() 38 return getattr(default_values, option_name) 39 40 41class BaseCommand(Command): 42 _os_args = None 43 44 def parse_args(self, ctx, args): 45 """ 46 Override base `parse_args` to store the argument part of `sys.argv`. 47 """ 48 self._os_args = set(args) 49 return super(BaseCommand, self).parse_args(ctx, args) 50 51 def has_arg(self, arg_name): 52 """ 53 Detect whether a given arg name (including negative counterparts 54 to the arg, e.g. --no-arg) is present in the argument part of `sys.argv`. 55 """ 56 command_options = {option.name: option for option in self.params} 57 option = command_options[arg_name] 58 args = set(option.opts + option.secondary_opts) 59 return bool(self._os_args & args) 60 61 62@click.command( 63 cls=BaseCommand, context_settings={"help_option_names": ("-h", "--help")} 64) 65@click.version_option() 66@click.pass_context 67@click.option("-v", "--verbose", count=True, help="Show more output") 68@click.option("-q", "--quiet", count=True, help="Give less output") 69@click.option( 70 "-n", 71 "--dry-run", 72 is_flag=True, 73 help="Only show what would happen, don't change anything", 74) 75@click.option( 76 "-p", 77 "--pre", 78 is_flag=True, 79 default=None, 80 help="Allow resolving to prereleases (default is not)", 81) 82@click.option( 83 "-r", 84 "--rebuild", 85 is_flag=True, 86 help="Clear any caches upfront, rebuild from scratch", 87) 88@click.option( 89 "-f", 90 "--find-links", 91 multiple=True, 92 help="Look for archives in this directory or on this HTML page", 93 envvar="PIP_FIND_LINKS", 94) 95@click.option( 96 "-i", 97 "--index-url", 98 help="Change index URL (defaults to {index_url})".format( 99 index_url=redact_auth_from_url(_get_default_option("index_url")) 100 ), 101 envvar="PIP_INDEX_URL", 102) 103@click.option( 104 "--extra-index-url", 105 multiple=True, 106 help="Add additional index URL to search", 107 envvar="PIP_EXTRA_INDEX_URL", 108) 109@click.option("--cert", help="Path to alternate CA bundle.") 110@click.option( 111 "--client-cert", 112 help="Path to SSL client certificate, a single file containing " 113 "the private key and the certificate in PEM format.", 114) 115@click.option( 116 "--trusted-host", 117 multiple=True, 118 envvar="PIP_TRUSTED_HOST", 119 help="Mark this host as trusted, even though it does not have " 120 "valid or any HTTPS.", 121) 122@click.option( 123 "--header/--no-header", 124 is_flag=True, 125 default=True, 126 help="Add header to generated file", 127) 128@click.option( 129 "--index/--no-index", 130 is_flag=True, 131 default=True, 132 help="DEPRECATED: Add index URL to generated file", 133) 134@click.option( 135 "--emit-trusted-host/--no-emit-trusted-host", 136 is_flag=True, 137 default=True, 138 help="Add trusted host option to generated file", 139) 140@click.option( 141 "--annotate/--no-annotate", 142 is_flag=True, 143 default=True, 144 help="Annotate results, indicating where dependencies come from", 145) 146@click.option( 147 "-U", 148 "--upgrade", 149 is_flag=True, 150 default=False, 151 help="Try to upgrade all dependencies to their latest versions", 152) 153@click.option( 154 "-P", 155 "--upgrade-package", 156 "upgrade_packages", 157 nargs=1, 158 multiple=True, 159 help="Specify particular packages to upgrade.", 160) 161@click.option( 162 "-o", 163 "--output-file", 164 nargs=1, 165 default=None, 166 type=click.File("w+b", atomic=True, lazy=True), 167 help=( 168 "Output file name. Required if more than one input file is given. " 169 "Will be derived from input file otherwise." 170 ), 171) 172@click.option( 173 "--allow-unsafe", 174 is_flag=True, 175 default=False, 176 help="Pin packages considered unsafe: {}".format( 177 ", ".join(sorted(UNSAFE_PACKAGES)) 178 ), 179) 180@click.option( 181 "--generate-hashes", 182 is_flag=True, 183 default=False, 184 help="Generate pip 8 style hashes in the resulting requirements file.", 185) 186@click.option( 187 "--reuse-hashes/--no-reuse-hashes", 188 is_flag=True, 189 default=True, 190 help=( 191 "Improve the speed of --generate-hashes by reusing the hashes from an " 192 "existing output file." 193 ), 194) 195@click.option( 196 "--max-rounds", 197 default=10, 198 help="Maximum number of rounds before resolving the requirements aborts.", 199) 200@click.argument("src_files", nargs=-1, type=click.Path(exists=True, allow_dash=True)) 201@click.option( 202 "--build-isolation/--no-build-isolation", 203 is_flag=True, 204 default=True, 205 help="Enable isolation when building a modern source distribution. " 206 "Build dependencies specified by PEP 518 must be already installed " 207 "if build isolation is disabled.", 208) 209@click.option( 210 "--emit-find-links/--no-emit-find-links", 211 is_flag=True, 212 default=True, 213 help="Add the find-links option to generated file", 214) 215@click.option( 216 "--cache-dir", 217 help="Store the cache data in DIRECTORY.", 218 default=CACHE_DIR, 219 envvar="PIP_TOOLS_CACHE_DIR", 220 show_default=True, 221 show_envvar=True, 222 type=click.Path(file_okay=False, writable=True), 223) 224@click.option("--pip-args", help="Arguments to pass directly to the pip command.") 225@click.option( 226 "--emit-index-url/--no-emit-index-url", 227 is_flag=True, 228 default=True, 229 help="Add index URL to generated file", 230) 231def cli( 232 ctx, 233 verbose, 234 quiet, 235 dry_run, 236 pre, 237 rebuild, 238 find_links, 239 index_url, 240 extra_index_url, 241 cert, 242 client_cert, 243 trusted_host, 244 header, 245 index, 246 emit_trusted_host, 247 annotate, 248 upgrade, 249 upgrade_packages, 250 output_file, 251 allow_unsafe, 252 generate_hashes, 253 reuse_hashes, 254 src_files, 255 max_rounds, 256 build_isolation, 257 emit_find_links, 258 cache_dir, 259 pip_args, 260 emit_index_url, 261): 262 """Compiles requirements.txt from requirements.in specs.""" 263 log.verbosity = verbose - quiet 264 265 if len(src_files) == 0: 266 if os.path.exists(DEFAULT_REQUIREMENTS_FILE): 267 src_files = (DEFAULT_REQUIREMENTS_FILE,) 268 elif os.path.exists("setup.py"): 269 src_files = ("setup.py",) 270 else: 271 raise click.BadParameter( 272 ( 273 "If you do not specify an input file, " 274 "the default is {} or setup.py" 275 ).format(DEFAULT_REQUIREMENTS_FILE) 276 ) 277 278 if not output_file: 279 # An output file must be provided for stdin 280 if src_files == ("-",): 281 raise click.BadParameter("--output-file is required if input is from stdin") 282 # Use default requirements output file if there is a setup.py the source file 283 elif src_files == ("setup.py",): 284 file_name = DEFAULT_REQUIREMENTS_OUTPUT_FILE 285 # An output file must be provided if there are multiple source files 286 elif len(src_files) > 1: 287 raise click.BadParameter( 288 "--output-file is required if two or more input files are given." 289 ) 290 # Otherwise derive the output file from the source file 291 else: 292 base_name = src_files[0].rsplit(".", 1)[0] 293 file_name = base_name + ".txt" 294 295 output_file = click.open_file(file_name, "w+b", atomic=True, lazy=True) 296 297 # Close the file at the end of the context execution 298 ctx.call_on_close(safecall(output_file.close_intelligently)) 299 300 if cli.has_arg("index") and cli.has_arg("emit_index_url"): 301 raise click.BadParameter( 302 "--index/--no-index and --emit-index-url/--no-emit-index-url " 303 "are mutually exclusive." 304 ) 305 elif cli.has_arg("index"): 306 warnings.warn( 307 "--index and --no-index are deprecated and will be removed " 308 "in future versions. Use --emit-index-url/--no-emit-index-url instead.", 309 category=FutureWarning, 310 ) 311 emit_index_url = index 312 313 ### 314 # Setup 315 ### 316 317 right_args = shlex.split(pip_args or "") 318 pip_args = [] 319 if find_links: 320 for link in find_links: 321 pip_args.extend(["-f", link]) 322 if index_url: 323 pip_args.extend(["-i", index_url]) 324 if extra_index_url: 325 for extra_index in extra_index_url: 326 pip_args.extend(["--extra-index-url", extra_index]) 327 if cert: 328 pip_args.extend(["--cert", cert]) 329 if client_cert: 330 pip_args.extend(["--client-cert", client_cert]) 331 if pre: 332 pip_args.extend(["--pre"]) 333 if trusted_host: 334 for host in trusted_host: 335 pip_args.extend(["--trusted-host", host]) 336 337 if not build_isolation: 338 pip_args.append("--no-build-isolation") 339 pip_args.extend(right_args) 340 341 repository = PyPIRepository(pip_args, cache_dir=cache_dir) 342 343 # Parse all constraints coming from --upgrade-package/-P 344 upgrade_reqs_gen = (install_req_from_line(pkg) for pkg in upgrade_packages) 345 upgrade_install_reqs = { 346 key_from_ireq(install_req): install_req for install_req in upgrade_reqs_gen 347 } 348 349 existing_pins_to_upgrade = set() 350 351 # Proxy with a LocalRequirementsRepository if --upgrade is not specified 352 # (= default invocation) 353 if not upgrade and os.path.exists(output_file.name): 354 # Use a temporary repository to ensure outdated(removed) options from 355 # existing requirements.txt wouldn't get into the current repository. 356 tmp_repository = PyPIRepository(pip_args, cache_dir=cache_dir) 357 ireqs = parse_requirements( 358 output_file.name, 359 finder=tmp_repository.finder, 360 session=tmp_repository.session, 361 options=tmp_repository.options, 362 ) 363 364 # Exclude packages from --upgrade-package/-P from the existing 365 # constraints, and separately gather pins to be upgraded 366 existing_pins = {} 367 for ireq in filter(is_pinned_requirement, ireqs): 368 key = key_from_ireq(ireq) 369 if key in upgrade_install_reqs: 370 existing_pins_to_upgrade.add(key) 371 else: 372 existing_pins[key] = ireq 373 repository = LocalRequirementsRepository( 374 existing_pins, repository, reuse_hashes=reuse_hashes 375 ) 376 377 ### 378 # Parsing/collecting initial requirements 379 ### 380 381 constraints = [] 382 for src_file in src_files: 383 is_setup_file = os.path.basename(src_file) == "setup.py" 384 if is_setup_file or src_file == "-": 385 # pip requires filenames and not files. Since we want to support 386 # piping from stdin, we need to briefly save the input from stdin 387 # to a temporary file and have pip read that. also used for 388 # reading requirements from install_requires in setup.py. 389 tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False) 390 if is_setup_file: 391 from distutils.core import run_setup 392 393 dist = run_setup(src_file) 394 tmpfile.write("\n".join(dist.install_requires)) 395 comes_from = "{name} ({filename})".format( 396 name=dist.get_name(), filename=src_file 397 ) 398 else: 399 tmpfile.write(sys.stdin.read()) 400 comes_from = "-r -" 401 tmpfile.flush() 402 reqs = list( 403 parse_requirements( 404 tmpfile.name, 405 finder=repository.finder, 406 session=repository.session, 407 options=repository.options, 408 ) 409 ) 410 for req in reqs: 411 req.comes_from = comes_from 412 constraints.extend(reqs) 413 else: 414 constraints.extend( 415 parse_requirements( 416 src_file, 417 finder=repository.finder, 418 session=repository.session, 419 options=repository.options, 420 ) 421 ) 422 423 primary_packages = { 424 key_from_ireq(ireq) for ireq in constraints if not ireq.constraint 425 } 426 427 allowed_upgrades = primary_packages | existing_pins_to_upgrade 428 constraints.extend( 429 ireq for key, ireq in upgrade_install_reqs.items() if key in allowed_upgrades 430 ) 431 432 # Filter out pip environment markers which do not match (PEP496) 433 constraints = [ 434 req for req in constraints if req.markers is None or req.markers.evaluate() 435 ] 436 437 log.debug("Using indexes:") 438 with log.indentation(): 439 for index_url in dedup(repository.finder.index_urls): 440 log.debug(redact_auth_from_url(index_url)) 441 442 if repository.finder.find_links: 443 log.debug("") 444 log.debug("Using links:") 445 with log.indentation(): 446 for find_link in dedup(repository.finder.find_links): 447 log.debug(redact_auth_from_url(find_link)) 448 449 try: 450 resolver = Resolver( 451 constraints, 452 repository, 453 prereleases=repository.finder.allow_all_prereleases or pre, 454 cache=DependencyCache(cache_dir), 455 clear_caches=rebuild, 456 allow_unsafe=allow_unsafe, 457 ) 458 results = resolver.resolve(max_rounds=max_rounds) 459 if generate_hashes: 460 hashes = resolver.resolve_hashes(results) 461 else: 462 hashes = None 463 except PipToolsError as e: 464 log.error(str(e)) 465 sys.exit(2) 466 467 log.debug("") 468 469 ## 470 # Output 471 ## 472 473 writer = OutputWriter( 474 src_files, 475 output_file, 476 click_ctx=ctx, 477 dry_run=dry_run, 478 emit_header=header, 479 emit_index_url=emit_index_url, 480 emit_trusted_host=emit_trusted_host, 481 annotate=annotate, 482 generate_hashes=generate_hashes, 483 default_index_url=repository.DEFAULT_INDEX_URL, 484 index_urls=repository.finder.index_urls, 485 trusted_hosts=repository.finder.trusted_hosts, 486 format_control=repository.finder.format_control, 487 allow_unsafe=allow_unsafe, 488 find_links=repository.finder.find_links, 489 emit_find_links=emit_find_links, 490 ) 491 writer.write( 492 results=results, 493 unsafe_requirements=resolver.unsafe_constraints, 494 markers={ 495 key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers 496 }, 497 hashes=hashes, 498 ) 499 500 if dry_run: 501 log.info("Dry-run, so nothing updated.") 502