1# coding: utf-8
2from __future__ import absolute_import, division, print_function, unicode_literals
3
4import os
5import shlex
6import sys
7import tempfile
8
9from click.utils import safecall
10from ._compat import InstallCommand
11from ._compat import install_req_from_line
12
13from .. import click
14from .._compat import parse_requirements
15from ..cache import DependencyCache
16from ..exceptions import PipToolsError
17from ..locations import CACHE_DIR
18from ..logging import log
19from ..repositories import LocalRequirementsRepository, PyPIRepository
20from ..resolver import Resolver
21from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_ireq
22from ..writer import OutputWriter
23
24DEFAULT_REQUIREMENTS_FILE = "requirements.in"
25DEFAULT_REQUIREMENTS_OUTPUT_FILE = "requirements.txt"
26
27# Get default values of the pip's options (including options from pipenv.patched.notpip.conf).
28install_command = InstallComand()
29pip_defaults = install_command.parser.get_default_values()
30
31
32@click.command()
33@click.version_option()
34@click.pass_context
35@click.option("-v", "--verbose", count=True, help="Show more output")
36@click.option("-q", "--quiet", count=True, help="Give less output")
37@click.option(
38    "-n",
39    "--dry-run",
40    is_flag=True,
41    help="Only show what would happen, don't change anything",
42)
43@click.option(
44    "-p",
45    "--pre",
46    is_flag=True,
47    default=None,
48    help="Allow resolving to prereleases (default is not)",
49)
50@click.option(
51    "-r",
52    "--rebuild",
53    is_flag=True,
54    help="Clear any caches upfront, rebuild from scratch",
55)
56@click.option(
57    "-f",
58    "--find-links",
59    multiple=True,
60    help="Look for archives in this directory or on this HTML page",
61    envvar="PIP_FIND_LINKS",
62)
63@click.option(
64    "-i",
65    "--index-url",
66    help="Change index URL (defaults to {})".format(pip_defaults.index_url),
67    envvar="PIP_INDEX_URL",
68)
69@click.option(
70    "--extra-index-url",
71    multiple=True,
72    help="Add additional index URL to search",
73    envvar="PIP_EXTRA_INDEX_URL",
74)
75@click.option("--cert", help="Path to alternate CA bundle.")
76@click.option(
77    "--client-cert",
78    help="Path to SSL client certificate, a single file containing "
79    "the private key and the certificate in PEM format.",
80)
81@click.option(
82    "--trusted-host",
83    multiple=True,
84    envvar="PIP_TRUSTED_HOST",
85    help="Mark this host as trusted, even though it does not have "
86    "valid or any HTTPS.",
87)
88@click.option(
89    "--header/--no-header",
90    is_flag=True,
91    default=True,
92    help="Add header to generated file",
93)
94@click.option(
95    "--index/--no-index",
96    is_flag=True,
97    default=True,
98    help="Add index URL to generated file",
99)
100@click.option(
101    "--emit-trusted-host/--no-emit-trusted-host",
102    is_flag=True,
103    default=True,
104    help="Add trusted host option to generated file",
105)
106@click.option(
107    "--annotate/--no-annotate",
108    is_flag=True,
109    default=True,
110    help="Annotate results, indicating where dependencies come from",
111)
112@click.option(
113    "-U",
114    "--upgrade",
115    is_flag=True,
116    default=False,
117    help="Try to upgrade all dependencies to their latest versions",
118)
119@click.option(
120    "-P",
121    "--upgrade-package",
122    "upgrade_packages",
123    nargs=1,
124    multiple=True,
125    help="Specify particular packages to upgrade.",
126)
127@click.option(
128    "-o",
129    "--output-file",
130    nargs=1,
131    default=None,
132    type=click.File("w+b", atomic=True, lazy=True),
133    help=(
134        "Output file name. Required if more than one input file is given. "
135        "Will be derived from input file otherwise."
136    ),
137)
138@click.option(
139    "--allow-unsafe",
140    is_flag=True,
141    default=False,
142    help="Pin packages considered unsafe: {}".format(
143        ", ".join(sorted(UNSAFE_PACKAGES))
144    ),
145)
146@click.option(
147    "--generate-hashes",
148    is_flag=True,
149    default=False,
150    help="Generate pip 8 style hashes in the resulting requirements file.",
151)
152@click.option(
153    "--max-rounds",
154    default=10,
155    help="Maximum number of rounds before resolving the requirements aborts.",
156)
157@click.argument("src_files", nargs=-1, type=click.Path(exists=True, allow_dash=True))
158@click.option(
159    "--build-isolation/--no-build-isolation",
160    is_flag=True,
161    default=True,
162    help="Enable isolation when building a modern source distribution. "
163    "Build dependencies specified by PEP 518 must be already installed "
164    "if build isolation is disabled.",
165)
166@click.option(
167    "--emit-find-links/--no-emit-find-links",
168    is_flag=True,
169    default=True,
170    help="Add the find-links option to generated file",
171)
172@click.option(
173    "--cache-dir",
174    help="Store the cache data in DIRECTORY.",
175    default=CACHE_DIR,
176    envvar="PIP_TOOLS_CACHE_DIR",
177    show_default=True,
178    show_envvar=True,
179    type=click.Path(file_okay=False, writable=True),
180)
181@click.option("--pip-args", help="Arguments to pass directly to the pip command.")
182def cli(
183    ctx,
184    verbose,
185    quiet,
186    dry_run,
187    pre,
188    rebuild,
189    find_links,
190    index_url,
191    extra_index_url,
192    cert,
193    client_cert,
194    trusted_host,
195    header,
196    index,
197    emit_trusted_host,
198    annotate,
199    upgrade,
200    upgrade_packages,
201    output_file,
202    allow_unsafe,
203    generate_hashes,
204    src_files,
205    max_rounds,
206    build_isolation,
207    emit_find_links,
208    cache_dir,
209    pip_args,
210):
211    """Compiles requirements.txt from requirements.in specs."""
212    log.verbosity = verbose - quiet
213
214    if len(src_files) == 0:
215        if os.path.exists(DEFAULT_REQUIREMENTS_FILE):
216            src_files = (DEFAULT_REQUIREMENTS_FILE,)
217        elif os.path.exists("setup.py"):
218            src_files = ("setup.py",)
219        else:
220            raise click.BadParameter(
221                (
222                    "If you do not specify an input file, "
223                    "the default is {} or setup.py"
224                ).format(DEFAULT_REQUIREMENTS_FILE)
225            )
226
227    if not output_file:
228        # An output file must be provided for stdin
229        if src_files == ("-",):
230            raise click.BadParameter("--output-file is required if input is from stdin")
231        # Use default requirements output file if there is a setup.py the source file
232        elif src_files == ("setup.py",):
233            file_name = DEFAULT_REQUIREMENTS_OUTPUT_FILE
234        # An output file must be provided if there are multiple source files
235        elif len(src_files) > 1:
236            raise click.BadParameter(
237                "--output-file is required if two or more input files are given."
238            )
239        # Otherwise derive the output file from the source file
240        else:
241            base_name = src_files[0].rsplit(".", 1)[0]
242            file_name = base_name + ".txt"
243
244        output_file = click.open_file(file_name, "w+b", atomic=True, lazy=True)
245
246        # Close the file at the end of the context execution
247        ctx.call_on_close(safecall(output_file.close_intelligently))
248
249    ###
250    # Setup
251    ###
252
253    right_args = shlex.split(pip_args or "")
254    pip_args = []
255    if find_links:
256        for link in find_links:
257            pip_args.extend(["-f", link])
258    if index_url:
259        pip_args.extend(["-i", index_url])
260    if extra_index_url:
261        for extra_index in extra_index_url:
262            pip_args.extend(["--extra-index-url", extra_index])
263    if cert:
264        pip_args.extend(["--cert", cert])
265    if client_cert:
266        pip_args.extend(["--client-cert", client_cert])
267    if pre:
268        pip_args.extend(["--pre"])
269    if trusted_host:
270        for host in trusted_host:
271            pip_args.extend(["--trusted-host", host])
272
273    if not build_isolation:
274        pip_args.append("--no-build-isolation")
275    pip_args.extend(right_args)
276
277    repository = PyPIRepository(pip_args, cache_dir=cache_dir)
278
279    # Parse all constraints coming from --upgrade-package/-P
280    upgrade_reqs_gen = (install_req_from_line(pkg) for pkg in upgrade_packages)
281    upgrade_install_reqs = {
282        key_from_ireq(install_req): install_req for install_req in upgrade_reqs_gen
283    }
284
285    existing_pins_to_upgrade = set()
286
287    # Proxy with a LocalRequirementsRepository if --upgrade is not specified
288    # (= default invocation)
289    if not upgrade and os.path.exists(output_file.name):
290        # Use a temporary repository to ensure outdated(removed) options from
291        # existing requirements.txt wouldn't get into the current repository.
292        tmp_repository = PyPIRepository(pip_args, cache_dir=cache_dir)
293        ireqs = parse_requirements(
294            output_file.name,
295            finder=tmp_repository.finder,
296            session=tmp_repository.session,
297            options=tmp_repository.options,
298        )
299
300        # Exclude packages from --upgrade-package/-P from the existing
301        # constraints, and separately gather pins to be upgraded
302        existing_pins = {}
303        for ireq in filter(is_pinned_requirement, ireqs):
304            key = key_from_ireq(ireq)
305            if key in upgrade_install_reqs:
306                existing_pins_to_upgrade.add(key)
307            else:
308                existing_pins[key] = ireq
309        repository = LocalRequirementsRepository(existing_pins, repository)
310
311    ###
312    # Parsing/collecting initial requirements
313    ###
314
315    constraints = []
316    for src_file in src_files:
317        is_setup_file = os.path.basename(src_file) == "setup.py"
318        if is_setup_file or src_file == "-":
319            # pip requires filenames and not files. Since we want to support
320            # piping from stdin, we need to briefly save the input from stdin
321            # to a temporary file and have pip read that.  also used for
322            # reading requirements from install_requires in setup.py.
323            tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False)
324            if is_setup_file:
325                from distutils.core import run_setup
326
327                dist = run_setup(src_file)
328                tmpfile.write("\n".join(dist.install_requires))
329                comes_from = "{name} ({filename})".format(
330                    name=dist.get_name(), filename=src_file
331                )
332            else:
333                tmpfile.write(sys.stdin.read())
334                comes_from = "-r -"
335            tmpfile.flush()
336            reqs = list(
337                parse_requirements(
338                    tmpfile.name,
339                    finder=repository.finder,
340                    session=repository.session,
341                    options=repository.options,
342                )
343            )
344            for req in reqs:
345                req.comes_from = comes_from
346            constraints.extend(reqs)
347        else:
348            constraints.extend(
349                parse_requirements(
350                    src_file,
351                    finder=repository.finder,
352                    session=repository.session,
353                    options=repository.options,
354                )
355            )
356
357    primary_packages = {
358        key_from_ireq(ireq) for ireq in constraints if not ireq.constraint
359    }
360
361    allowed_upgrades = primary_packages | existing_pins_to_upgrade
362    constraints.extend(
363        ireq for key, ireq in upgrade_install_reqs.items() if key in allowed_upgrades
364    )
365
366    # Filter out pip environment markers which do not match (PEP496)
367    constraints = [
368        req for req in constraints if req.markers is None or req.markers.evaluate()
369    ]
370
371    log.debug("Using indexes:")
372    for index_url in dedup(repository.finder.index_urls):
373        log.debug("  {}".format(index_url))
374
375    if repository.finder.find_links:
376        log.debug("")
377        log.debug("Configuration:")
378        for find_link in dedup(repository.finder.find_links):
379            log.debug("  -f {}".format(find_link))
380
381    try:
382        resolver = Resolver(
383            constraints,
384            repository,
385            prereleases=repository.finder.allow_all_prereleases or pre,
386            cache=DependencyCache(cache_dir),
387            clear_caches=rebuild,
388            allow_unsafe=allow_unsafe,
389        )
390        results = resolver.resolve(max_rounds=max_rounds)
391        if generate_hashes:
392            hashes = resolver.resolve_hashes(results)
393        else:
394            hashes = None
395    except PipToolsError as e:
396        log.error(str(e))
397        sys.exit(2)
398
399    log.debug("")
400
401    ##
402    # Output
403    ##
404
405    writer = OutputWriter(
406        src_files,
407        output_file,
408        click_ctx=ctx,
409        dry_run=dry_run,
410        emit_header=header,
411        emit_index=index,
412        emit_trusted_host=emit_trusted_host,
413        annotate=annotate,
414        generate_hashes=generate_hashes,
415        default_index_url=repository.DEFAULT_INDEX_URL,
416        index_urls=repository.finder.index_urls,
417        trusted_hosts=repository.finder.trusted_hosts,
418        format_control=repository.finder.format_control,
419        allow_unsafe=allow_unsafe,
420        find_links=repository.finder.find_links,
421        emit_find_links=emit_find_links,
422    )
423    writer.write(
424        results=results,
425        unsafe_requirements=resolver.unsafe_constraints,
426        markers={
427            key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers
428        },
429        hashes=hashes,
430    )
431
432    if dry_run:
433        log.info("Dry-run, so nothing updated.")
434