1import collections
2import os
3import sys
4import tempfile
5from subprocess import check_call  # nosec
6
7from pip._internal.commands.freeze import DEV_PKGS
8from pip._internal.utils.compat import stdlib_pkgs
9
10from . import click
11from .exceptions import IncompatibleRequirements
12from .utils import (
13    flat_map,
14    format_requirement,
15    get_hashes_from_ireq,
16    is_url_requirement,
17    key_from_ireq,
18    key_from_req,
19)
20
21PACKAGES_TO_IGNORE = (
22    ["-markerlib", "pip", "pip-tools", "pip-review", "pkg-resources"]
23    + list(stdlib_pkgs)
24    + list(DEV_PKGS)
25)
26
27
28def dependency_tree(installed_keys, root_key):
29    """
30    Calculate the dependency tree for the package `root_key` and return
31    a collection of all its dependencies.  Uses a DFS traversal algorithm.
32
33    `installed_keys` should be a {key: requirement} mapping, e.g.
34        {'django': from_line('django==1.8')}
35    `root_key` should be the key to return the dependency tree for.
36    """
37    dependencies = set()
38    queue = collections.deque()
39
40    if root_key in installed_keys:
41        dep = installed_keys[root_key]
42        queue.append(dep)
43
44    while queue:
45        v = queue.popleft()
46        key = key_from_req(v)
47        if key in dependencies:
48            continue
49
50        dependencies.add(key)
51
52        for dep_specifier in v.requires():
53            dep_name = key_from_req(dep_specifier)
54            if dep_name in installed_keys:
55                dep = installed_keys[dep_name]
56
57                if dep_specifier.specifier.contains(dep.version):
58                    queue.append(dep)
59
60    return dependencies
61
62
63def get_dists_to_ignore(installed):
64    """
65    Returns a collection of package names to ignore when performing pip-sync,
66    based on the currently installed environment.  For example, when pip-tools
67    is installed in the local environment, it should be ignored, including all
68    of its dependencies (e.g. click).  When pip-tools is not installed
69    locally, click should also be installed/uninstalled depending on the given
70    requirements.
71    """
72    installed_keys = {key_from_req(r): r for r in installed}
73    return list(
74        flat_map(lambda req: dependency_tree(installed_keys, req), PACKAGES_TO_IGNORE)
75    )
76
77
78def merge(requirements, ignore_conflicts):
79    by_key = {}
80
81    for ireq in requirements:
82        # Limitation: URL requirements are merged by precise string match, so
83        # "file:///example.zip#egg=example", "file:///example.zip", and
84        # "example==1.0" will not merge with each other
85        if ireq.match_markers():
86            key = key_from_ireq(ireq)
87
88            if not ignore_conflicts:
89                existing_ireq = by_key.get(key)
90                if existing_ireq:
91                    # NOTE: We check equality here since we can assume that the
92                    # requirements are all pinned
93                    if ireq.specifier != existing_ireq.specifier:
94                        raise IncompatibleRequirements(ireq, existing_ireq)
95
96            # TODO: Always pick the largest specifier in case of a conflict
97            by_key[key] = ireq
98    return by_key.values()
99
100
101def diff_key_from_ireq(ireq):
102    """
103    Calculate a key for comparing a compiled requirement with installed modules.
104    For URL requirements, only provide a useful key if the url includes
105    #egg=name==version, which will set ireq.req.name and ireq.specifier.
106    Otherwise return ireq.link so the key will not match and the package will
107    reinstall. Reinstall is necessary to ensure that packages will reinstall
108    if the URL is changed but the version is not.
109    """
110    if is_url_requirement(ireq):
111        if (
112            ireq.req
113            and (getattr(ireq.req, "key", None) or getattr(ireq.req, "name", None))
114            and ireq.specifier
115        ):
116            return key_from_ireq(ireq)
117        return str(ireq.link)
118    return key_from_ireq(ireq)
119
120
121def diff(compiled_requirements, installed_dists):
122    """
123    Calculate which packages should be installed or uninstalled, given a set
124    of compiled requirements and a list of currently installed modules.
125    """
126    requirements_lut = {diff_key_from_ireq(r): r for r in compiled_requirements}
127
128    satisfied = set()  # holds keys
129    to_install = set()  # holds InstallRequirement objects
130    to_uninstall = set()  # holds keys
131
132    pkgs_to_ignore = get_dists_to_ignore(installed_dists)
133    for dist in installed_dists:
134        key = key_from_req(dist)
135        if key not in requirements_lut or not requirements_lut[key].match_markers():
136            to_uninstall.add(key)
137        elif requirements_lut[key].specifier.contains(dist.version):
138            satisfied.add(key)
139
140    for key, requirement in requirements_lut.items():
141        if key not in satisfied and requirement.match_markers():
142            to_install.add(requirement)
143
144    # Make sure to not uninstall any packages that should be ignored
145    to_uninstall -= set(pkgs_to_ignore)
146
147    return (to_install, to_uninstall)
148
149
150def sync(
151    to_install,
152    to_uninstall,
153    verbose=False,
154    dry_run=False,
155    install_flags=None,
156    ask=False,
157):
158    """
159    Install and uninstalls the given sets of modules.
160    """
161    exit_code = 0
162
163    if not to_uninstall and not to_install:
164        if verbose:
165            click.echo("Everything up-to-date")
166        return exit_code
167
168    pip_flags = []
169    if not verbose:
170        pip_flags += ["-q"]
171
172    if ask:
173        dry_run = True
174
175    if dry_run:
176        if to_uninstall:
177            click.echo("Would uninstall:")
178            for pkg in sorted(to_uninstall):
179                click.echo("  {}".format(pkg))
180
181        if to_install:
182            click.echo("Would install:")
183            for ireq in sorted(to_install, key=key_from_ireq):
184                click.echo("  {}".format(format_requirement(ireq)))
185
186        exit_code = 1
187
188    if ask and click.confirm("Would you like to proceed with these changes?"):
189        dry_run = False
190        exit_code = 0
191
192    if not dry_run:
193        if to_uninstall:
194            check_call(  # nosec
195                [sys.executable, "-m", "pip", "uninstall", "-y"]
196                + pip_flags
197                + sorted(to_uninstall)
198            )
199
200        if to_install:
201            if install_flags is None:
202                install_flags = []
203            # prepare requirement lines
204            req_lines = []
205            for ireq in sorted(to_install, key=key_from_ireq):
206                ireq_hashes = get_hashes_from_ireq(ireq)
207                req_lines.append(format_requirement(ireq, hashes=ireq_hashes))
208
209            # save requirement lines to a temporary file
210            tmp_req_file = tempfile.NamedTemporaryFile(mode="wt", delete=False)
211            tmp_req_file.write("\n".join(req_lines))
212            tmp_req_file.close()
213
214            try:
215                check_call(  # nosec
216                    [sys.executable, "-m", "pip", "install", "-r", tmp_req_file.name]
217                    + pip_flags
218                    + install_flags
219                )
220            finally:
221                os.unlink(tmp_req_file.name)
222
223    return exit_code
224