1# coding: utf-8
2from __future__ import (absolute_import, division, print_function,
3                        unicode_literals)
4
5import copy
6from functools import partial
7from itertools import chain, count
8import os
9
10from first import first
11from pip9.req import InstallRequirement
12
13from . import click
14from .cache import DependencyCache
15from .exceptions import UnsupportedConstraint
16from .logging import log
17from .utils import (format_requirement, format_specifier, full_groupby, dedup,
18                    is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES)
19
20green = partial(click.style, fg='green')
21magenta = partial(click.style, fg='magenta')
22
23
24class RequirementSummary(object):
25    """
26    Summary of a requirement's properties for comparison purposes.
27    """
28    def __init__(self, ireq):
29        self.req = ireq.req
30        self.key = key_from_req(ireq.req)
31        self.markers = ireq.markers
32        self.extras = str(sorted(ireq.extras))
33        self.specifier = str(ireq.specifier)
34
35    def __eq__(self, other):
36        return str(self) == str(other)
37
38    def __hash__(self):
39        return hash(str(self))
40
41    def __str__(self):
42        return repr([self.key, self.specifier, self.extras])
43
44
45class Resolver(object):
46    def __init__(self, constraints, repository, cache=None, prereleases=False, clear_caches=False, allow_unsafe=False):
47        """
48        This class resolves a given set of constraints (a collection of
49        InstallRequirement objects) by consulting the given Repository and the
50        DependencyCache.
51        """
52        self.our_constraints = set(constraints)
53        self.their_constraints = set()
54        self.repository = repository
55        if cache is None:
56            cache = DependencyCache()  # pragma: no cover
57        self.dependency_cache = cache
58        self.prereleases = prereleases
59        self.clear_caches = clear_caches
60        self.allow_unsafe = allow_unsafe
61        self.unsafe_constraints = set()
62
63    @property
64    def constraints(self):
65        return set(self._group_constraints(chain(self.our_constraints,
66                                                 self.their_constraints)))
67
68    def resolve_hashes(self, ireqs):
69        """
70        Finds acceptable hashes for all of the given InstallRequirements.
71        """
72        with self.repository.allow_all_wheels():
73            return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs}
74
75    def resolve(self, max_rounds=12):
76        """
77        Finds concrete package versions for all the given InstallRequirements
78        and their recursive dependencies.  The end result is a flat list of
79        (name, version) tuples.  (Or an editable package.)
80
81        Resolves constraints one round at a time, until they don't change
82        anymore.  Protects against infinite loops by breaking out after a max
83        number rounds.
84        """
85        if self.clear_caches:
86            self.dependency_cache.clear()
87            self.repository.clear_caches()
88
89        self.check_constraints(chain(self.our_constraints,
90                                     self.their_constraints))
91
92        # Ignore existing packages
93        os.environ[str('PIP_EXISTS_ACTION')] = str('i')  # NOTE: str() wrapping necessary for Python 2/3 compat
94        for current_round in count(start=1):
95            if current_round > max_rounds:
96                raise RuntimeError('No stable configuration of concrete packages '
97                                   'could be found for the given constraints after '
98                                   '%d rounds of resolving.\n'
99                                   'This is likely a bug.' % max_rounds)
100
101            log.debug('')
102            log.debug(magenta('{:^60}'.format('ROUND {}'.format(current_round))))
103            has_changed, best_matches = self._resolve_one_round()
104            log.debug('-' * 60)
105            log.debug('Result of round {}: {}'.format(current_round,
106                                                      'not stable' if has_changed else 'stable, done'))
107            if not has_changed:
108                break
109
110            # If a package version (foo==2.0) was built in a previous round,
111            # and in this round a different version of foo needs to be built
112            # (i.e. foo==1.0), the directory will exist already, which will
113            # cause a pip build failure.  The trick is to start with a new
114            # build cache dir for every round, so this can never happen.
115            self.repository.freshen_build_caches()
116
117        del os.environ['PIP_EXISTS_ACTION']
118        # Only include hard requirements and not pip constraints
119        return {req for req in best_matches if not req.constraint}
120
121    @staticmethod
122    def check_constraints(constraints):
123        for constraint in constraints:
124            if constraint.link is not None and not constraint.editable and not constraint.is_wheel:
125                msg = ('pip-compile does not support URLs as packages, unless they are editable. '
126                       'Perhaps add -e option?')
127                raise UnsupportedConstraint(msg, constraint)
128
129    def _group_constraints(self, constraints):
130        """
131        Groups constraints (remember, InstallRequirements!) by their key name,
132        and combining their SpecifierSets into a single InstallRequirement per
133        package.  For example, given the following constraints:
134
135            Django<1.9,>=1.4.2
136            django~=1.5
137            Flask~=0.7
138
139        This will be combined into a single entry per package:
140
141            django~=1.5,<1.9,>=1.4.2
142            flask~=0.7
143
144        """
145        for _, ireqs in full_groupby(constraints, key=key_from_ireq):
146            ireqs = list(ireqs)
147            editable_ireq = first(ireqs, key=lambda ireq: ireq.editable)
148            if editable_ireq:
149                yield editable_ireq  # ignore all the other specs: the editable one is the one that counts
150                continue
151
152            ireqs = iter(ireqs)
153            # deepcopy the accumulator so as to not modify the self.our_constraints invariant
154            combined_ireq = copy.deepcopy(next(ireqs))
155            combined_ireq.comes_from = None
156            for ireq in ireqs:
157                # NOTE we may be losing some info on dropped reqs here
158                combined_ireq.req.specifier &= ireq.req.specifier
159                combined_ireq.constraint &= ireq.constraint
160                combined_ireq.markers = ireq.markers
161                # Return a sorted, de-duped tuple of extras
162                combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras))))
163            yield combined_ireq
164
165    def _resolve_one_round(self):
166        """
167        Resolves one level of the current constraints, by finding the best
168        match for each package in the repository and adding all requirements
169        for those best package versions.  Some of these constraints may be new
170        or updated.
171
172        Returns whether new constraints appeared in this round.  If no
173        constraints were added or changed, this indicates a stable
174        configuration.
175        """
176        # Sort this list for readability of terminal output
177        constraints = sorted(self.constraints, key=key_from_ireq)
178        unsafe_constraints = []
179        original_constraints = copy.copy(constraints)
180        if not self.allow_unsafe:
181            for constraint in original_constraints:
182                if constraint.name in UNSAFE_PACKAGES:
183                    constraints.remove(constraint)
184                    constraint.req.specifier = None
185                    unsafe_constraints.append(constraint)
186
187        log.debug('Current constraints:')
188        for constraint in constraints:
189            log.debug('  {}'.format(constraint))
190
191        log.debug('')
192        log.debug('Finding the best candidates:')
193        best_matches = {self.get_best_match(ireq) for ireq in constraints}
194
195        # Find the new set of secondary dependencies
196        log.debug('')
197        log.debug('Finding secondary dependencies:')
198
199        safe_constraints = []
200        for best_match in best_matches:
201            for dep in self._iter_dependencies(best_match):
202                if self.allow_unsafe or dep.name not in UNSAFE_PACKAGES:
203                    safe_constraints.append(dep)
204        # Grouping constraints to make clean diff between rounds
205        theirs = set(self._group_constraints(safe_constraints))
206
207        # NOTE: We need to compare RequirementSummary objects, since
208        # InstallRequirement does not define equality
209        diff = {RequirementSummary(t) for t in theirs} - {RequirementSummary(t) for t in self.their_constraints}
210        removed = ({RequirementSummary(t) for t in self.their_constraints} -
211                   {RequirementSummary(t) for t in theirs})
212        unsafe = ({RequirementSummary(t) for t in unsafe_constraints} -
213                  {RequirementSummary(t) for t in self.unsafe_constraints})
214
215        has_changed = len(diff) > 0 or len(removed) > 0 or len(unsafe) > 0
216        if has_changed:
217            log.debug('')
218            log.debug('New dependencies found in this round:')
219            for new_dependency in sorted(diff, key=lambda req: key_from_req(req.req)):
220                log.debug('  adding {}'.format(new_dependency))
221            log.debug('Removed dependencies in this round:')
222            for removed_dependency in sorted(removed, key=lambda req: key_from_req(req.req)):
223                log.debug('  removing {}'.format(removed_dependency))
224            log.debug('Unsafe dependencies in this round:')
225            for unsafe_dependency in sorted(unsafe, key=lambda req: key_from_req(req.req)):
226                log.debug('  remembering unsafe {}'.format(unsafe_dependency))
227
228        # Store the last round's results in the their_constraints
229        self.their_constraints = theirs
230        # Store the last round's unsafe constraints
231        self.unsafe_constraints = unsafe_constraints
232        return has_changed, best_matches
233
234    def get_best_match(self, ireq):
235        """
236        Returns a (pinned or editable) InstallRequirement, indicating the best
237        match to use for the given InstallRequirement (in the form of an
238        InstallRequirement).
239
240        Example:
241        Given the constraint Flask>=0.10, may return Flask==0.10.1 at
242        a certain moment in time.
243
244        Pinned requirements will always return themselves, i.e.
245
246            Flask==0.10.1 => Flask==0.10.1
247
248        """
249        if ireq.editable:
250            # NOTE: it's much quicker to immediately return instead of
251            # hitting the index server
252            best_match = ireq
253        elif is_pinned_requirement(ireq):
254            # NOTE: it's much quicker to immediately return instead of
255            # hitting the index server
256            best_match = ireq
257        else:
258            best_match = self.repository.find_best_match(ireq, prereleases=self.prereleases)
259
260        # Format the best match
261        log.debug('  found candidate {} (constraint was {})'.format(format_requirement(best_match),
262                                                                    format_specifier(ireq)))
263        return best_match
264
265    def _iter_dependencies(self, ireq):
266        """
267        Given a pinned or editable InstallRequirement, collects all the
268        secondary dependencies for them, either by looking them up in a local
269        cache, or by reaching out to the repository.
270
271        Editable requirements will never be looked up, as they may have
272        changed at any time.
273        """
274        if ireq.editable:
275            for dependency in self.repository.get_dependencies(ireq):
276                yield dependency
277            return
278        elif ireq.markers:
279            for dependency in self.repository.get_dependencies(ireq):
280                dependency.prepared = False
281                yield dependency
282            return
283        elif ireq.extras:
284            for dependency in self.repository.get_dependencies(ireq):
285                dependency.prepared = False
286                yield dependency
287            return
288        elif not is_pinned_requirement(ireq):
289            raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq))
290
291        # Now, either get the dependencies from the dependency cache (for
292        # speed), or reach out to the external repository to
293        # download and inspect the package version and get dependencies
294        # from there
295        if ireq not in self.dependency_cache:
296            log.debug('  {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow')
297            dependencies = self.repository.get_dependencies(ireq)
298            import sys
299            self.dependency_cache[ireq] = sorted(format_requirement(ireq) for ireq in dependencies)
300
301        # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4']
302        dependency_strings = self.dependency_cache[ireq]
303        log.debug('  {:25} requires {}'.format(format_requirement(ireq),
304                                               ', '.join(sorted(dependency_strings, key=lambda s: s.lower())) or '-'))
305        from notpip._vendor.packaging.markers import InvalidMarker
306        for dependency_string in dependency_strings:
307            try:
308                _dependency_string = dependency_string
309                if ';' in dependency_string:
310                    # split off markers and remove any duplicates by comparing against deps
311                    _dependencies = [dep.strip() for dep in dependency_string.split(';')]
312                    _dependency_string = '; '.join([dep for dep in dedup(_dependencies)])
313
314                yield InstallRequirement.from_line(_dependency_string, constraint=ireq.constraint)
315            except InvalidMarker:
316                yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint)
317
318
319    def reverse_dependencies(self, ireqs):
320        non_editable = [ireq for ireq in ireqs if not ireq.editable]
321        return self.dependency_cache.reverse_dependencies(non_editable)
322