1# coding: utf-8 2from __future__ import (absolute_import, division, print_function, 3 unicode_literals) 4 5import copy 6from functools import partial 7from itertools import chain, count 8import os 9 10from first import first 11from pip9.req import InstallRequirement 12 13from . import click 14from .cache import DependencyCache 15from .exceptions import UnsupportedConstraint 16from .logging import log 17from .utils import (format_requirement, format_specifier, full_groupby, dedup, 18 is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) 19 20green = partial(click.style, fg='green') 21magenta = partial(click.style, fg='magenta') 22 23 24class RequirementSummary(object): 25 """ 26 Summary of a requirement's properties for comparison purposes. 27 """ 28 def __init__(self, ireq): 29 self.req = ireq.req 30 self.key = key_from_req(ireq.req) 31 self.markers = ireq.markers 32 self.extras = str(sorted(ireq.extras)) 33 self.specifier = str(ireq.specifier) 34 35 def __eq__(self, other): 36 return str(self) == str(other) 37 38 def __hash__(self): 39 return hash(str(self)) 40 41 def __str__(self): 42 return repr([self.key, self.specifier, self.extras]) 43 44 45class Resolver(object): 46 def __init__(self, constraints, repository, cache=None, prereleases=False, clear_caches=False, allow_unsafe=False): 47 """ 48 This class resolves a given set of constraints (a collection of 49 InstallRequirement objects) by consulting the given Repository and the 50 DependencyCache. 51 """ 52 self.our_constraints = set(constraints) 53 self.their_constraints = set() 54 self.repository = repository 55 if cache is None: 56 cache = DependencyCache() # pragma: no cover 57 self.dependency_cache = cache 58 self.prereleases = prereleases 59 self.clear_caches = clear_caches 60 self.allow_unsafe = allow_unsafe 61 self.unsafe_constraints = set() 62 63 @property 64 def constraints(self): 65 return set(self._group_constraints(chain(self.our_constraints, 66 self.their_constraints))) 67 68 def resolve_hashes(self, ireqs): 69 """ 70 Finds acceptable hashes for all of the given InstallRequirements. 71 """ 72 with self.repository.allow_all_wheels(): 73 return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs} 74 75 def resolve(self, max_rounds=12): 76 """ 77 Finds concrete package versions for all the given InstallRequirements 78 and their recursive dependencies. The end result is a flat list of 79 (name, version) tuples. (Or an editable package.) 80 81 Resolves constraints one round at a time, until they don't change 82 anymore. Protects against infinite loops by breaking out after a max 83 number rounds. 84 """ 85 if self.clear_caches: 86 self.dependency_cache.clear() 87 self.repository.clear_caches() 88 89 self.check_constraints(chain(self.our_constraints, 90 self.their_constraints)) 91 92 # Ignore existing packages 93 os.environ[str('PIP_EXISTS_ACTION')] = str('i') # NOTE: str() wrapping necessary for Python 2/3 compat 94 for current_round in count(start=1): 95 if current_round > max_rounds: 96 raise RuntimeError('No stable configuration of concrete packages ' 97 'could be found for the given constraints after ' 98 '%d rounds of resolving.\n' 99 'This is likely a bug.' % max_rounds) 100 101 log.debug('') 102 log.debug(magenta('{:^60}'.format('ROUND {}'.format(current_round)))) 103 has_changed, best_matches = self._resolve_one_round() 104 log.debug('-' * 60) 105 log.debug('Result of round {}: {}'.format(current_round, 106 'not stable' if has_changed else 'stable, done')) 107 if not has_changed: 108 break 109 110 # If a package version (foo==2.0) was built in a previous round, 111 # and in this round a different version of foo needs to be built 112 # (i.e. foo==1.0), the directory will exist already, which will 113 # cause a pip build failure. The trick is to start with a new 114 # build cache dir for every round, so this can never happen. 115 self.repository.freshen_build_caches() 116 117 del os.environ['PIP_EXISTS_ACTION'] 118 # Only include hard requirements and not pip constraints 119 return {req for req in best_matches if not req.constraint} 120 121 @staticmethod 122 def check_constraints(constraints): 123 for constraint in constraints: 124 if constraint.link is not None and not constraint.editable and not constraint.is_wheel: 125 msg = ('pip-compile does not support URLs as packages, unless they are editable. ' 126 'Perhaps add -e option?') 127 raise UnsupportedConstraint(msg, constraint) 128 129 def _group_constraints(self, constraints): 130 """ 131 Groups constraints (remember, InstallRequirements!) by their key name, 132 and combining their SpecifierSets into a single InstallRequirement per 133 package. For example, given the following constraints: 134 135 Django<1.9,>=1.4.2 136 django~=1.5 137 Flask~=0.7 138 139 This will be combined into a single entry per package: 140 141 django~=1.5,<1.9,>=1.4.2 142 flask~=0.7 143 144 """ 145 for _, ireqs in full_groupby(constraints, key=key_from_ireq): 146 ireqs = list(ireqs) 147 editable_ireq = first(ireqs, key=lambda ireq: ireq.editable) 148 if editable_ireq: 149 yield editable_ireq # ignore all the other specs: the editable one is the one that counts 150 continue 151 152 ireqs = iter(ireqs) 153 # deepcopy the accumulator so as to not modify the self.our_constraints invariant 154 combined_ireq = copy.deepcopy(next(ireqs)) 155 combined_ireq.comes_from = None 156 for ireq in ireqs: 157 # NOTE we may be losing some info on dropped reqs here 158 combined_ireq.req.specifier &= ireq.req.specifier 159 combined_ireq.constraint &= ireq.constraint 160 combined_ireq.markers = ireq.markers 161 # Return a sorted, de-duped tuple of extras 162 combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) 163 yield combined_ireq 164 165 def _resolve_one_round(self): 166 """ 167 Resolves one level of the current constraints, by finding the best 168 match for each package in the repository and adding all requirements 169 for those best package versions. Some of these constraints may be new 170 or updated. 171 172 Returns whether new constraints appeared in this round. If no 173 constraints were added or changed, this indicates a stable 174 configuration. 175 """ 176 # Sort this list for readability of terminal output 177 constraints = sorted(self.constraints, key=key_from_ireq) 178 unsafe_constraints = [] 179 original_constraints = copy.copy(constraints) 180 if not self.allow_unsafe: 181 for constraint in original_constraints: 182 if constraint.name in UNSAFE_PACKAGES: 183 constraints.remove(constraint) 184 constraint.req.specifier = None 185 unsafe_constraints.append(constraint) 186 187 log.debug('Current constraints:') 188 for constraint in constraints: 189 log.debug(' {}'.format(constraint)) 190 191 log.debug('') 192 log.debug('Finding the best candidates:') 193 best_matches = {self.get_best_match(ireq) for ireq in constraints} 194 195 # Find the new set of secondary dependencies 196 log.debug('') 197 log.debug('Finding secondary dependencies:') 198 199 safe_constraints = [] 200 for best_match in best_matches: 201 for dep in self._iter_dependencies(best_match): 202 if self.allow_unsafe or dep.name not in UNSAFE_PACKAGES: 203 safe_constraints.append(dep) 204 # Grouping constraints to make clean diff between rounds 205 theirs = set(self._group_constraints(safe_constraints)) 206 207 # NOTE: We need to compare RequirementSummary objects, since 208 # InstallRequirement does not define equality 209 diff = {RequirementSummary(t) for t in theirs} - {RequirementSummary(t) for t in self.their_constraints} 210 removed = ({RequirementSummary(t) for t in self.their_constraints} - 211 {RequirementSummary(t) for t in theirs}) 212 unsafe = ({RequirementSummary(t) for t in unsafe_constraints} - 213 {RequirementSummary(t) for t in self.unsafe_constraints}) 214 215 has_changed = len(diff) > 0 or len(removed) > 0 or len(unsafe) > 0 216 if has_changed: 217 log.debug('') 218 log.debug('New dependencies found in this round:') 219 for new_dependency in sorted(diff, key=lambda req: key_from_req(req.req)): 220 log.debug(' adding {}'.format(new_dependency)) 221 log.debug('Removed dependencies in this round:') 222 for removed_dependency in sorted(removed, key=lambda req: key_from_req(req.req)): 223 log.debug(' removing {}'.format(removed_dependency)) 224 log.debug('Unsafe dependencies in this round:') 225 for unsafe_dependency in sorted(unsafe, key=lambda req: key_from_req(req.req)): 226 log.debug(' remembering unsafe {}'.format(unsafe_dependency)) 227 228 # Store the last round's results in the their_constraints 229 self.their_constraints = theirs 230 # Store the last round's unsafe constraints 231 self.unsafe_constraints = unsafe_constraints 232 return has_changed, best_matches 233 234 def get_best_match(self, ireq): 235 """ 236 Returns a (pinned or editable) InstallRequirement, indicating the best 237 match to use for the given InstallRequirement (in the form of an 238 InstallRequirement). 239 240 Example: 241 Given the constraint Flask>=0.10, may return Flask==0.10.1 at 242 a certain moment in time. 243 244 Pinned requirements will always return themselves, i.e. 245 246 Flask==0.10.1 => Flask==0.10.1 247 248 """ 249 if ireq.editable: 250 # NOTE: it's much quicker to immediately return instead of 251 # hitting the index server 252 best_match = ireq 253 elif is_pinned_requirement(ireq): 254 # NOTE: it's much quicker to immediately return instead of 255 # hitting the index server 256 best_match = ireq 257 else: 258 best_match = self.repository.find_best_match(ireq, prereleases=self.prereleases) 259 260 # Format the best match 261 log.debug(' found candidate {} (constraint was {})'.format(format_requirement(best_match), 262 format_specifier(ireq))) 263 return best_match 264 265 def _iter_dependencies(self, ireq): 266 """ 267 Given a pinned or editable InstallRequirement, collects all the 268 secondary dependencies for them, either by looking them up in a local 269 cache, or by reaching out to the repository. 270 271 Editable requirements will never be looked up, as they may have 272 changed at any time. 273 """ 274 if ireq.editable: 275 for dependency in self.repository.get_dependencies(ireq): 276 yield dependency 277 return 278 elif ireq.markers: 279 for dependency in self.repository.get_dependencies(ireq): 280 dependency.prepared = False 281 yield dependency 282 return 283 elif ireq.extras: 284 for dependency in self.repository.get_dependencies(ireq): 285 dependency.prepared = False 286 yield dependency 287 return 288 elif not is_pinned_requirement(ireq): 289 raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) 290 291 # Now, either get the dependencies from the dependency cache (for 292 # speed), or reach out to the external repository to 293 # download and inspect the package version and get dependencies 294 # from there 295 if ireq not in self.dependency_cache: 296 log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') 297 dependencies = self.repository.get_dependencies(ireq) 298 import sys 299 self.dependency_cache[ireq] = sorted(format_requirement(ireq) for ireq in dependencies) 300 301 # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] 302 dependency_strings = self.dependency_cache[ireq] 303 log.debug(' {:25} requires {}'.format(format_requirement(ireq), 304 ', '.join(sorted(dependency_strings, key=lambda s: s.lower())) or '-')) 305 from notpip._vendor.packaging.markers import InvalidMarker 306 for dependency_string in dependency_strings: 307 try: 308 _dependency_string = dependency_string 309 if ';' in dependency_string: 310 # split off markers and remove any duplicates by comparing against deps 311 _dependencies = [dep.strip() for dep in dependency_string.split(';')] 312 _dependency_string = '; '.join([dep for dep in dedup(_dependencies)]) 313 314 yield InstallRequirement.from_line(_dependency_string, constraint=ireq.constraint) 315 except InvalidMarker: 316 yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) 317 318 319 def reverse_dependencies(self, ireqs): 320 non_editable = [ireq for ireq in ireqs if not ireq.editable] 321 return self.dependency_cache.reverse_dependencies(non_editable) 322