1# coding: utf-8 2from __future__ import absolute_import, division, print_function, unicode_literals 3 4import collections 5import copy 6import hashlib 7import os 8from contextlib import contextmanager 9from shutil import rmtree 10 11from pip_shims.shims import ( 12 TempDirectory, 13 global_tempdir_manager, 14 get_requirement_tracker, 15 InstallCommand 16) 17from packaging.requirements import Requirement 18from packaging.specifiers import Specifier, SpecifierSet 19 20from .._compat import ( 21 FAVORITE_HASH, 22 PIP_VERSION, 23 InstallationError, 24 InstallRequirement, 25 Link, 26 normalize_path, 27 PyPI, 28 RequirementSet, 29 RequirementTracker, 30 SafeFileCache, 31 TemporaryDirectory, 32 VcsSupport, 33 Wheel, 34 WheelCache, 35 contextlib, 36 path_to_url, 37 pip_version, 38 url_to_path, 39) 40from ..locations import CACHE_DIR 41from ..click import progressbar 42from ..exceptions import NoCandidateFound 43from ..logging import log 44from ..utils import ( 45 dedup, 46 clean_requires_python, 47 fs_str, 48 is_pinned_requirement, 49 is_url_requirement, 50 lookup_table, 51 make_install_requirement, 52) 53from .base import BaseRepository 54 55os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") 56FILE_CHUNK_SIZE = 4096 57FileStream = collections.namedtuple("FileStream", "stream size") 58 59 60class HashCache(SafeFileCache): 61 """Caches hashes of PyPI artifacts so we do not need to re-download them 62 63 Hashes are only cached when the URL appears to contain a hash in it and the cache key includes 64 the hash value returned from the server). This ought to avoid ssues where the location on the 65 server changes.""" 66 def __init__(self, *args, **kwargs): 67 session = kwargs.pop('session') 68 self.session = session 69 kwargs.setdefault('directory', os.path.join(CACHE_DIR, 'hash-cache')) 70 super(HashCache, self).__init__(*args, **kwargs) 71 72 def get_hash(self, location): 73 # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it 74 hash_value = None 75 vcs = VcsSupport() 76 orig_scheme = location.scheme 77 new_location = copy.deepcopy(location) 78 if orig_scheme in vcs.all_schemes: 79 new_location.url = new_location.url.split("+", 1)[-1] 80 can_hash = new_location.hash 81 if can_hash: 82 # hash url WITH fragment 83 hash_value = self.get(new_location.url) 84 if not hash_value and new_location.hash_name == FAVORITE_HASH: 85 hash_value = "{}:{}".format(FAVORITE_HASH, new_location.hash) 86 hash_value = hash_value.encode('utf8') 87 if not hash_value: 88 hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None 89 hash_value = hash_value.encode('utf8') if hash_value else None 90 if can_hash: 91 self.set(new_location.url, hash_value) 92 return hash_value.decode('utf8') if hash_value else None 93 94 def _get_file_hash(self, location): 95 h = hashlib.new(FAVORITE_HASH) 96 with open_local_or_remote_file(location, self.session) as (fp, size): 97 for chunk in iter(lambda: fp.read(8096), b""): 98 h.update(chunk) 99 return ":".join([FAVORITE_HASH, h.hexdigest()]) 100 101 102class PyPIRepository(BaseRepository): 103 DEFAULT_INDEX_URL = PyPI.simple_url 104 105 """ 106 The PyPIRepository will use the provided Finder instance to lookup 107 packages. Typically, it looks up packages on PyPI (the default implicit 108 config), but any other PyPI mirror can be used if index_urls is 109 changed/configured on the Finder. 110 """ 111 112 def __init__(self, pip_args, cache_dir=CACHE_DIR, session=None, build_isolation=False, use_json=False): 113 self.build_isolation = build_isolation 114 self.use_json = use_json 115 self.cache_dir = cache_dir 116 117 # Use pip's parser for pip.conf management and defaults. 118 # General options (find_links, index_url, extra_index_url, trusted_host, 119 # and pre) are deferred to pip. 120 self.command = InstallCommand() 121 self.options, _ = self.command.parse_args(pip_args) 122 if self.build_isolation is not None: 123 self.options.build_isolation = build_isolation 124 if self.options.cache_dir: 125 self.options.cache_dir = normalize_path(self.options.cache_dir) 126 127 self.options.require_hashes = False 128 self.options.ignore_dependencies = False 129 130 if session is None: 131 session = self.command._build_session(self.options) 132 self.session = session 133 self.finder = self.command._build_package_finder( 134 options=self.options, session=self.session, ignore_requires_python=True 135 ) 136 137 # Caches 138 # stores project_name => InstallationCandidate mappings for all 139 # versions reported by PyPI, so we only have to ask once for each 140 # project 141 self._available_candidates_cache = {} 142 143 # stores InstallRequirement => list(InstallRequirement) mappings 144 # of all secondary dependencies for the given requirement, so we 145 # only have to go to disk once for each requirement 146 self._dependencies_cache = {} 147 self._json_dep_cache = {} 148 149 # stores *full* path + fragment => sha256 150 self._hash_cache = HashCache(session=session) 151 152 # Setup file paths 153 self.freshen_build_caches() 154 self._cache_dir = normalize_path(cache_dir) 155 self._download_dir = fs_str(os.path.join(self._cache_dir, "pkgs")) 156 self._wheel_download_dir = fs_str(os.path.join(self._cache_dir, "wheels")) 157 158 def freshen_build_caches(self): 159 """ 160 Start with fresh build/source caches. Will remove any old build 161 caches from disk automatically. 162 """ 163 self._build_dir = TemporaryDirectory(fs_str("build")) 164 self._source_dir = TemporaryDirectory(fs_str("source")) 165 166 @property 167 def build_dir(self): 168 return self._build_dir.name 169 170 @property 171 def source_dir(self): 172 return self._source_dir.name 173 174 def clear_caches(self): 175 rmtree(self._download_dir, ignore_errors=True) 176 rmtree(self._wheel_download_dir, ignore_errors=True) 177 178 def find_all_candidates(self, req_name): 179 if req_name not in self._available_candidates_cache: 180 candidates = self.finder.find_all_candidates(req_name) 181 self._available_candidates_cache[req_name] = candidates 182 return self._available_candidates_cache[req_name] 183 184 def find_best_match(self, ireq, prereleases=None): 185 """ 186 Returns a Version object that indicates the best match for the given 187 InstallRequirement according to the external repository. 188 """ 189 if ireq.editable or is_url_requirement(ireq): 190 return ireq # return itself as the best match 191 192 all_candidates = clean_requires_python(self.find_all_candidates(ireq.name)) 193 candidates_by_version = lookup_table( 194 all_candidates, key=lambda c: c.version, unique=True 195 ) 196 try: 197 matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), 198 prereleases=prereleases) 199 except TypeError: 200 matching_versions = [candidate.version for candidate in all_candidates] 201 202 # Reuses pip's internal candidate sort key to sort 203 matching_candidates = [candidates_by_version[ver] for ver in matching_versions] 204 if not matching_candidates: 205 raise NoCandidateFound(ireq, all_candidates, self.finder) 206 207 evaluator = self.finder.make_candidate_evaluator(ireq.name) 208 best_candidate_result = evaluator.compute_best_candidate(matching_candidates) 209 best_candidate = best_candidate_result.best_candidate 210 211 # Turn the candidate into a pinned InstallRequirement 212 return make_install_requirement( 213 best_candidate.name, 214 best_candidate.version, 215 ireq.extras, 216 ireq.markers, 217 constraint=ireq.constraint, 218 ) 219 220 def get_dependencies(self, ireq): 221 json_results = set() 222 223 if self.use_json: 224 try: 225 json_results = self.get_json_dependencies(ireq) 226 except TypeError: 227 json_results = set() 228 229 legacy_results = self.get_legacy_dependencies(ireq) 230 json_results.update(legacy_results) 231 232 return json_results 233 234 def get_json_dependencies(self, ireq): 235 236 if not (is_pinned_requirement(ireq)): 237 raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq)) 238 239 def gen(ireq): 240 if self.DEFAULT_INDEX_URL not in self.finder.index_urls: 241 return 242 243 url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) 244 releases = self.session.get(url).json()['releases'] 245 246 matches = [ 247 r for r in releases 248 if '=={0}'.format(r) == str(ireq.req.specifier) 249 ] 250 if not matches: 251 return 252 253 release_requires = self.session.get( 254 'https://pypi.org/pypi/{0}/{1}/json'.format( 255 ireq.req.name, matches[0], 256 ), 257 ).json() 258 try: 259 requires_dist = release_requires['info']['requires_dist'] 260 except KeyError: 261 return 262 263 for requires in requires_dist: 264 i = InstallRequirement.from_line(requires) 265 if 'extra' not in repr(i.markers): 266 yield i 267 268 try: 269 if ireq not in self._json_dep_cache: 270 self._json_dep_cache[ireq] = [g for g in gen(ireq)] 271 272 return set(self._json_dep_cache[ireq]) 273 except Exception: 274 return set() 275 276 def resolve_reqs(self, download_dir, ireq, wheel_cache): 277 with get_requirement_tracker() as req_tracker, TempDirectory( 278 kind="resolver" 279 ) as temp_dir: 280 preparer = self.command.make_requirement_preparer( 281 temp_build_dir=temp_dir, 282 options=self.options, 283 req_tracker=req_tracker, 284 session=self.session, 285 finder=self.finder, 286 use_user_site=False, 287 download_dir=download_dir, 288 wheel_download_dir=self._wheel_download_dir, 289 ) 290 291 reqset = RequirementSet() 292 ireq.is_direct = True 293 reqset.add_requirement(ireq) 294 295 resolver = self.command.make_resolver( 296 preparer=preparer, 297 finder=self.finder, 298 options=self.options, 299 wheel_cache=wheel_cache, 300 use_user_site=False, 301 ignore_installed=True, 302 ignore_requires_python=True, 303 force_reinstall=False, 304 upgrade_strategy="to-satisfy-only", 305 ) 306 results = resolver._resolve_one(reqset, ireq) 307 308 if PIP_VERSION[:2] <= (20, 0): 309 reqset.cleanup_files() 310 results = set(results) if results else set() 311 312 return results, ireq 313 314 def get_legacy_dependencies(self, ireq): 315 """ 316 Given a pinned, URL, or editable InstallRequirement, returns a set of 317 dependencies (also InstallRequirements, but not necessarily pinned). 318 They indicate the secondary dependencies for the given requirement. 319 """ 320 if not ( 321 ireq.editable or is_url_requirement(ireq) or is_pinned_requirement(ireq) 322 ): 323 raise TypeError( 324 "Expected url, pinned or editable InstallRequirement, got {}".format( 325 ireq 326 ) 327 ) 328 329 if ireq not in self._dependencies_cache: 330 if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): 331 # No download_dir for locally available editable requirements. 332 # If a download_dir is passed, pip will unnecessarely 333 # archive the entire source directory 334 download_dir = None 335 elif ireq.link and ireq.link.is_vcs: 336 # No download_dir for VCS sources. This also works around pip 337 # using git-checkout-index, which gets rid of the .git dir. 338 download_dir = None 339 else: 340 download_dir = self._download_dir 341 if not os.path.isdir(download_dir): 342 os.makedirs(download_dir) 343 if not os.path.isdir(self._wheel_download_dir): 344 os.makedirs(self._wheel_download_dir) 345 346 with global_tempdir_manager(): 347 wheel_cache = WheelCache(self._cache_dir, self.options.format_control) 348 prev_tracker = os.environ.get("PIP_REQ_TRACKER") 349 try: 350 results, ireq = self.resolve_reqs( 351 download_dir, ireq, wheel_cache 352 ) 353 self._dependencies_cache[ireq] = results 354 finally: 355 if "PIP_REQ_TRACKER" in os.environ: 356 if prev_tracker: 357 os.environ["PIP_REQ_TRACKER"] = prev_tracker 358 else: 359 del os.environ["PIP_REQ_TRACKER"] 360 361 if PIP_VERSION[:2] <= (20, 0): 362 wheel_cache.cleanup() 363 364 return self._dependencies_cache[ireq] 365 366 def get_hashes(self, ireq): 367 """ 368 Given an InstallRequirement, return a set of hashes that represent all 369 of the files for a given requirement. Unhashable requirements return an 370 empty set. Unpinned requirements raise a TypeError. 371 """ 372 373 if ireq.link: 374 link = ireq.link 375 376 if link.is_vcs or (link.is_file and link.is_existing_dir()): 377 # Return empty set for unhashable requirements. 378 # Unhashable logic modeled on pip's 379 # RequirementPreparer.prepare_linked_requirement 380 return set() 381 382 if is_url_requirement(ireq): 383 # Directly hash URL requirements. 384 # URL requirements may have been previously downloaded and cached 385 # locally by self.resolve_reqs() 386 cached_path = os.path.join(self._download_dir, link.filename) 387 if os.path.exists(cached_path): 388 cached_link = Link(path_to_url(cached_path)) 389 else: 390 cached_link = link 391 return {self._hash_cache._get_file_hash(cached_link)} 392 393 if not is_pinned_requirement(ireq): 394 raise TypeError("Expected pinned requirement, got {}".format(ireq)) 395 396 # We need to get all of the candidates that match our current version 397 # pin, these will represent all of the files that could possibly 398 # satisfy this constraint. 399 400 result = {} 401 with self.allow_all_links(): 402 matching_candidates = ( 403 c for c in clean_requires_python(self.find_all_candidates(ireq.name)) 404 if c.version in ireq.specifier 405 ) 406 log.debug(" {}".format(ireq.name)) 407 result = { 408 h for h in 409 map(lambda c: self._hash_cache.get_hash(c.link), matching_candidates) 410 if h is not None 411 } 412 return result 413 414 @contextmanager 415 def allow_all_links(self): 416 try: 417 self.finder._ignore_compatibility = True 418 yield 419 finally: 420 self.finder._ignore_compatibility = False 421 422 @contextmanager 423 def allow_all_wheels(self): 424 """ 425 Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. 426 427 This also saves the candidate cache and set a new one, or else the results from 428 the previous non-patched calls will interfere. 429 """ 430 431 def _wheel_supported(self, tags=None): 432 # Ignore current platform. Support everything. 433 return True 434 435 def _wheel_support_index_min(self, tags=None): 436 # All wheels are equal priority for sorting. 437 return 0 438 439 original_wheel_supported = Wheel.supported 440 original_support_index_min = Wheel.support_index_min 441 original_cache = self._available_candidates_cache 442 443 Wheel.supported = _wheel_supported 444 Wheel.support_index_min = _wheel_support_index_min 445 self._available_candidates_cache = {} 446 447 try: 448 yield 449 finally: 450 Wheel.supported = original_wheel_supported 451 Wheel.support_index_min = original_support_index_min 452 self._available_candidates_cache = original_cache 453 454 455@contextmanager 456def open_local_or_remote_file(link, session): 457 """ 458 Open local or remote file for reading. 459 460 :type link: pip.index.Link 461 :type session: requests.Session 462 :raises ValueError: If link points to a local directory. 463 :return: a context manager to a FileStream with the opened file-like object 464 """ 465 url = link.url_without_fragment 466 467 if link.is_file: 468 # Local URL 469 local_path = url_to_path(url) 470 if os.path.isdir(local_path): 471 raise ValueError("Cannot open directory for read: {}".format(url)) 472 else: 473 st = os.stat(local_path) 474 with open(local_path, "rb") as local_file: 475 yield FileStream(stream=local_file, size=st.st_size) 476 else: 477 # Remote URL 478 headers = {"Accept-Encoding": "identity"} 479 response = session.get(url, headers=headers, stream=True) 480 481 # Content length must be int or None 482 try: 483 content_length = int(response.headers["content-length"]) 484 except (ValueError, KeyError, TypeError): 485 content_length = None 486 487 try: 488 yield FileStream(stream=response.raw, size=content_length) 489 finally: 490 response.close() 491