1############################################################################# 2# 3# Copyright (c) 2005 Zope Foundation and Contributors. 4# All Rights Reserved. 5# 6# This software is subject to the provisions of the Zope Public License, 7# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11# FOR A PARTICULAR PURPOSE. 12# 13############################################################################## 14"""Python easy_install API 15 16This module provides a high-level Python API for installing packages. 17It doesn't install scripts. It uses setuptools and requires it to be 18installed. 19""" 20 21import distutils.errors 22import errno 23import glob 24import logging 25import os 26import pkg_resources 27import py_compile 28import re 29import setuptools.archive_util 30import setuptools.command.easy_install 31import setuptools.command.setopt 32import setuptools.package_index 33import shutil 34import subprocess 35import sys 36import tempfile 37import zc.buildout 38import warnings 39 40try: 41 from setuptools.wheel import Wheel # This is the important import 42 from setuptools import __version__ as setuptools_version 43 # Now we need to check if we have at least 38.2.3 for namespace support. 44 SETUPTOOLS_SUPPORTS_WHEELS = ( 45 pkg_resources.parse_version(setuptools_version) >= 46 pkg_resources.parse_version('38.2.3')) 47except ImportError: 48 SETUPTOOLS_SUPPORTS_WHEELS = False 49 50warnings.filterwarnings( 51 'ignore', '.+is being parsed as a legacy, non PEP 440, version') 52 53_oprp = getattr(os.path, 'realpath', lambda path: path) 54def realpath(path): 55 return os.path.normcase(os.path.abspath(_oprp(path))) 56 57default_index_url = os.environ.get( 58 'buildout-testing-index-url', 59 'https://pypi.org/simple', 60 ) 61 62logger = logging.getLogger('zc.buildout.easy_install') 63 64url_match = re.compile('[a-z0-9+.-]+://').match 65is_source_encoding_line = re.compile('coding[:=]\s*([-\w.]+)').search 66# Source encoding regex from http://www.python.org/dev/peps/pep-0263/ 67 68is_win32 = sys.platform == 'win32' 69is_jython = sys.platform.startswith('java') 70 71if is_jython: 72 import java.lang.System 73 jython_os_name = (java.lang.System.getProperties()['os.name']).lower() 74 75# Make sure we're not being run with an older bootstrap.py that gives us 76# setuptools instead of setuptools 77has_distribute = pkg_resources.working_set.find( 78 pkg_resources.Requirement.parse('distribute')) is not None 79has_setuptools = pkg_resources.working_set.find( 80 pkg_resources.Requirement.parse('setuptools')) is not None 81if has_distribute and not has_setuptools: 82 sys.exit("zc.buildout 2 needs setuptools, not distribute." 83 " Are you using an outdated bootstrap.py? Make sure" 84 " you have the latest version downloaded from" 85 " https://bootstrap.pypa.io/bootstrap-buildout.py") 86 87# Include buildout and setuptools eggs in paths. We get this 88# initially from the entire working set. Later, we'll use the install 89# function to narrow to just the buildout and setuptools paths. 90buildout_and_setuptools_path = [d.location for d in pkg_resources.working_set] 91setuptools_path = buildout_and_setuptools_path 92 93FILE_SCHEME = re.compile('file://', re.I).match 94DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$") 95 96class _Monkey(object): 97 def __init__(self, module, **kw): 98 mdict = self._mdict = module.__dict__ 99 self._before = mdict.copy() 100 self._overrides = kw 101 102 def __enter__(self): 103 self._mdict.update(self._overrides) 104 return self 105 106 def __exit__(self, exc_type, exc_value, traceback): 107 self._mdict.clear() 108 self._mdict.update(self._before) 109 110class _NoWarn(object): 111 def warn(self, *args, **kw): 112 pass 113 114_no_warn = _NoWarn() 115 116class AllowHostsPackageIndex(setuptools.package_index.PackageIndex): 117 """Will allow urls that are local to the system. 118 119 No matter what is allow_hosts. 120 """ 121 def url_ok(self, url, fatal=False): 122 if FILE_SCHEME(url): 123 return True 124 # distutils has its own logging, which can't be hooked / suppressed, 125 # so we monkey-patch the 'log' submodule to suppress the stupid 126 # "Link to <URL> ***BLOCKED*** by --allow-hosts" message. 127 with _Monkey(setuptools.package_index, log=_no_warn): 128 return setuptools.package_index.PackageIndex.url_ok( 129 self, url, False) 130 131 132_indexes = {} 133def _get_index(index_url, find_links, allow_hosts=('*',)): 134 key = index_url, tuple(find_links) 135 index = _indexes.get(key) 136 if index is not None: 137 return index 138 139 if index_url is None: 140 index_url = default_index_url 141 if index_url.startswith('file://'): 142 index_url = index_url[7:] 143 index = AllowHostsPackageIndex(index_url, hosts=allow_hosts) 144 145 if find_links: 146 index.add_find_links(find_links) 147 148 _indexes[key] = index 149 return index 150 151clear_index_cache = _indexes.clear 152 153if is_win32: 154 # work around spawn lamosity on windows 155 # XXX need safe quoting (see the subproces.list2cmdline) and test 156 def _safe_arg(arg): 157 return '"%s"' % arg 158else: 159 _safe_arg = str 160 161def call_subprocess(args, **kw): 162 if subprocess.call(args, **kw) != 0: 163 raise Exception( 164 "Failed to run command:\n%s" 165 % repr(args)[1:-1]) 166 167 168def _execute_permission(): 169 current_umask = os.umask(0o022) 170 # os.umask only returns the current umask if you also give it one, so we 171 # have to give it a dummy one and immediately set it back to the real 172 # value... Distribute does the same. 173 os.umask(current_umask) 174 return 0o777 - current_umask 175 176 177_easy_install_cmd = 'from setuptools.command.easy_install import main; main()' 178 179def get_namespace_package_paths(dist): 180 """ 181 Generator of the expected pathname of each __init__.py file of the 182 namespaces of a distribution. 183 """ 184 base = [dist.location] 185 init = ['__init__.py'] 186 for namespace in dist.get_metadata_lines('namespace_packages.txt'): 187 yield os.path.join(*(base + namespace.split('.') + init)) 188 189def namespace_packages_need_pkg_resources(dist): 190 if os.path.isfile(dist.location): 191 # Zipped egg, with namespaces, surely needs setuptools 192 return True 193 # If they have `__init__.py` files that use pkg_resources and don't 194 # fallback to using `pkgutil`, then they need setuptools/pkg_resources: 195 for path in get_namespace_package_paths(dist): 196 if os.path.isfile(path): 197 with open(path, 'rb') as f: 198 source = f.read() 199 if (source and 200 b'pkg_resources' in source and 201 not b'pkgutil' in source): 202 return True 203 return False 204 205def dist_needs_pkg_resources(dist): 206 """ 207 A distribution needs setuptools/pkg_resources added as requirement if: 208 209 * It has namespace packages declared with: 210 - `pkg_resources.declare_namespace()` 211 * Those namespace packages don't fall back to `pkgutil` 212 * It doesn't have `setuptools/pkg_resources` as requirement already 213 """ 214 215 return ( 216 dist.has_metadata('namespace_packages.txt') and 217 # This will need to change when `pkg_resources` gets its own 218 # project: 219 'setuptools' not in {r.project_name for r in dist.requires()} and 220 namespace_packages_need_pkg_resources(dist) 221 ) 222 223 224class Installer: 225 226 _versions = {} 227 _required_by = {} 228 _picked_versions = {} 229 _download_cache = None 230 _install_from_cache = False 231 _prefer_final = True 232 _use_dependency_links = True 233 _allow_picked_versions = True 234 _store_required_by = False 235 236 def __init__(self, 237 dest=None, 238 links=(), 239 index=None, 240 executable=sys.executable, 241 always_unzip=None, # Backward compat :/ 242 path=None, 243 newest=True, 244 versions=None, 245 use_dependency_links=None, 246 allow_hosts=('*',), 247 check_picked=True, 248 ): 249 assert executable == sys.executable, (executable, sys.executable) 250 self._dest = dest if dest is None else pkg_resources.normalize_path(dest) 251 self._allow_hosts = allow_hosts 252 253 if self._install_from_cache: 254 if not self._download_cache: 255 raise ValueError("install_from_cache set to true with no" 256 " download cache") 257 links = () 258 index = 'file://' + self._download_cache 259 260 if use_dependency_links is not None: 261 self._use_dependency_links = use_dependency_links 262 self._links = links = list(self._fix_file_links(links)) 263 if self._download_cache and (self._download_cache not in links): 264 links.insert(0, self._download_cache) 265 266 self._index_url = index 267 path = (path and path[:] or []) + buildout_and_setuptools_path 268 self._path = path 269 if self._dest is None: 270 newest = False 271 self._newest = newest 272 self._env = self._make_env() 273 self._index = _get_index(index, links, self._allow_hosts) 274 self._requirements_and_constraints = [] 275 self._check_picked = check_picked 276 277 if versions is not None: 278 self._versions = normalize_versions(versions) 279 280 def _make_env(self): 281 full_path = self._get_dest_dist_paths() + self._path 282 env = pkg_resources.Environment(full_path) 283 # this needs to be called whenever self._env is modified (or we could 284 # make an Environment subclass): 285 self._eggify_env_dest_dists(env, self._dest) 286 return env 287 288 def _env_rescan_dest(self): 289 self._env.scan(self._get_dest_dist_paths()) 290 self._eggify_env_dest_dists(self._env, self._dest) 291 292 def _get_dest_dist_paths(self): 293 dest = self._dest 294 if dest is None: 295 return [] 296 eggs = glob.glob(os.path.join(dest, '*.egg')) 297 dists = [os.path.dirname(dist_info) for dist_info in 298 glob.glob(os.path.join(dest, '*', '*.dist-info'))] 299 return list(set(eggs + dists)) 300 301 @staticmethod 302 def _eggify_env_dest_dists(env, dest): 303 """ 304 Make sure everything found under `dest` is seen as an egg, even if it's 305 some other kind of dist. 306 """ 307 for project_name in env: 308 for dist in env[project_name]: 309 if os.path.dirname(dist.location) == dest: 310 dist.precedence = pkg_resources.EGG_DIST 311 312 def _version_conflict_information(self, name): 313 """Return textual requirements/constraint information for debug purposes 314 315 We do a very simple textual search, as that filters out most 316 extraneous information witout missing anything. 317 318 """ 319 output = [ 320 "Version and requirements information containing %s:" % name] 321 version_constraint = self._versions.get(name) 322 if version_constraint: 323 output.append( 324 "[versions] constraint on %s: %s" % (name, version_constraint)) 325 output += [line for line in self._requirements_and_constraints 326 if name.lower() in line.lower()] 327 return '\n '.join(output) 328 329 def _satisfied(self, req, source=None): 330 dists = [dist for dist in self._env[req.project_name] if dist in req] 331 if not dists: 332 logger.debug('We have no distributions for %s that satisfies %r.', 333 req.project_name, str(req)) 334 335 return None, self._obtain(req, source) 336 337 # Note that dists are sorted from best to worst, as promised by 338 # env.__getitem__ 339 340 for dist in dists: 341 if (dist.precedence == pkg_resources.DEVELOP_DIST): 342 logger.debug('We have a develop egg: %s', dist) 343 return dist, None 344 345 # Special common case, we have a specification for a single version: 346 specs = req.specs 347 if len(specs) == 1 and specs[0][0] == '==': 348 logger.debug('We have the distribution that satisfies %r.', 349 str(req)) 350 return dists[0], None 351 352 if self._prefer_final: 353 fdists = [dist for dist in dists 354 if self._final_version(dist.parsed_version) 355 ] 356 if fdists: 357 # There are final dists, so only use those 358 dists = fdists 359 360 if not self._newest: 361 # We don't need the newest, so we'll use the newest one we 362 # find, which is the first returned by 363 # Environment.__getitem__. 364 return dists[0], None 365 366 best_we_have = dists[0] # Because dists are sorted from best to worst 367 368 # We have some installed distros. There might, theoretically, be 369 # newer ones. Let's find out which ones are available and see if 370 # any are newer. We only do this if we're willing to install 371 # something, which is only true if dest is not None: 372 373 best_available = self._obtain(req, source) 374 375 if best_available is None: 376 # That's a bit odd. There aren't any distros available. 377 # We should use the best one we have that meets the requirement. 378 logger.debug( 379 'There are no distros available that meet %r.\n' 380 'Using our best, %s.', 381 str(req), best_available) 382 return best_we_have, None 383 384 if self._prefer_final: 385 if self._final_version(best_available.parsed_version): 386 if self._final_version(best_we_have.parsed_version): 387 if (best_we_have.parsed_version 388 < 389 best_available.parsed_version 390 ): 391 return None, best_available 392 else: 393 return None, best_available 394 else: 395 if (not self._final_version(best_we_have.parsed_version) 396 and 397 (best_we_have.parsed_version 398 < 399 best_available.parsed_version 400 ) 401 ): 402 return None, best_available 403 else: 404 if (best_we_have.parsed_version 405 < 406 best_available.parsed_version 407 ): 408 return None, best_available 409 410 logger.debug( 411 'We have the best distribution that satisfies %r.', 412 str(req)) 413 return best_we_have, None 414 415 def _call_easy_install(self, spec, dest, dist): 416 417 tmp = tempfile.mkdtemp(dir=dest) 418 try: 419 paths = call_easy_install(spec, tmp) 420 421 dists = [] 422 env = pkg_resources.Environment(paths) 423 for project in env: 424 dists.extend(env[project]) 425 426 if not dists: 427 raise zc.buildout.UserError("Couldn't install: %s" % dist) 428 429 if len(dists) > 1: 430 logger.warn("Installing %s\n" 431 "caused multiple distributions to be installed:\n" 432 "%s\n", 433 dist, '\n'.join(map(str, dists))) 434 else: 435 d = dists[0] 436 if d.project_name != dist.project_name: 437 logger.warn("Installing %s\n" 438 "Caused installation of a distribution:\n" 439 "%s\n" 440 "with a different project name.", 441 dist, d) 442 if d.version != dist.version: 443 logger.warn("Installing %s\n" 444 "Caused installation of a distribution:\n" 445 "%s\n" 446 "with a different version.", 447 dist, d) 448 449 result = [] 450 for d in dists: 451 result.append(_move_to_eggs_dir_and_compile(d, dest)) 452 453 return result 454 455 finally: 456 shutil.rmtree(tmp) 457 458 def _obtain(self, requirement, source=None): 459 # initialize out index for this project: 460 index = self._index 461 462 if index.obtain(requirement) is None: 463 # Nothing is available. 464 return None 465 466 # Filter the available dists for the requirement and source flag 467 dists = [dist for dist in index[requirement.project_name] 468 if ((dist in requirement) 469 and 470 ((not source) or 471 (dist.precedence == pkg_resources.SOURCE_DIST) 472 ) 473 ) 474 ] 475 476 # If we prefer final dists, filter for final and use the 477 # result if it is non empty. 478 if self._prefer_final: 479 fdists = [dist for dist in dists 480 if self._final_version(dist.parsed_version) 481 ] 482 if fdists: 483 # There are final dists, so only use those 484 dists = fdists 485 486 # Now find the best one: 487 best = [] 488 bestv = None 489 for dist in dists: 490 distv = dist.parsed_version 491 if bestv is None or distv > bestv: 492 best = [dist] 493 bestv = distv 494 elif distv == bestv: 495 best.append(dist) 496 497 if not best: 498 return None 499 500 if len(best) == 1: 501 return best[0] 502 503 if self._download_cache: 504 for dist in best: 505 if (realpath(os.path.dirname(dist.location)) 506 == 507 self._download_cache 508 ): 509 return dist 510 511 best.sort() 512 return best[-1] 513 514 def _fetch(self, dist, tmp, download_cache): 515 if (download_cache 516 and (realpath(os.path.dirname(dist.location)) == download_cache) 517 ): 518 logger.debug("Download cache has %s at: %s", dist, dist.location) 519 return dist 520 521 logger.debug("Fetching %s from: %s", dist, dist.location) 522 new_location = self._index.download(dist.location, tmp) 523 if (download_cache 524 and (realpath(new_location) == realpath(dist.location)) 525 and os.path.isfile(new_location) 526 ): 527 # setuptools avoids making extra copies, but we want to copy 528 # to the download cache 529 shutil.copy2(new_location, tmp) 530 new_location = os.path.join(tmp, os.path.basename(new_location)) 531 532 return dist.clone(location=new_location) 533 534 def _get_dist(self, requirement, ws): 535 __doing__ = 'Getting distribution for %r.', str(requirement) 536 537 # Maybe an existing dist is already the best dist that satisfies the 538 # requirement 539 dist, avail = self._satisfied(requirement) 540 541 if dist is None: 542 if self._dest is None: 543 raise zc.buildout.UserError( 544 "We don't have a distribution for %s\n" 545 "and can't install one in offline (no-install) mode.\n" 546 % requirement) 547 548 logger.info(*__doing__) 549 550 # Retrieve the dist: 551 if avail is None: 552 self._index.obtain(requirement) 553 raise MissingDistribution(requirement, ws) 554 555 # We may overwrite distributions, so clear importer 556 # cache. 557 sys.path_importer_cache.clear() 558 559 tmp = self._download_cache 560 if tmp is None: 561 tmp = tempfile.mkdtemp('get_dist') 562 563 try: 564 dist = self._fetch(avail, tmp, self._download_cache) 565 566 if dist is None: 567 raise zc.buildout.UserError( 568 "Couldn't download distribution %s." % avail) 569 570 dists = [_move_to_eggs_dir_and_compile(dist, self._dest)] 571 for _d in dists: 572 if _d not in ws: 573 ws.add(_d, replace=True) 574 575 finally: 576 if tmp != self._download_cache: 577 shutil.rmtree(tmp) 578 579 self._env_rescan_dest() 580 dist = self._env.best_match(requirement, ws) 581 582 logger.info("Got %s.", dist) 583 584 else: 585 dists = [dist] 586 if dist not in ws: 587 ws.add(dist) 588 589 590 if not self._install_from_cache and self._use_dependency_links: 591 self._add_dependency_links_from_dists(dists) 592 593 if self._check_picked: 594 self._check_picked_requirement_versions(requirement, dists) 595 596 return dists 597 598 def _add_dependency_links_from_dists(self, dists): 599 reindex = False 600 links = self._links 601 for dist in dists: 602 if dist.has_metadata('dependency_links.txt'): 603 for link in dist.get_metadata_lines('dependency_links.txt'): 604 link = link.strip() 605 if link not in links: 606 logger.debug('Adding find link %r from %s', 607 link, dist) 608 links.append(link) 609 reindex = True 610 if reindex: 611 self._index = _get_index(self._index_url, links, self._allow_hosts) 612 613 def _check_picked_requirement_versions(self, requirement, dists): 614 """ Check whether we picked a version and, if we did, report it """ 615 for dist in dists: 616 if not (dist.precedence == pkg_resources.DEVELOP_DIST 617 or 618 (len(requirement.specs) == 1 619 and 620 requirement.specs[0][0] == '==') 621 ): 622 logger.debug('Picked: %s = %s', 623 dist.project_name, dist.version) 624 self._picked_versions[dist.project_name] = dist.version 625 626 if not self._allow_picked_versions: 627 raise zc.buildout.UserError( 628 'Picked: %s = %s' % (dist.project_name, 629 dist.version) 630 ) 631 632 def _maybe_add_setuptools(self, ws, dist): 633 if dist_needs_pkg_resources(dist): 634 # We have a namespace package but no requirement for setuptools 635 if dist.precedence == pkg_resources.DEVELOP_DIST: 636 logger.warn( 637 "Develop distribution: %s\n" 638 "uses namespace packages but the distribution " 639 "does not require setuptools.", 640 dist) 641 requirement = self._constrain( 642 pkg_resources.Requirement.parse('setuptools') 643 ) 644 if ws.find(requirement) is None: 645 self._get_dist(requirement, ws) 646 647 def _constrain(self, requirement): 648 """Return requirement with optional [versions] constraint added.""" 649 constraint = self._versions.get(requirement.project_name.lower()) 650 if constraint: 651 try: 652 requirement = _constrained_requirement(constraint, 653 requirement) 654 except IncompatibleConstraintError: 655 logger.info(self._version_conflict_information( 656 requirement.project_name.lower())) 657 raise 658 659 return requirement 660 661 def install(self, specs, working_set=None): 662 663 logger.debug('Installing %s.', repr(specs)[1:-1]) 664 self._requirements_and_constraints.append( 665 "Base installation request: %s" % repr(specs)[1:-1]) 666 667 for_buildout_run = bool(working_set) 668 669 requirements = [self._constrain(pkg_resources.Requirement.parse(spec)) 670 for spec in specs] 671 672 if working_set is None: 673 ws = pkg_resources.WorkingSet([]) 674 else: 675 ws = working_set 676 677 for requirement in requirements: 678 for dist in self._get_dist(requirement, ws): 679 self._maybe_add_setuptools(ws, dist) 680 681 # OK, we have the requested distributions and they're in the working 682 # set, but they may have unmet requirements. We'll resolve these 683 # requirements. This is code modified from 684 # pkg_resources.WorkingSet.resolve. We can't reuse that code directly 685 # because we have to constrain our requirements (see 686 # versions_section_ignored_for_dependency_in_favor_of_site_packages in 687 # zc.buildout.tests). 688 requirements.reverse() # Set up the stack. 689 processed = {} # This is a set of processed requirements. 690 best = {} # This is a mapping of package name -> dist. 691 # Note that we don't use the existing environment, because we want 692 # to look for new eggs unless what we have is the best that 693 # matches the requirement. 694 env = pkg_resources.Environment(ws.entries) 695 696 while requirements: 697 # Process dependencies breadth-first. 698 current_requirement = requirements.pop(0) 699 req = self._constrain(current_requirement) 700 if req in processed: 701 # Ignore cyclic or redundant dependencies. 702 continue 703 dist = best.get(req.key) 704 if dist is None: 705 try: 706 dist = env.best_match(req, ws) 707 except pkg_resources.VersionConflict as err: 708 logger.debug( 709 "Version conflict while processing requirement %s " 710 "(constrained to %s)", 711 current_requirement, req) 712 # Installing buildout itself and its extensions and 713 # recipes requires the global 714 # ``pkg_resources.working_set`` to be active, which also 715 # includes all system packages. So there might be 716 # conflicts, which are fine to ignore. We'll grab the 717 # correct version a few lines down. 718 if not for_buildout_run: 719 raise VersionConflict(err, ws) 720 if dist is None: 721 if self._dest: 722 logger.debug('Getting required %r', str(req)) 723 else: 724 logger.debug('Adding required %r', str(req)) 725 self._log_requirement(ws, req) 726 for dist in self._get_dist(req, ws): 727 self._maybe_add_setuptools(ws, dist) 728 if dist not in req: 729 # Oops, the "best" so far conflicts with a dependency. 730 logger.info(self._version_conflict_information(req.key)) 731 raise VersionConflict( 732 pkg_resources.VersionConflict(dist, req), ws) 733 734 best[req.key] = dist 735 extra_requirements = dist.requires(req.extras)[::-1] 736 for extra_requirement in extra_requirements: 737 self._requirements_and_constraints.append( 738 "Requirement of %s: %s" % ( 739 current_requirement, extra_requirement)) 740 requirements.extend(extra_requirements) 741 742 processed[req] = True 743 return ws 744 745 def build(self, spec, build_ext): 746 747 requirement = self._constrain(pkg_resources.Requirement.parse(spec)) 748 749 dist, avail = self._satisfied(requirement, 1) 750 if dist is not None: 751 return [dist.location] 752 753 # Retrieve the dist: 754 if avail is None: 755 raise zc.buildout.UserError( 756 "Couldn't find a source distribution for %r." 757 % str(requirement)) 758 759 if self._dest is None: 760 raise zc.buildout.UserError( 761 "We don't have a distribution for %s\n" 762 "and can't build one in offline (no-install) mode.\n" 763 % requirement 764 ) 765 766 logger.debug('Building %r', spec) 767 768 tmp = self._download_cache 769 if tmp is None: 770 tmp = tempfile.mkdtemp('get_dist') 771 772 try: 773 dist = self._fetch(avail, tmp, self._download_cache) 774 775 build_tmp = tempfile.mkdtemp('build') 776 try: 777 setuptools.archive_util.unpack_archive(dist.location, 778 build_tmp) 779 if os.path.exists(os.path.join(build_tmp, 'setup.py')): 780 base = build_tmp 781 else: 782 setups = glob.glob( 783 os.path.join(build_tmp, '*', 'setup.py')) 784 if not setups: 785 raise distutils.errors.DistutilsError( 786 "Couldn't find a setup script in %s" 787 % os.path.basename(dist.location) 788 ) 789 if len(setups) > 1: 790 raise distutils.errors.DistutilsError( 791 "Multiple setup scripts in %s" 792 % os.path.basename(dist.location) 793 ) 794 base = os.path.dirname(setups[0]) 795 796 setup_cfg = os.path.join(base, 'setup.cfg') 797 if not os.path.exists(setup_cfg): 798 f = open(setup_cfg, 'w') 799 f.close() 800 setuptools.command.setopt.edit_config( 801 setup_cfg, dict(build_ext=build_ext)) 802 803 dists = self._call_easy_install(base, self._dest, dist) 804 805 return [dist.location for dist in dists] 806 finally: 807 shutil.rmtree(build_tmp) 808 809 finally: 810 if tmp != self._download_cache: 811 shutil.rmtree(tmp) 812 813 def _fix_file_links(self, links): 814 for link in links: 815 if link.startswith('file://') and link[-1] != '/': 816 if os.path.isdir(link[7:]): 817 # work around excessive restriction in setuptools: 818 link += '/' 819 yield link 820 821 def _log_requirement(self, ws, req): 822 if (not logger.isEnabledFor(logging.DEBUG) and 823 not Installer._store_required_by): 824 # Sorting the working set and iterating over it's requirements 825 # is expensive, so short circuit the work if it won't even be 826 # logged. When profiling a simple buildout with 10 parts with 827 # identical and large working sets, this resulted in a 828 # decrease of run time from 93.411 to 15.068 seconds, about a 829 # 6 fold improvement. 830 return 831 832 ws = list(ws) 833 ws.sort() 834 for dist in ws: 835 if req in dist.requires(): 836 logger.debug(" required by %s." % dist) 837 req_ = str(req) 838 if req_ not in Installer._required_by: 839 Installer._required_by[req_] = set() 840 Installer._required_by[req_].add(str(dist.as_requirement())) 841 842 def _final_version(self, parsed_version): 843 return not parsed_version.is_prerelease 844 845 846def normalize_versions(versions): 847 """Return version dict with keys normalized to lowercase. 848 849 PyPI is case-insensitive and not all distributions are consistent in 850 their own naming. 851 """ 852 return dict([(k.lower(), v) for (k, v) in versions.items()]) 853 854 855def default_versions(versions=None): 856 old = Installer._versions 857 if versions is not None: 858 Installer._versions = normalize_versions(versions) 859 return old 860 861def download_cache(path=-1): 862 old = Installer._download_cache 863 if path != -1: 864 if path: 865 path = realpath(path) 866 Installer._download_cache = path 867 return old 868 869def install_from_cache(setting=None): 870 old = Installer._install_from_cache 871 if setting is not None: 872 Installer._install_from_cache = bool(setting) 873 return old 874 875def prefer_final(setting=None): 876 old = Installer._prefer_final 877 if setting is not None: 878 Installer._prefer_final = bool(setting) 879 return old 880 881def use_dependency_links(setting=None): 882 old = Installer._use_dependency_links 883 if setting is not None: 884 Installer._use_dependency_links = bool(setting) 885 return old 886 887def allow_picked_versions(setting=None): 888 old = Installer._allow_picked_versions 889 if setting is not None: 890 Installer._allow_picked_versions = bool(setting) 891 return old 892 893def store_required_by(setting=None): 894 old = Installer._store_required_by 895 if setting is not None: 896 Installer._store_required_by = bool(setting) 897 return old 898 899def get_picked_versions(): 900 picked_versions = sorted(Installer._picked_versions.items()) 901 required_by = Installer._required_by 902 return (picked_versions, required_by) 903 904 905def install(specs, dest, 906 links=(), index=None, 907 executable=sys.executable, 908 always_unzip=None, # Backward compat :/ 909 path=None, working_set=None, newest=True, versions=None, 910 use_dependency_links=None, allow_hosts=('*',), 911 include_site_packages=None, 912 allowed_eggs_from_site_packages=None, 913 check_picked=True, 914 ): 915 assert executable == sys.executable, (executable, sys.executable) 916 assert include_site_packages is None 917 assert allowed_eggs_from_site_packages is None 918 919 installer = Installer(dest, links, index, sys.executable, 920 always_unzip, path, 921 newest, versions, use_dependency_links, 922 allow_hosts=allow_hosts, 923 check_picked=check_picked) 924 return installer.install(specs, working_set) 925 926buildout_and_setuptools_dists = list(install(['zc.buildout'], None, 927 check_picked=False)) 928buildout_and_setuptools_path = [d.location 929 for d in buildout_and_setuptools_dists] 930setuptools_path = [d.location 931 for d in install(['setuptools'], None, check_picked=False)] 932setuptools_pythonpath = os.pathsep.join(setuptools_path) 933 934def build(spec, dest, build_ext, 935 links=(), index=None, 936 executable=sys.executable, 937 path=None, newest=True, versions=None, allow_hosts=('*',)): 938 assert executable == sys.executable, (executable, sys.executable) 939 installer = Installer(dest, links, index, executable, 940 True, path, newest, 941 versions, allow_hosts=allow_hosts) 942 return installer.build(spec, build_ext) 943 944 945def _rm(*paths): 946 for path in paths: 947 if os.path.isdir(path): 948 shutil.rmtree(path) 949 elif os.path.exists(path): 950 os.remove(path) 951 952 953def _copyeggs(src, dest, suffix, undo): 954 result = [] 955 undo.append(lambda : _rm(*result)) 956 for name in os.listdir(src): 957 if name.endswith(suffix): 958 new = os.path.join(dest, name) 959 _rm(new) 960 os.rename(os.path.join(src, name), new) 961 result.append(new) 962 963 assert len(result) == 1, str(result) 964 undo.pop() 965 966 return result[0] 967 968 969_develop_distutils_scripts = {} 970 971 972def _detect_distutils_scripts(directory): 973 """Record detected distutils scripts from develop eggs 974 975 ``setup.py develop`` doesn't generate metadata on distutils scripts, in 976 contrast to ``setup.py install``. So we have to store the information for 977 later. 978 979 """ 980 dir_contents = os.listdir(directory) 981 egginfo_filenames = [filename for filename in dir_contents 982 if filename.endswith('.egg-link')] 983 if not egginfo_filenames: 984 return 985 egg_name = egginfo_filenames[0].replace('.egg-link', '') 986 marker = 'EASY-INSTALL-DEV-SCRIPT' 987 scripts_found = [] 988 for filename in dir_contents: 989 if filename.endswith('.exe'): 990 continue 991 filepath = os.path.join(directory, filename) 992 if not os.path.isfile(filepath): 993 continue 994 with open(filepath) as fp: 995 dev_script_content = fp.read() 996 if marker in dev_script_content: 997 # The distutils bin script points at the actual file we need. 998 for line in dev_script_content.splitlines(): 999 match = DUNDER_FILE_PATTERN.search(line) 1000 if match: 1001 # The ``__file__ =`` line in the generated script points 1002 # at the actual distutils script we need. 1003 actual_script_filename = match.group('filename') 1004 with open(actual_script_filename) as fp: 1005 actual_script_content = fp.read() 1006 scripts_found.append([filename, actual_script_content]) 1007 1008 if scripts_found: 1009 logger.debug( 1010 "Distutils scripts found for develop egg %s: %s", 1011 egg_name, scripts_found) 1012 _develop_distutils_scripts[egg_name] = scripts_found 1013 1014 1015def develop(setup, dest, 1016 build_ext=None, 1017 executable=sys.executable): 1018 assert executable == sys.executable, (executable, sys.executable) 1019 if os.path.isdir(setup): 1020 directory = setup 1021 setup = os.path.join(directory, 'setup.py') 1022 else: 1023 directory = os.path.dirname(setup) 1024 1025 undo = [] 1026 try: 1027 if build_ext: 1028 setup_cfg = os.path.join(directory, 'setup.cfg') 1029 if os.path.exists(setup_cfg): 1030 os.rename(setup_cfg, setup_cfg+'-develop-aside') 1031 def restore_old_setup(): 1032 if os.path.exists(setup_cfg): 1033 os.remove(setup_cfg) 1034 os.rename(setup_cfg+'-develop-aside', setup_cfg) 1035 undo.append(restore_old_setup) 1036 else: 1037 f = open(setup_cfg, 'w') 1038 f.close() 1039 undo.append(lambda: os.remove(setup_cfg)) 1040 setuptools.command.setopt.edit_config( 1041 setup_cfg, dict(build_ext=build_ext)) 1042 1043 fd, tsetup = tempfile.mkstemp() 1044 undo.append(lambda: os.remove(tsetup)) 1045 undo.append(lambda: os.close(fd)) 1046 1047 os.write(fd, (runsetup_template % dict( 1048 setupdir=directory, 1049 setup=setup, 1050 __file__ = setup, 1051 )).encode()) 1052 1053 tmp3 = tempfile.mkdtemp('build', dir=dest) 1054 undo.append(lambda : shutil.rmtree(tmp3)) 1055 1056 args = [executable, tsetup, '-q', 'develop', '-mN', '-d', tmp3] 1057 1058 log_level = logger.getEffectiveLevel() 1059 if log_level <= 0: 1060 if log_level == 0: 1061 del args[2] 1062 else: 1063 args[2] == '-v' 1064 if log_level < logging.DEBUG: 1065 logger.debug("in: %r\n%s", directory, ' '.join(args)) 1066 1067 call_subprocess(args) 1068 _detect_distutils_scripts(tmp3) 1069 return _copyeggs(tmp3, dest, '.egg-link', undo) 1070 1071 finally: 1072 undo.reverse() 1073 [f() for f in undo] 1074 1075 1076def working_set(specs, executable, path=None, 1077 include_site_packages=None, 1078 allowed_eggs_from_site_packages=None): 1079 # Backward compat: 1080 if path is None: 1081 path = executable 1082 else: 1083 assert executable == sys.executable, (executable, sys.executable) 1084 assert include_site_packages is None 1085 assert allowed_eggs_from_site_packages is None 1086 1087 return install(specs, None, path=path) 1088 1089 1090 1091def scripts(reqs, working_set, executable, dest=None, 1092 scripts=None, 1093 extra_paths=(), 1094 arguments='', 1095 interpreter=None, 1096 initialization='', 1097 relative_paths=False, 1098 ): 1099 assert executable == sys.executable, (executable, sys.executable) 1100 1101 path = [dist.location for dist in working_set] 1102 path.extend(extra_paths) 1103 # order preserving unique 1104 unique_path = [] 1105 for p in path: 1106 if p not in unique_path: 1107 unique_path.append(p) 1108 path = list(map(realpath, unique_path)) 1109 1110 generated = [] 1111 1112 if isinstance(reqs, str): 1113 raise TypeError('Expected iterable of requirements or entry points,' 1114 ' got string.') 1115 1116 if initialization: 1117 initialization = '\n'+initialization+'\n' 1118 1119 entry_points = [] 1120 distutils_scripts = [] 1121 for req in reqs: 1122 if isinstance(req, str): 1123 req = pkg_resources.Requirement.parse(req) 1124 dist = working_set.find(req) 1125 # regular console_scripts entry points 1126 for name in pkg_resources.get_entry_map(dist, 'console_scripts'): 1127 entry_point = dist.get_entry_info('console_scripts', name) 1128 entry_points.append( 1129 (name, entry_point.module_name, 1130 '.'.join(entry_point.attrs)) 1131 ) 1132 # The metadata on "old-style" distutils scripts is not retained by 1133 # distutils/setuptools, except by placing the original scripts in 1134 # /EGG-INFO/scripts/. 1135 if dist.metadata_isdir('scripts'): 1136 # egg-info metadata from installed egg. 1137 for name in dist.metadata_listdir('scripts'): 1138 if dist.metadata_isdir('scripts/' + name): 1139 # Probably Python 3 __pycache__ directory. 1140 continue 1141 if name.lower().endswith('.exe'): 1142 # windows: scripts are implemented with 2 files 1143 # the .exe gets also into metadata_listdir 1144 # get_metadata chokes on the binary 1145 continue 1146 contents = dist.get_metadata('scripts/' + name) 1147 distutils_scripts.append((name, contents)) 1148 elif dist.key in _develop_distutils_scripts: 1149 # Development eggs don't have metadata about scripts, so we 1150 # collected it ourselves in develop()/ and 1151 # _detect_distutils_scripts(). 1152 for name, contents in _develop_distutils_scripts[dist.key]: 1153 distutils_scripts.append((name, contents)) 1154 1155 else: 1156 entry_points.append(req) 1157 1158 for name, module_name, attrs in entry_points: 1159 if scripts is not None: 1160 sname = scripts.get(name) 1161 if sname is None: 1162 continue 1163 else: 1164 sname = name 1165 1166 sname = os.path.join(dest, sname) 1167 spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths) 1168 1169 generated.extend( 1170 _script(module_name, attrs, spath, sname, arguments, 1171 initialization, rpsetup) 1172 ) 1173 1174 for name, contents in distutils_scripts: 1175 if scripts is not None: 1176 sname = scripts.get(name) 1177 if sname is None: 1178 continue 1179 else: 1180 sname = name 1181 1182 sname = os.path.join(dest, sname) 1183 spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths) 1184 1185 generated.extend( 1186 _distutils_script(spath, sname, contents, initialization, rpsetup) 1187 ) 1188 1189 if interpreter: 1190 sname = os.path.join(dest, interpreter) 1191 spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths) 1192 generated.extend(_pyscript(spath, sname, rpsetup, initialization)) 1193 1194 return generated 1195 1196 1197def _relative_path_and_setup(sname, path, relative_paths): 1198 if relative_paths: 1199 relative_paths = os.path.normcase(relative_paths) 1200 sname = os.path.normcase(os.path.abspath(sname)) 1201 spath = ',\n '.join( 1202 [_relativitize(os.path.normcase(path_item), sname, relative_paths) 1203 for path_item in path] 1204 ) 1205 rpsetup = relative_paths_setup 1206 for i in range(_relative_depth(relative_paths, sname)): 1207 rpsetup += "base = os.path.dirname(base)\n" 1208 else: 1209 spath = repr(path)[1:-1].replace(', ', ',\n ') 1210 rpsetup = '' 1211 return spath, rpsetup 1212 1213 1214def _relative_depth(common, path): 1215 n = 0 1216 while 1: 1217 dirname = os.path.dirname(path) 1218 if dirname == path: 1219 raise AssertionError("dirname of %s is the same" % dirname) 1220 if dirname == common: 1221 break 1222 n += 1 1223 path = dirname 1224 return n 1225 1226 1227def _relative_path(common, path): 1228 r = [] 1229 while 1: 1230 dirname, basename = os.path.split(path) 1231 r.append(basename) 1232 if dirname == common: 1233 break 1234 if dirname == path: 1235 raise AssertionError("dirname of %s is the same" % dirname) 1236 path = dirname 1237 r.reverse() 1238 return os.path.join(*r) 1239 1240 1241def _relativitize(path, script, relative_paths): 1242 if path == script: 1243 raise AssertionError("path == script") 1244 if path == relative_paths: 1245 return "base" 1246 common = os.path.dirname(os.path.commonprefix([path, script])) 1247 if (common == relative_paths or 1248 common.startswith(os.path.join(relative_paths, '')) 1249 ): 1250 return "join(base, %r)" % _relative_path(common, path) 1251 else: 1252 return repr(path) 1253 1254 1255relative_paths_setup = """ 1256import os 1257 1258join = os.path.join 1259base = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) 1260""" 1261 1262def _script(module_name, attrs, path, dest, arguments, initialization, rsetup): 1263 if is_win32: 1264 dest += '-script.py' 1265 1266 python = _safe_arg(sys.executable) 1267 1268 contents = script_template % dict( 1269 python = python, 1270 path = path, 1271 module_name = module_name, 1272 attrs = attrs, 1273 arguments = arguments, 1274 initialization = initialization, 1275 relative_paths_setup = rsetup, 1276 ) 1277 return _create_script(contents, dest) 1278 1279 1280def _distutils_script(path, dest, script_content, initialization, rsetup): 1281 if is_win32: 1282 dest += '-script.py' 1283 1284 lines = script_content.splitlines(True) 1285 if not ('#!' in lines[0]) and ('python' in lines[0]): 1286 # The script doesn't follow distutil's rules. Ignore it. 1287 return [] 1288 lines = lines[1:] # Strip off the first hashbang line. 1289 line_with_first_import = len(lines) 1290 for line_number, line in enumerate(lines): 1291 if not 'import' in line: 1292 continue 1293 if not (line.startswith('import') or line.startswith('from')): 1294 continue 1295 if '__future__' in line: 1296 continue 1297 line_with_first_import = line_number 1298 break 1299 1300 before = ''.join(lines[:line_with_first_import]) 1301 after = ''.join(lines[line_with_first_import:]) 1302 1303 python = _safe_arg(sys.executable) 1304 1305 contents = distutils_script_template % dict( 1306 python = python, 1307 path = path, 1308 initialization = initialization, 1309 relative_paths_setup = rsetup, 1310 before = before, 1311 after = after 1312 ) 1313 return _create_script(contents, dest) 1314 1315def _file_changed(filename, old_contents, mode='r'): 1316 try: 1317 with open(filename, mode) as f: 1318 return f.read() != old_contents 1319 except EnvironmentError as e: 1320 if e.errno == errno.ENOENT: 1321 return True 1322 else: 1323 raise 1324 1325def _create_script(contents, dest): 1326 generated = [] 1327 script = dest 1328 1329 changed = _file_changed(dest, contents) 1330 1331 if is_win32: 1332 # generate exe file and give the script a magic name: 1333 win32_exe = os.path.splitext(dest)[0] # remove ".py" 1334 if win32_exe.endswith('-script'): 1335 win32_exe = win32_exe[:-7] # remove "-script" 1336 win32_exe = win32_exe + '.exe' # add ".exe" 1337 try: 1338 new_data = setuptools.command.easy_install.get_win_launcher('cli') 1339 except AttributeError: 1340 # fall back for compatibility with older Distribute versions 1341 new_data = pkg_resources.resource_string('setuptools', 'cli.exe') 1342 1343 if _file_changed(win32_exe, new_data, 'rb'): 1344 # Only write it if it's different. 1345 with open(win32_exe, 'wb') as f: 1346 f.write(new_data) 1347 generated.append(win32_exe) 1348 1349 if changed: 1350 with open(dest, 'w') as f: 1351 f.write(contents) 1352 logger.info( 1353 "Generated script %r.", 1354 # Normalize for windows 1355 script.endswith('-script.py') and script[:-10] or script) 1356 1357 try: 1358 os.chmod(dest, _execute_permission()) 1359 except (AttributeError, os.error): 1360 pass 1361 1362 generated.append(dest) 1363 return generated 1364 1365 1366if is_jython and jython_os_name == 'linux': 1367 script_header = '#!/usr/bin/env %(python)s' 1368else: 1369 script_header = '#!%(python)s' 1370 1371 1372script_template = script_header + '''\ 1373 1374%(relative_paths_setup)s 1375import sys 1376sys.path[0:0] = [ 1377 %(path)s, 1378 ] 1379%(initialization)s 1380import %(module_name)s 1381 1382if __name__ == '__main__': 1383 sys.exit(%(module_name)s.%(attrs)s(%(arguments)s)) 1384''' 1385 1386distutils_script_template = script_header + ''' 1387%(before)s 1388%(relative_paths_setup)s 1389import sys 1390sys.path[0:0] = [ 1391 %(path)s, 1392 ] 1393%(initialization)s 1394 1395%(after)s''' 1396 1397 1398def _pyscript(path, dest, rsetup, initialization=''): 1399 generated = [] 1400 script = dest 1401 if is_win32: 1402 dest += '-script.py' 1403 1404 python = _safe_arg(sys.executable) 1405 if path: 1406 path += ',' # Courtesy comma at the end of the list. 1407 1408 contents = py_script_template % dict( 1409 python = python, 1410 path = path, 1411 relative_paths_setup = rsetup, 1412 initialization=initialization, 1413 ) 1414 changed = _file_changed(dest, contents) 1415 1416 if is_win32: 1417 # generate exe file and give the script a magic name: 1418 exe = script + '.exe' 1419 with open(exe, 'wb') as f: 1420 f.write( 1421 pkg_resources.resource_string('setuptools', 'cli.exe') 1422 ) 1423 generated.append(exe) 1424 1425 if changed: 1426 with open(dest, 'w') as f: 1427 f.write(contents) 1428 try: 1429 os.chmod(dest, _execute_permission()) 1430 except (AttributeError, os.error): 1431 pass 1432 logger.info("Generated interpreter %r.", script) 1433 1434 generated.append(dest) 1435 return generated 1436 1437py_script_template = script_header + '''\ 1438 1439%(relative_paths_setup)s 1440import sys 1441 1442sys.path[0:0] = [ 1443 %(path)s 1444 ] 1445%(initialization)s 1446 1447_interactive = True 1448if len(sys.argv) > 1: 1449 _options, _args = __import__("getopt").getopt(sys.argv[1:], 'ic:m:') 1450 _interactive = False 1451 for (_opt, _val) in _options: 1452 if _opt == '-i': 1453 _interactive = True 1454 elif _opt == '-c': 1455 exec(_val) 1456 elif _opt == '-m': 1457 sys.argv[1:] = _args 1458 _args = [] 1459 __import__("runpy").run_module( 1460 _val, {}, "__main__", alter_sys=True) 1461 1462 if _args: 1463 sys.argv[:] = _args 1464 __file__ = _args[0] 1465 del _options, _args 1466 with open(__file__, 'U') as __file__f: 1467 exec(compile(__file__f.read(), __file__, "exec")) 1468 1469if _interactive: 1470 del _interactive 1471 __import__("code").interact(banner="", local=globals()) 1472''' 1473 1474runsetup_template = """ 1475import sys 1476sys.path.insert(0, %%(setupdir)r) 1477sys.path[0:0] = %r 1478 1479import os, setuptools 1480 1481__file__ = %%(__file__)r 1482 1483os.chdir(%%(setupdir)r) 1484sys.argv[0] = %%(setup)r 1485 1486with open(%%(setup)r, 'U') as f: 1487 exec(compile(f.read(), %%(setup)r, 'exec')) 1488""" % setuptools_path 1489 1490 1491class VersionConflict(zc.buildout.UserError): 1492 1493 def __init__(self, err, ws): 1494 ws = list(ws) 1495 ws.sort() 1496 self.err, self.ws = err, ws 1497 1498 def __str__(self): 1499 result = ["There is a version conflict."] 1500 if len(self.err.args) == 2: 1501 existing_dist, req = self.err.args 1502 result.append("We already have: %s" % existing_dist) 1503 for dist in self.ws: 1504 if req in dist.requires(): 1505 result.append("but %s requires %r." % (dist, str(req))) 1506 else: 1507 # The error argument is already a nice error string. 1508 result.append(self.err.args[0]) 1509 return '\n'.join(result) 1510 1511 1512class MissingDistribution(zc.buildout.UserError): 1513 1514 def __init__(self, req, ws): 1515 ws = list(ws) 1516 ws.sort() 1517 self.data = req, ws 1518 1519 def __str__(self): 1520 req, ws = self.data 1521 return "Couldn't find a distribution for %r." % str(req) 1522 1523def redo_pyc(egg): 1524 if not os.path.isdir(egg): 1525 return 1526 for dirpath, dirnames, filenames in os.walk(egg): 1527 for filename in filenames: 1528 if not filename.endswith('.py'): 1529 continue 1530 filepath = os.path.join(dirpath, filename) 1531 if not (os.path.exists(filepath+'c') 1532 or os.path.exists(filepath+'o')): 1533 # If it wasn't compiled, it may not be compilable 1534 continue 1535 1536 # OK, it looks like we should try to compile. 1537 1538 # Remove old files. 1539 for suffix in 'co': 1540 if os.path.exists(filepath+suffix): 1541 os.remove(filepath+suffix) 1542 1543 # Compile under current optimization 1544 try: 1545 py_compile.compile(filepath) 1546 except py_compile.PyCompileError: 1547 logger.warning("Couldn't compile %s", filepath) 1548 else: 1549 # Recompile under other optimization. :) 1550 args = [sys.executable] 1551 if __debug__: 1552 args.append('-O') 1553 args.extend(['-m', 'py_compile', filepath]) 1554 1555 call_subprocess(args) 1556 1557def _constrained_requirement(constraint, requirement): 1558 if constraint[0] not in '<>': 1559 if constraint.startswith('='): 1560 assert constraint.startswith('==') 1561 constraint = constraint[2:] 1562 if constraint not in requirement: 1563 msg = ("The requirement (%r) is not allowed by your [versions] " 1564 "constraint (%s)" % (str(requirement), constraint)) 1565 raise IncompatibleConstraintError(msg) 1566 1567 # Sigh, copied from Requirement.__str__ 1568 extras = ','.join(requirement.extras) 1569 if extras: 1570 extras = '[%s]' % extras 1571 return pkg_resources.Requirement.parse( 1572 "%s%s==%s" % (requirement.project_name, extras, constraint)) 1573 1574 if requirement.specs: 1575 return pkg_resources.Requirement.parse( 1576 str(requirement) + ',' + constraint 1577 ) 1578 else: 1579 return pkg_resources.Requirement.parse( 1580 str(requirement) + ' ' + constraint 1581 ) 1582 1583class IncompatibleConstraintError(zc.buildout.UserError): 1584 """A specified version is incompatible with a given requirement. 1585 """ 1586 1587IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility 1588 1589 1590def call_easy_install(spec, dest): 1591 """ 1592 Call `easy_install` from setuptools as a subprocess to install a 1593 distribution specified by `spec` into `dest`. 1594 Returns all the paths inside `dest` created by the above. 1595 """ 1596 path = setuptools_path 1597 1598 args = [sys.executable, '-c', 1599 ('import sys; sys.path[0:0] = %r; ' % path) + 1600 _easy_install_cmd, '-mZUNxd', dest] 1601 level = logger.getEffectiveLevel() 1602 if level > 0: 1603 args.append('-q') 1604 elif level < 0: 1605 args.append('-v') 1606 1607 args.append(spec) 1608 1609 if level <= logging.DEBUG: 1610 logger.debug('Running easy_install:\n"%s"\npath=%s\n', 1611 '" "'.join(args), path) 1612 1613 sys.stdout.flush() # We want any pending output first 1614 1615 exit_code = subprocess.call(list(args)) 1616 1617 if exit_code: 1618 logger.error( 1619 "An error occurred when trying to install %s. " 1620 "Look above this message for any errors that " 1621 "were output by easy_install.", 1622 spec) 1623 return glob.glob(os.path.join(dest, '*')) 1624 1625 1626def unpack_egg(location, dest): 1627 # Buildout 2 no longer installs zipped eggs, 1628 # so we always want to unpack it. 1629 dest = os.path.join(dest, os.path.basename(location)) 1630 setuptools.archive_util.unpack_archive(location, dest) 1631 1632 1633WHEEL_WARNING = """ 1634*.whl file detected (%s), you'll need setuptools >= 38.2.3 for that 1635or an extension like buildout.wheel > 0.2.0. 1636""" 1637 1638 1639def unpack_wheel(location, dest): 1640 if SETUPTOOLS_SUPPORTS_WHEELS: 1641 wheel = Wheel(location) 1642 wheel.install_as_egg(os.path.join(dest, wheel.egg_name())) 1643 else: 1644 raise zc.buildout.UserError(WHEEL_WARNING % location) 1645 1646 1647UNPACKERS = { 1648 '.egg': unpack_egg, 1649 '.whl': unpack_wheel, 1650} 1651 1652 1653def _get_matching_dist_in_location(dist, location): 1654 """ 1655 Check if `locations` contain only the one intended dist. 1656 Return the dist with metadata in the new location. 1657 """ 1658 # Getting the dist from the environment causes the 1659 # distribution meta data to be read. Cloning isn't 1660 # good enough. 1661 env = pkg_resources.Environment([location]) 1662 dists = [ d for project_name in env for d in env[project_name] ] 1663 dist_infos = [ (d.project_name.lower(), d.version) for d in dists ] 1664 if dist_infos == [(dist.project_name.lower(), dist.version)]: 1665 return dists.pop() 1666 1667def _move_to_eggs_dir_and_compile(dist, dest): 1668 """Move distribution to the eggs destination directory. 1669 1670 And compile the py files, if we have actually moved the dist. 1671 1672 Its new location is expected not to exist there yet, otherwise we 1673 would not be calling this function: the egg is already there. But 1674 the new location might exist at this point if another buildout is 1675 running in parallel. So we copy to a temporary directory first. 1676 See discussion at https://github.com/buildout/buildout/issues/307 1677 1678 We return the new distribution with properly loaded metadata. 1679 """ 1680 # First make sure the destination directory exists. This could suffer from 1681 # the same kind of race condition as the rest: if we check that it does not 1682 # exist, and we then create it, it will fail when a second buildout is 1683 # doing the same thing. 1684 try: 1685 os.makedirs(dest) 1686 except OSError: 1687 if not os.path.isdir(dest): 1688 # Unknown reason. Reraise original error. 1689 raise 1690 tmp_dest = tempfile.mkdtemp(dir=dest) 1691 try: 1692 if (os.path.isdir(dist.location) and 1693 dist.precedence >= pkg_resources.BINARY_DIST): 1694 # We got a pre-built directory. It must have been obtained locally. 1695 # Just copy it. 1696 tmp_loc = os.path.join(tmp_dest, os.path.basename(dist.location)) 1697 shutil.copytree(dist.location, tmp_loc) 1698 else: 1699 # It is an archive of some sort. 1700 # Figure out how to unpack it, or fall back to easy_install. 1701 _, ext = os.path.splitext(dist.location) 1702 unpacker = UNPACKERS.get(ext, call_easy_install) 1703 unpacker(dist.location, tmp_dest) 1704 [tmp_loc] = glob.glob(os.path.join(tmp_dest, '*')) 1705 1706 # We have installed the dist. Now try to rename/move it. 1707 newloc = os.path.join(dest, os.path.basename(tmp_loc)) 1708 try: 1709 os.rename(tmp_loc, newloc) 1710 except OSError: 1711 # Might be for various reasons. If it is because newloc already 1712 # exists, we can investigate. 1713 if not os.path.exists(newloc): 1714 # No, it is a different reason. Give up. 1715 raise 1716 # Try to use it as environment and check if our project is in it. 1717 newdist = _get_matching_dist_in_location(dist, newloc) 1718 if newdist is None: 1719 # Path exists, but is not our package. We could 1720 # try something, but it seems safer to bail out 1721 # with the original error. 1722 raise 1723 # newloc looks okay to use. Do print a warning. 1724 logger.warn( 1725 "Path %s unexpectedly already exists.\n" 1726 "Maybe a buildout running in parallel has added it. " 1727 "We will accept it.\n" 1728 "If this contains a wrong package, please remove it yourself.", 1729 newloc) 1730 else: 1731 # There were no problems during the rename. 1732 # Do the compile step. 1733 redo_pyc(newloc) 1734 newdist = _get_matching_dist_in_location(dist, newloc) 1735 assert newdist is not None # newloc above is missing our dist?! 1736 finally: 1737 # Remember that temporary directories must be removed 1738 shutil.rmtree(tmp_dest) 1739 return newdist 1740