1from __future__ import absolute_import 2 3import errno 4import logging 5import operator 6import os 7import shutil 8import site 9from optparse import SUPPRESS_HELP 10 11from pip._vendor import pkg_resources 12from pip._vendor.packaging.utils import canonicalize_name 13 14from pip._internal.cache import WheelCache 15from pip._internal.cli import cmdoptions 16from pip._internal.cli.cmdoptions import make_target_python 17from pip._internal.cli.req_command import RequirementCommand, with_cleanup 18from pip._internal.cli.status_codes import ERROR, SUCCESS 19from pip._internal.exceptions import CommandError, InstallationError 20from pip._internal.locations import distutils_scheme 21from pip._internal.operations.check import check_install_conflicts 22from pip._internal.req import install_given_reqs 23from pip._internal.req.req_tracker import get_requirement_tracker 24from pip._internal.utils.datetime import today_is_later_than 25from pip._internal.utils.deprecation import deprecated 26from pip._internal.utils.distutils_args import parse_distutils_args 27from pip._internal.utils.filesystem import test_writable_dir 28from pip._internal.utils.misc import ( 29 ensure_dir, 30 get_installed_version, 31 get_pip_version, 32 protect_pip_from_modification_on_windows, 33 write_output, 34) 35from pip._internal.utils.temp_dir import TempDirectory 36from pip._internal.utils.typing import MYPY_CHECK_RUNNING 37from pip._internal.utils.virtualenv import virtualenv_no_global 38from pip._internal.wheel_builder import build, should_build_for_install_command 39 40if MYPY_CHECK_RUNNING: 41 from optparse import Values 42 from typing import Iterable, List, Optional 43 44 from pip._internal.models.format_control import FormatControl 45 from pip._internal.operations.check import ConflictDetails 46 from pip._internal.req.req_install import InstallRequirement 47 from pip._internal.wheel_builder import BinaryAllowedPredicate 48 49 50logger = logging.getLogger(__name__) 51 52 53def get_check_binary_allowed(format_control): 54 # type: (FormatControl) -> BinaryAllowedPredicate 55 def check_binary_allowed(req): 56 # type: (InstallRequirement) -> bool 57 if req.use_pep517: 58 return True 59 canonical_name = canonicalize_name(req.name) 60 allowed_formats = format_control.get_allowed_formats(canonical_name) 61 return "binary" in allowed_formats 62 63 return check_binary_allowed 64 65 66class InstallCommand(RequirementCommand): 67 """ 68 Install packages from: 69 70 - PyPI (and other indexes) using requirement specifiers. 71 - VCS project urls. 72 - Local project directories. 73 - Local or remote source archives. 74 75 pip also supports installing from "requirements files", which provide 76 an easy way to specify a whole environment to be installed. 77 """ 78 79 usage = """ 80 %prog [options] <requirement specifier> [package-index-options] ... 81 %prog [options] -r <requirements file> [package-index-options] ... 82 %prog [options] [-e] <vcs project url> ... 83 %prog [options] [-e] <local project path> ... 84 %prog [options] <archive url/path> ...""" 85 86 def add_options(self): 87 # type: () -> None 88 self.cmd_opts.add_option(cmdoptions.requirements()) 89 self.cmd_opts.add_option(cmdoptions.constraints()) 90 self.cmd_opts.add_option(cmdoptions.no_deps()) 91 self.cmd_opts.add_option(cmdoptions.pre()) 92 93 self.cmd_opts.add_option(cmdoptions.editable()) 94 self.cmd_opts.add_option( 95 '-t', '--target', 96 dest='target_dir', 97 metavar='dir', 98 default=None, 99 help='Install packages into <dir>. ' 100 'By default this will not replace existing files/folders in ' 101 '<dir>. Use --upgrade to replace existing packages in <dir> ' 102 'with new versions.' 103 ) 104 cmdoptions.add_target_python_options(self.cmd_opts) 105 106 self.cmd_opts.add_option( 107 '--user', 108 dest='use_user_site', 109 action='store_true', 110 help="Install to the Python user install directory for your " 111 "platform. Typically ~/.local/, or %APPDATA%\\Python on " 112 "Windows. (See the Python documentation for site.USER_BASE " 113 "for full details.)") 114 self.cmd_opts.add_option( 115 '--no-user', 116 dest='use_user_site', 117 action='store_false', 118 help=SUPPRESS_HELP) 119 self.cmd_opts.add_option( 120 '--root', 121 dest='root_path', 122 metavar='dir', 123 default=None, 124 help="Install everything relative to this alternate root " 125 "directory.") 126 self.cmd_opts.add_option( 127 '--prefix', 128 dest='prefix_path', 129 metavar='dir', 130 default=None, 131 help="Installation prefix where lib, bin and other top-level " 132 "folders are placed") 133 134 self.cmd_opts.add_option(cmdoptions.build_dir()) 135 136 self.cmd_opts.add_option(cmdoptions.src()) 137 138 self.cmd_opts.add_option( 139 '-U', '--upgrade', 140 dest='upgrade', 141 action='store_true', 142 help='Upgrade all specified packages to the newest available ' 143 'version. The handling of dependencies depends on the ' 144 'upgrade-strategy used.' 145 ) 146 147 self.cmd_opts.add_option( 148 '--upgrade-strategy', 149 dest='upgrade_strategy', 150 default='only-if-needed', 151 choices=['only-if-needed', 'eager'], 152 help='Determines how dependency upgrading should be handled ' 153 '[default: %default]. ' 154 '"eager" - dependencies are upgraded regardless of ' 155 'whether the currently installed version satisfies the ' 156 'requirements of the upgraded package(s). ' 157 '"only-if-needed" - are upgraded only when they do not ' 158 'satisfy the requirements of the upgraded package(s).' 159 ) 160 161 self.cmd_opts.add_option( 162 '--force-reinstall', 163 dest='force_reinstall', 164 action='store_true', 165 help='Reinstall all packages even if they are already ' 166 'up-to-date.') 167 168 self.cmd_opts.add_option( 169 '-I', '--ignore-installed', 170 dest='ignore_installed', 171 action='store_true', 172 help='Ignore the installed packages, overwriting them. ' 173 'This can break your system if the existing package ' 174 'is of a different version or was installed ' 175 'with a different package manager!' 176 ) 177 178 self.cmd_opts.add_option(cmdoptions.ignore_requires_python()) 179 self.cmd_opts.add_option(cmdoptions.no_build_isolation()) 180 self.cmd_opts.add_option(cmdoptions.use_pep517()) 181 self.cmd_opts.add_option(cmdoptions.no_use_pep517()) 182 183 self.cmd_opts.add_option(cmdoptions.install_options()) 184 self.cmd_opts.add_option(cmdoptions.global_options()) 185 186 self.cmd_opts.add_option( 187 "--compile", 188 action="store_true", 189 dest="compile", 190 default=True, 191 help="Compile Python source files to bytecode", 192 ) 193 194 self.cmd_opts.add_option( 195 "--no-compile", 196 action="store_false", 197 dest="compile", 198 help="Do not compile Python source files to bytecode", 199 ) 200 201 self.cmd_opts.add_option( 202 "--no-warn-script-location", 203 action="store_false", 204 dest="warn_script_location", 205 default=True, 206 help="Do not warn when installing scripts outside PATH", 207 ) 208 self.cmd_opts.add_option( 209 "--no-warn-conflicts", 210 action="store_false", 211 dest="warn_about_conflicts", 212 default=True, 213 help="Do not warn about broken dependencies", 214 ) 215 216 self.cmd_opts.add_option(cmdoptions.no_binary()) 217 self.cmd_opts.add_option(cmdoptions.only_binary()) 218 self.cmd_opts.add_option(cmdoptions.prefer_binary()) 219 self.cmd_opts.add_option(cmdoptions.require_hashes()) 220 self.cmd_opts.add_option(cmdoptions.progress_bar()) 221 222 index_opts = cmdoptions.make_option_group( 223 cmdoptions.index_group, 224 self.parser, 225 ) 226 227 self.parser.insert_option_group(0, index_opts) 228 self.parser.insert_option_group(0, self.cmd_opts) 229 230 @with_cleanup 231 def run(self, options, args): 232 # type: (Values, List[str]) -> int 233 if options.use_user_site and options.target_dir is not None: 234 raise CommandError("Can not combine '--user' and '--target'") 235 236 cmdoptions.check_install_build_global(options) 237 upgrade_strategy = "to-satisfy-only" 238 if options.upgrade: 239 upgrade_strategy = options.upgrade_strategy 240 241 cmdoptions.check_dist_restriction(options, check_target=True) 242 243 install_options = options.install_options or [] 244 245 logger.debug("Using %s", get_pip_version()) 246 options.use_user_site = decide_user_install( 247 options.use_user_site, 248 prefix_path=options.prefix_path, 249 target_dir=options.target_dir, 250 root_path=options.root_path, 251 isolated_mode=options.isolated_mode, 252 ) 253 254 target_temp_dir = None # type: Optional[TempDirectory] 255 target_temp_dir_path = None # type: Optional[str] 256 if options.target_dir: 257 options.ignore_installed = True 258 options.target_dir = os.path.abspath(options.target_dir) 259 if (os.path.exists(options.target_dir) and not 260 os.path.isdir(options.target_dir)): 261 raise CommandError( 262 "Target path exists but is not a directory, will not " 263 "continue." 264 ) 265 266 # Create a target directory for using with the target option 267 target_temp_dir = TempDirectory(kind="target") 268 target_temp_dir_path = target_temp_dir.path 269 self.enter_context(target_temp_dir) 270 271 global_options = options.global_options or [] 272 273 session = self.get_default_session(options) 274 275 target_python = make_target_python(options) 276 finder = self._build_package_finder( 277 options=options, 278 session=session, 279 target_python=target_python, 280 ignore_requires_python=options.ignore_requires_python, 281 ) 282 build_delete = (not (options.no_clean or options.build_dir)) 283 wheel_cache = WheelCache(options.cache_dir, options.format_control) 284 285 req_tracker = self.enter_context(get_requirement_tracker()) 286 287 directory = TempDirectory( 288 options.build_dir, 289 delete=build_delete, 290 kind="install", 291 globally_managed=True, 292 ) 293 294 try: 295 reqs = self.get_requirements(args, options, finder, session) 296 297 reject_location_related_install_options( 298 reqs, options.install_options 299 ) 300 301 preparer = self.make_requirement_preparer( 302 temp_build_dir=directory, 303 options=options, 304 req_tracker=req_tracker, 305 session=session, 306 finder=finder, 307 use_user_site=options.use_user_site, 308 ) 309 resolver = self.make_resolver( 310 preparer=preparer, 311 finder=finder, 312 options=options, 313 wheel_cache=wheel_cache, 314 use_user_site=options.use_user_site, 315 ignore_installed=options.ignore_installed, 316 ignore_requires_python=options.ignore_requires_python, 317 force_reinstall=options.force_reinstall, 318 upgrade_strategy=upgrade_strategy, 319 use_pep517=options.use_pep517, 320 ) 321 322 self.trace_basic_info(finder) 323 324 requirement_set = resolver.resolve( 325 reqs, check_supported_wheels=not options.target_dir 326 ) 327 328 try: 329 pip_req = requirement_set.get_requirement("pip") 330 except KeyError: 331 modifying_pip = False 332 else: 333 # If we're not replacing an already installed pip, 334 # we're not modifying it. 335 modifying_pip = pip_req.satisfied_by is None 336 protect_pip_from_modification_on_windows( 337 modifying_pip=modifying_pip 338 ) 339 340 check_binary_allowed = get_check_binary_allowed( 341 finder.format_control 342 ) 343 344 reqs_to_build = [ 345 r for r in requirement_set.requirements.values() 346 if should_build_for_install_command( 347 r, check_binary_allowed 348 ) 349 ] 350 351 _, build_failures = build( 352 reqs_to_build, 353 wheel_cache=wheel_cache, 354 build_options=[], 355 global_options=[], 356 ) 357 358 # If we're using PEP 517, we cannot do a direct install 359 # so we fail here. 360 pep517_build_failure_names = [ 361 r.name # type: ignore 362 for r in build_failures if r.use_pep517 363 ] # type: List[str] 364 if pep517_build_failure_names: 365 raise InstallationError( 366 "Could not build wheels for {} which use" 367 " PEP 517 and cannot be installed directly".format( 368 ", ".join(pep517_build_failure_names) 369 ) 370 ) 371 372 # For now, we just warn about failures building legacy 373 # requirements, as we'll fall through to a direct 374 # install for those. 375 legacy_build_failure_names = [ 376 r.name # type: ignore 377 for r in build_failures if not r.use_pep517 378 ] # type: List[str] 379 if legacy_build_failure_names: 380 deprecated( 381 reason=( 382 "Could not build wheels for {} which do not use " 383 "PEP 517. pip will fall back to legacy 'setup.py " 384 "install' for these.".format( 385 ", ".join(legacy_build_failure_names) 386 ) 387 ), 388 replacement="to fix the wheel build issue reported above", 389 gone_in="21.0", 390 issue=8368, 391 ) 392 393 to_install = resolver.get_installation_order( 394 requirement_set 395 ) 396 397 # Check for conflicts in the package set we're installing. 398 conflicts = None # type: Optional[ConflictDetails] 399 should_warn_about_conflicts = ( 400 not options.ignore_dependencies and 401 options.warn_about_conflicts 402 ) 403 if should_warn_about_conflicts: 404 conflicts = self._determine_conflicts(to_install) 405 406 # Don't warn about script install locations if 407 # --target has been specified 408 warn_script_location = options.warn_script_location 409 if options.target_dir: 410 warn_script_location = False 411 412 installed = install_given_reqs( 413 to_install, 414 install_options, 415 global_options, 416 root=options.root_path, 417 home=target_temp_dir_path, 418 prefix=options.prefix_path, 419 warn_script_location=warn_script_location, 420 use_user_site=options.use_user_site, 421 pycompile=options.compile, 422 ) 423 424 lib_locations = get_lib_location_guesses( 425 user=options.use_user_site, 426 home=target_temp_dir_path, 427 root=options.root_path, 428 prefix=options.prefix_path, 429 isolated=options.isolated_mode, 430 ) 431 working_set = pkg_resources.WorkingSet(lib_locations) 432 433 installed.sort(key=operator.attrgetter('name')) 434 items = [] 435 for result in installed: 436 item = result.name 437 try: 438 installed_version = get_installed_version( 439 result.name, working_set=working_set 440 ) 441 if installed_version: 442 item += '-' + installed_version 443 except Exception: 444 pass 445 items.append(item) 446 447 if conflicts is not None: 448 self._warn_about_conflicts( 449 conflicts, 450 new_resolver='2020-resolver' in options.features_enabled, 451 ) 452 453 installed_desc = ' '.join(items) 454 if installed_desc: 455 write_output( 456 'Successfully installed %s', installed_desc, 457 ) 458 except EnvironmentError as error: 459 show_traceback = (self.verbosity >= 1) 460 461 message = create_env_error_message( 462 error, show_traceback, options.use_user_site, 463 ) 464 logger.error(message, exc_info=show_traceback) # noqa 465 466 return ERROR 467 468 if options.target_dir: 469 assert target_temp_dir 470 self._handle_target_dir( 471 options.target_dir, target_temp_dir, options.upgrade 472 ) 473 474 return SUCCESS 475 476 def _handle_target_dir(self, target_dir, target_temp_dir, upgrade): 477 # type: (str, TempDirectory, bool) -> None 478 ensure_dir(target_dir) 479 480 # Checking both purelib and platlib directories for installed 481 # packages to be moved to target directory 482 lib_dir_list = [] 483 484 # Checking both purelib and platlib directories for installed 485 # packages to be moved to target directory 486 scheme = distutils_scheme('', home=target_temp_dir.path) 487 purelib_dir = scheme['purelib'] 488 platlib_dir = scheme['platlib'] 489 data_dir = scheme['data'] 490 491 if os.path.exists(purelib_dir): 492 lib_dir_list.append(purelib_dir) 493 if os.path.exists(platlib_dir) and platlib_dir != purelib_dir: 494 lib_dir_list.append(platlib_dir) 495 if os.path.exists(data_dir): 496 lib_dir_list.append(data_dir) 497 498 for lib_dir in lib_dir_list: 499 for item in os.listdir(lib_dir): 500 if lib_dir == data_dir: 501 ddir = os.path.join(data_dir, item) 502 if any(s.startswith(ddir) for s in lib_dir_list[:-1]): 503 continue 504 target_item_dir = os.path.join(target_dir, item) 505 if os.path.exists(target_item_dir): 506 if not upgrade: 507 logger.warning( 508 'Target directory %s already exists. Specify ' 509 '--upgrade to force replacement.', 510 target_item_dir 511 ) 512 continue 513 if os.path.islink(target_item_dir): 514 logger.warning( 515 'Target directory %s already exists and is ' 516 'a link. pip will not automatically replace ' 517 'links, please remove if replacement is ' 518 'desired.', 519 target_item_dir 520 ) 521 continue 522 if os.path.isdir(target_item_dir): 523 shutil.rmtree(target_item_dir) 524 else: 525 os.remove(target_item_dir) 526 527 shutil.move( 528 os.path.join(lib_dir, item), 529 target_item_dir 530 ) 531 532 def _determine_conflicts(self, to_install): 533 # type: (List[InstallRequirement]) -> Optional[ConflictDetails] 534 try: 535 return check_install_conflicts(to_install) 536 except Exception: 537 logger.exception( 538 "Error while checking for conflicts. Please file an issue on " 539 "pip's issue tracker: https://github.com/pypa/pip/issues/new" 540 ) 541 return None 542 543 def _warn_about_conflicts(self, conflict_details, new_resolver): 544 # type: (ConflictDetails, bool) -> None 545 package_set, (missing, conflicting) = conflict_details 546 if not missing and not conflicting: 547 return 548 549 parts = [] # type: List[str] 550 if not new_resolver: 551 parts.append( 552 "After October 2020 you may experience errors when installing " 553 "or updating packages. This is because pip will change the " 554 "way that it resolves dependency conflicts.\n" 555 ) 556 parts.append( 557 "We recommend you use --use-feature=2020-resolver to test " 558 "your packages with the new resolver before it becomes the " 559 "default.\n" 560 ) 561 elif not today_is_later_than(year=2020, month=7, day=31): 562 # NOTE: trailing newlines here are intentional 563 parts.append( 564 "Pip will install or upgrade your package(s) and its " 565 "dependencies without taking into account other packages you " 566 "already have installed. This may cause an uncaught " 567 "dependency conflict.\n" 568 ) 569 form_link = "https://forms.gle/cWKMoDs8sUVE29hz9" 570 parts.append( 571 "If you would like pip to take your other packages into " 572 "account, please tell us here: {}\n".format(form_link) 573 ) 574 575 # NOTE: There is some duplication here, with commands/check.py 576 for project_name in missing: 577 version = package_set[project_name][0] 578 for dependency in missing[project_name]: 579 message = ( 580 "{name} {version} requires {requirement}, " 581 "which is not installed." 582 ).format( 583 name=project_name, 584 version=version, 585 requirement=dependency[1], 586 ) 587 parts.append(message) 588 589 for project_name in conflicting: 590 version = package_set[project_name][0] 591 for dep_name, dep_version, req in conflicting[project_name]: 592 message = ( 593 "{name} {version} requires {requirement}, but you'll have " 594 "{dep_name} {dep_version} which is incompatible." 595 ).format( 596 name=project_name, 597 version=version, 598 requirement=req, 599 dep_name=dep_name, 600 dep_version=dep_version, 601 ) 602 parts.append(message) 603 604 logger.critical("\n".join(parts)) 605 606 607def get_lib_location_guesses( 608 user=False, # type: bool 609 home=None, # type: Optional[str] 610 root=None, # type: Optional[str] 611 isolated=False, # type: bool 612 prefix=None # type: Optional[str] 613): 614 # type:(...) -> List[str] 615 scheme = distutils_scheme('', user=user, home=home, root=root, 616 isolated=isolated, prefix=prefix) 617 return [scheme['purelib'], scheme['platlib']] 618 619 620def site_packages_writable(root, isolated): 621 # type: (Optional[str], bool) -> bool 622 return all( 623 test_writable_dir(d) for d in set( 624 get_lib_location_guesses(root=root, isolated=isolated)) 625 ) 626 627 628def decide_user_install( 629 use_user_site, # type: Optional[bool] 630 prefix_path=None, # type: Optional[str] 631 target_dir=None, # type: Optional[str] 632 root_path=None, # type: Optional[str] 633 isolated_mode=False, # type: bool 634): 635 # type: (...) -> bool 636 """Determine whether to do a user install based on the input options. 637 638 If use_user_site is False, no additional checks are done. 639 If use_user_site is True, it is checked for compatibility with other 640 options. 641 If use_user_site is None, the default behaviour depends on the environment, 642 which is provided by the other arguments. 643 """ 644 # In some cases (config from tox), use_user_site can be set to an integer 645 # rather than a bool, which 'use_user_site is False' wouldn't catch. 646 if (use_user_site is not None) and (not use_user_site): 647 logger.debug("Non-user install by explicit request") 648 return False 649 650 if use_user_site: 651 if prefix_path: 652 raise CommandError( 653 "Can not combine '--user' and '--prefix' as they imply " 654 "different installation locations" 655 ) 656 if virtualenv_no_global(): 657 raise InstallationError( 658 "Can not perform a '--user' install. User site-packages " 659 "are not visible in this virtualenv." 660 ) 661 logger.debug("User install by explicit request") 662 return True 663 664 # If we are here, user installs have not been explicitly requested/avoided 665 assert use_user_site is None 666 667 # user install incompatible with --prefix/--target 668 if prefix_path or target_dir: 669 logger.debug("Non-user install due to --prefix or --target option") 670 return False 671 672 # If user installs are not enabled, choose a non-user install 673 if not site.ENABLE_USER_SITE: 674 logger.debug("Non-user install because user site-packages disabled") 675 return False 676 677 # If we have permission for a non-user install, do that, 678 # otherwise do a user install. 679 if site_packages_writable(root=root_path, isolated=isolated_mode): 680 logger.debug("Non-user install because site-packages writeable") 681 return False 682 683 logger.info("Defaulting to user installation because normal site-packages " 684 "is not writeable") 685 return True 686 687 688def reject_location_related_install_options(requirements, options): 689 # type: (List[InstallRequirement], Optional[List[str]]) -> None 690 """If any location-changing --install-option arguments were passed for 691 requirements or on the command-line, then show a deprecation warning. 692 """ 693 def format_options(option_names): 694 # type: (Iterable[str]) -> List[str] 695 return ["--{}".format(name.replace("_", "-")) for name in option_names] 696 697 offenders = [] 698 699 for requirement in requirements: 700 install_options = requirement.install_options 701 location_options = parse_distutils_args(install_options) 702 if location_options: 703 offenders.append( 704 "{!r} from {}".format( 705 format_options(location_options.keys()), requirement 706 ) 707 ) 708 709 if options: 710 location_options = parse_distutils_args(options) 711 if location_options: 712 offenders.append( 713 "{!r} from command line".format( 714 format_options(location_options.keys()) 715 ) 716 ) 717 718 if not offenders: 719 return 720 721 raise CommandError( 722 "Location-changing options found in --install-option: {}." 723 " This is unsupported, use pip-level options like --user," 724 " --prefix, --root, and --target instead.".format( 725 "; ".join(offenders) 726 ) 727 ) 728 729 730def create_env_error_message(error, show_traceback, using_user_site): 731 # type: (EnvironmentError, bool, bool) -> str 732 """Format an error message for an EnvironmentError 733 734 It may occur anytime during the execution of the install command. 735 """ 736 parts = [] 737 738 # Mention the error if we are not going to show a traceback 739 parts.append("Could not install packages due to an EnvironmentError") 740 if not show_traceback: 741 parts.append(": ") 742 parts.append(str(error)) 743 else: 744 parts.append(".") 745 746 # Spilt the error indication from a helper message (if any) 747 parts[-1] += "\n" 748 749 # Suggest useful actions to the user: 750 # (1) using user site-packages or (2) verifying the permissions 751 if error.errno == errno.EACCES: 752 user_option_part = "Consider using the `--user` option" 753 permissions_part = "Check the permissions" 754 755 if not using_user_site: 756 parts.extend([ 757 user_option_part, " or ", 758 permissions_part.lower(), 759 ]) 760 else: 761 parts.append(permissions_part) 762 parts.append(".\n") 763 764 return "".join(parts).strip() + "\n" 765