1from __future__ import absolute_import 2 3import errno 4import logging 5import operator 6import os 7import shutil 8import site 9from optparse import SUPPRESS_HELP 10 11from pip._vendor import pkg_resources 12from pip._vendor.packaging.utils import canonicalize_name 13 14from pip._internal.cache import WheelCache 15from pip._internal.cli import cmdoptions 16from pip._internal.cli.cmdoptions import make_target_python 17from pip._internal.cli.req_command import RequirementCommand, with_cleanup 18from pip._internal.cli.status_codes import ERROR, SUCCESS 19from pip._internal.exceptions import CommandError, InstallationError 20from pip._internal.locations import distutils_scheme 21from pip._internal.operations.check import check_install_conflicts 22from pip._internal.req import install_given_reqs 23from pip._internal.req.req_tracker import get_requirement_tracker 24from pip._internal.utils.distutils_args import parse_distutils_args 25from pip._internal.utils.filesystem import test_writable_dir 26from pip._internal.utils.misc import ( 27 ensure_dir, 28 get_installed_version, 29 get_pip_version, 30 protect_pip_from_modification_on_windows, 31 write_output, 32) 33from pip._internal.utils.temp_dir import TempDirectory 34from pip._internal.utils.typing import MYPY_CHECK_RUNNING 35from pip._internal.utils.virtualenv import virtualenv_no_global 36from pip._internal.wheel_builder import build, should_build_for_install_command 37 38if MYPY_CHECK_RUNNING: 39 from optparse import Values 40 from typing import Iterable, List, Optional 41 42 from pip._internal.models.format_control import FormatControl 43 from pip._internal.operations.check import ConflictDetails 44 from pip._internal.req.req_install import InstallRequirement 45 from pip._internal.wheel_builder import BinaryAllowedPredicate 46 47 48logger = logging.getLogger(__name__) 49 50 51def get_check_binary_allowed(format_control): 52 # type: (FormatControl) -> BinaryAllowedPredicate 53 def check_binary_allowed(req): 54 # type: (InstallRequirement) -> bool 55 if req.use_pep517: 56 return True 57 canonical_name = canonicalize_name(req.name) 58 allowed_formats = format_control.get_allowed_formats(canonical_name) 59 return "binary" in allowed_formats 60 61 return check_binary_allowed 62 63 64class InstallCommand(RequirementCommand): 65 """ 66 Install packages from: 67 68 - PyPI (and other indexes) using requirement specifiers. 69 - VCS project urls. 70 - Local project directories. 71 - Local or remote source archives. 72 73 pip also supports installing from "requirements files", which provide 74 an easy way to specify a whole environment to be installed. 75 """ 76 77 usage = """ 78 %prog [options] <requirement specifier> [package-index-options] ... 79 %prog [options] -r <requirements file> [package-index-options] ... 80 %prog [options] [-e] <vcs project url> ... 81 %prog [options] [-e] <local project path> ... 82 %prog [options] <archive url/path> ...""" 83 84 def add_options(self): 85 # type: () -> None 86 self.cmd_opts.add_option(cmdoptions.requirements()) 87 self.cmd_opts.add_option(cmdoptions.constraints()) 88 self.cmd_opts.add_option(cmdoptions.no_deps()) 89 self.cmd_opts.add_option(cmdoptions.pre()) 90 91 self.cmd_opts.add_option(cmdoptions.editable()) 92 self.cmd_opts.add_option( 93 '-t', '--target', 94 dest='target_dir', 95 metavar='dir', 96 default=None, 97 help='Install packages into <dir>. ' 98 'By default this will not replace existing files/folders in ' 99 '<dir>. Use --upgrade to replace existing packages in <dir> ' 100 'with new versions.' 101 ) 102 cmdoptions.add_target_python_options(self.cmd_opts) 103 104 self.cmd_opts.add_option( 105 '--user', 106 dest='use_user_site', 107 action='store_true', 108 help="Install to the Python user install directory for your " 109 "platform. Typically ~/.local/, or %APPDATA%\\Python on " 110 "Windows. (See the Python documentation for site.USER_BASE " 111 "for full details.)") 112 self.cmd_opts.add_option( 113 '--no-user', 114 dest='use_user_site', 115 action='store_false', 116 help=SUPPRESS_HELP) 117 self.cmd_opts.add_option( 118 '--root', 119 dest='root_path', 120 metavar='dir', 121 default=None, 122 help="Install everything relative to this alternate root " 123 "directory.") 124 self.cmd_opts.add_option( 125 '--prefix', 126 dest='prefix_path', 127 metavar='dir', 128 default=None, 129 help="Installation prefix where lib, bin and other top-level " 130 "folders are placed") 131 132 self.cmd_opts.add_option(cmdoptions.build_dir()) 133 134 self.cmd_opts.add_option(cmdoptions.src()) 135 136 self.cmd_opts.add_option( 137 '-U', '--upgrade', 138 dest='upgrade', 139 action='store_true', 140 help='Upgrade all specified packages to the newest available ' 141 'version. The handling of dependencies depends on the ' 142 'upgrade-strategy used.' 143 ) 144 145 self.cmd_opts.add_option( 146 '--upgrade-strategy', 147 dest='upgrade_strategy', 148 default='only-if-needed', 149 choices=['only-if-needed', 'eager'], 150 help='Determines how dependency upgrading should be handled ' 151 '[default: %default]. ' 152 '"eager" - dependencies are upgraded regardless of ' 153 'whether the currently installed version satisfies the ' 154 'requirements of the upgraded package(s). ' 155 '"only-if-needed" - are upgraded only when they do not ' 156 'satisfy the requirements of the upgraded package(s).' 157 ) 158 159 self.cmd_opts.add_option( 160 '--force-reinstall', 161 dest='force_reinstall', 162 action='store_true', 163 help='Reinstall all packages even if they are already ' 164 'up-to-date.') 165 166 self.cmd_opts.add_option( 167 '-I', '--ignore-installed', 168 dest='ignore_installed', 169 action='store_true', 170 help='Ignore the installed packages, overwriting them. ' 171 'This can break your system if the existing package ' 172 'is of a different version or was installed ' 173 'with a different package manager!' 174 ) 175 176 self.cmd_opts.add_option(cmdoptions.ignore_requires_python()) 177 self.cmd_opts.add_option(cmdoptions.no_build_isolation()) 178 self.cmd_opts.add_option(cmdoptions.use_pep517()) 179 self.cmd_opts.add_option(cmdoptions.no_use_pep517()) 180 181 self.cmd_opts.add_option(cmdoptions.install_options()) 182 self.cmd_opts.add_option(cmdoptions.global_options()) 183 184 self.cmd_opts.add_option( 185 "--compile", 186 action="store_true", 187 dest="compile", 188 default=True, 189 help="Compile Python source files to bytecode", 190 ) 191 192 self.cmd_opts.add_option( 193 "--no-compile", 194 action="store_false", 195 dest="compile", 196 help="Do not compile Python source files to bytecode", 197 ) 198 199 self.cmd_opts.add_option( 200 "--no-warn-script-location", 201 action="store_false", 202 dest="warn_script_location", 203 default=True, 204 help="Do not warn when installing scripts outside PATH", 205 ) 206 self.cmd_opts.add_option( 207 "--no-warn-conflicts", 208 action="store_false", 209 dest="warn_about_conflicts", 210 default=True, 211 help="Do not warn about broken dependencies", 212 ) 213 214 self.cmd_opts.add_option(cmdoptions.no_binary()) 215 self.cmd_opts.add_option(cmdoptions.only_binary()) 216 self.cmd_opts.add_option(cmdoptions.prefer_binary()) 217 self.cmd_opts.add_option(cmdoptions.require_hashes()) 218 self.cmd_opts.add_option(cmdoptions.progress_bar()) 219 220 index_opts = cmdoptions.make_option_group( 221 cmdoptions.index_group, 222 self.parser, 223 ) 224 225 self.parser.insert_option_group(0, index_opts) 226 self.parser.insert_option_group(0, self.cmd_opts) 227 228 @with_cleanup 229 def run(self, options, args): 230 # type: (Values, List[str]) -> int 231 if options.use_user_site and options.target_dir is not None: 232 raise CommandError("Can not combine '--user' and '--target'") 233 234 cmdoptions.check_install_build_global(options) 235 upgrade_strategy = "to-satisfy-only" 236 if options.upgrade: 237 upgrade_strategy = options.upgrade_strategy 238 239 cmdoptions.check_dist_restriction(options, check_target=True) 240 241 install_options = options.install_options or [] 242 243 logger.debug("Using %s", get_pip_version()) 244 options.use_user_site = decide_user_install( 245 options.use_user_site, 246 prefix_path=options.prefix_path, 247 target_dir=options.target_dir, 248 root_path=options.root_path, 249 isolated_mode=options.isolated_mode, 250 ) 251 252 target_temp_dir = None # type: Optional[TempDirectory] 253 target_temp_dir_path = None # type: Optional[str] 254 if options.target_dir: 255 options.ignore_installed = True 256 options.target_dir = os.path.abspath(options.target_dir) 257 if (os.path.exists(options.target_dir) and not 258 os.path.isdir(options.target_dir)): 259 raise CommandError( 260 "Target path exists but is not a directory, will not " 261 "continue." 262 ) 263 264 # Create a target directory for using with the target option 265 target_temp_dir = TempDirectory(kind="target") 266 target_temp_dir_path = target_temp_dir.path 267 self.enter_context(target_temp_dir) 268 269 global_options = options.global_options or [] 270 271 session = self.get_default_session(options) 272 273 target_python = make_target_python(options) 274 finder = self._build_package_finder( 275 options=options, 276 session=session, 277 target_python=target_python, 278 ignore_requires_python=options.ignore_requires_python, 279 ) 280 wheel_cache = WheelCache(options.cache_dir, options.format_control) 281 282 req_tracker = self.enter_context(get_requirement_tracker()) 283 284 directory = TempDirectory( 285 delete=not options.no_clean, 286 kind="install", 287 globally_managed=True, 288 ) 289 290 try: 291 reqs = self.get_requirements(args, options, finder, session) 292 293 reject_location_related_install_options( 294 reqs, options.install_options 295 ) 296 297 preparer = self.make_requirement_preparer( 298 temp_build_dir=directory, 299 options=options, 300 req_tracker=req_tracker, 301 session=session, 302 finder=finder, 303 use_user_site=options.use_user_site, 304 ) 305 resolver = self.make_resolver( 306 preparer=preparer, 307 finder=finder, 308 options=options, 309 wheel_cache=wheel_cache, 310 use_user_site=options.use_user_site, 311 ignore_installed=options.ignore_installed, 312 ignore_requires_python=options.ignore_requires_python, 313 force_reinstall=options.force_reinstall, 314 upgrade_strategy=upgrade_strategy, 315 use_pep517=options.use_pep517, 316 ) 317 318 self.trace_basic_info(finder) 319 320 requirement_set = resolver.resolve( 321 reqs, check_supported_wheels=not options.target_dir 322 ) 323 324 try: 325 pip_req = requirement_set.get_requirement("pip") 326 except KeyError: 327 modifying_pip = False 328 else: 329 # If we're not replacing an already installed pip, 330 # we're not modifying it. 331 modifying_pip = pip_req.satisfied_by is None 332 protect_pip_from_modification_on_windows( 333 modifying_pip=modifying_pip 334 ) 335 336 check_binary_allowed = get_check_binary_allowed( 337 finder.format_control 338 ) 339 340 reqs_to_build = [ 341 r for r in requirement_set.requirements.values() 342 if should_build_for_install_command( 343 r, check_binary_allowed 344 ) 345 ] 346 347 _, build_failures = build( 348 reqs_to_build, 349 wheel_cache=wheel_cache, 350 verify=True, 351 build_options=[], 352 global_options=[], 353 ) 354 355 # If we're using PEP 517, we cannot do a direct install 356 # so we fail here. 357 pep517_build_failure_names = [ 358 r.name # type: ignore 359 for r in build_failures if r.use_pep517 360 ] # type: List[str] 361 if pep517_build_failure_names: 362 raise InstallationError( 363 "Could not build wheels for {} which use" 364 " PEP 517 and cannot be installed directly".format( 365 ", ".join(pep517_build_failure_names) 366 ) 367 ) 368 369 # For now, we just warn about failures building legacy 370 # requirements, as we'll fall through to a direct 371 # install for those. 372 for r in build_failures: 373 if not r.use_pep517: 374 r.legacy_install_reason = 8368 375 376 to_install = resolver.get_installation_order( 377 requirement_set 378 ) 379 380 # Check for conflicts in the package set we're installing. 381 conflicts = None # type: Optional[ConflictDetails] 382 should_warn_about_conflicts = ( 383 not options.ignore_dependencies and 384 options.warn_about_conflicts 385 ) 386 if should_warn_about_conflicts: 387 conflicts = self._determine_conflicts(to_install) 388 389 # Don't warn about script install locations if 390 # --target has been specified 391 warn_script_location = options.warn_script_location 392 if options.target_dir: 393 warn_script_location = False 394 395 installed = install_given_reqs( 396 to_install, 397 install_options, 398 global_options, 399 root=options.root_path, 400 home=target_temp_dir_path, 401 prefix=options.prefix_path, 402 warn_script_location=warn_script_location, 403 use_user_site=options.use_user_site, 404 pycompile=options.compile, 405 ) 406 407 lib_locations = get_lib_location_guesses( 408 user=options.use_user_site, 409 home=target_temp_dir_path, 410 root=options.root_path, 411 prefix=options.prefix_path, 412 isolated=options.isolated_mode, 413 ) 414 working_set = pkg_resources.WorkingSet(lib_locations) 415 416 installed.sort(key=operator.attrgetter('name')) 417 items = [] 418 for result in installed: 419 item = result.name 420 try: 421 installed_version = get_installed_version( 422 result.name, working_set=working_set 423 ) 424 if installed_version: 425 item += '-' + installed_version 426 except Exception: 427 pass 428 items.append(item) 429 430 if conflicts is not None: 431 self._warn_about_conflicts( 432 conflicts, 433 resolver_variant=self.determine_resolver_variant(options), 434 ) 435 436 installed_desc = ' '.join(items) 437 if installed_desc: 438 write_output( 439 'Successfully installed %s', installed_desc, 440 ) 441 except EnvironmentError as error: 442 show_traceback = (self.verbosity >= 1) 443 444 message = create_env_error_message( 445 error, show_traceback, options.use_user_site, 446 ) 447 logger.error(message, exc_info=show_traceback) # noqa 448 449 return ERROR 450 451 if options.target_dir: 452 assert target_temp_dir 453 self._handle_target_dir( 454 options.target_dir, target_temp_dir, options.upgrade 455 ) 456 457 return SUCCESS 458 459 def _handle_target_dir(self, target_dir, target_temp_dir, upgrade): 460 # type: (str, TempDirectory, bool) -> None 461 ensure_dir(target_dir) 462 463 # Checking both purelib and platlib directories for installed 464 # packages to be moved to target directory 465 lib_dir_list = [] 466 467 # Checking both purelib and platlib directories for installed 468 # packages to be moved to target directory 469 scheme = distutils_scheme('', home=target_temp_dir.path) 470 purelib_dir = scheme['purelib'] 471 platlib_dir = scheme['platlib'] 472 data_dir = scheme['data'] 473 474 if os.path.exists(purelib_dir): 475 lib_dir_list.append(purelib_dir) 476 if os.path.exists(platlib_dir) and platlib_dir != purelib_dir: 477 lib_dir_list.append(platlib_dir) 478 if os.path.exists(data_dir): 479 lib_dir_list.append(data_dir) 480 481 for lib_dir in lib_dir_list: 482 for item in os.listdir(lib_dir): 483 if lib_dir == data_dir: 484 ddir = os.path.join(data_dir, item) 485 if any(s.startswith(ddir) for s in lib_dir_list[:-1]): 486 continue 487 target_item_dir = os.path.join(target_dir, item) 488 if os.path.exists(target_item_dir): 489 if not upgrade: 490 logger.warning( 491 'Target directory %s already exists. Specify ' 492 '--upgrade to force replacement.', 493 target_item_dir 494 ) 495 continue 496 if os.path.islink(target_item_dir): 497 logger.warning( 498 'Target directory %s already exists and is ' 499 'a link. pip will not automatically replace ' 500 'links, please remove if replacement is ' 501 'desired.', 502 target_item_dir 503 ) 504 continue 505 if os.path.isdir(target_item_dir): 506 shutil.rmtree(target_item_dir) 507 else: 508 os.remove(target_item_dir) 509 510 shutil.move( 511 os.path.join(lib_dir, item), 512 target_item_dir 513 ) 514 515 def _determine_conflicts(self, to_install): 516 # type: (List[InstallRequirement]) -> Optional[ConflictDetails] 517 try: 518 return check_install_conflicts(to_install) 519 except Exception: 520 logger.exception( 521 "Error while checking for conflicts. Please file an issue on " 522 "pip's issue tracker: https://github.com/pypa/pip/issues/new" 523 ) 524 return None 525 526 def _warn_about_conflicts(self, conflict_details, resolver_variant): 527 # type: (ConflictDetails, str) -> None 528 package_set, (missing, conflicting) = conflict_details 529 if not missing and not conflicting: 530 return 531 532 parts = [] # type: List[str] 533 if resolver_variant == "legacy": 534 parts.append( 535 "pip's legacy dependency resolver does not consider dependency " 536 "conflicts when selecting packages. This behaviour is the " 537 "source of the following dependency conflicts." 538 ) 539 else: 540 assert resolver_variant == "2020-resolver" 541 parts.append( 542 "pip's dependency resolver does not currently take into account " 543 "all the packages that are installed. This behaviour is the " 544 "source of the following dependency conflicts." 545 ) 546 547 # NOTE: There is some duplication here, with commands/check.py 548 for project_name in missing: 549 version = package_set[project_name][0] 550 for dependency in missing[project_name]: 551 message = ( 552 "{name} {version} requires {requirement}, " 553 "which is not installed." 554 ).format( 555 name=project_name, 556 version=version, 557 requirement=dependency[1], 558 ) 559 parts.append(message) 560 561 for project_name in conflicting: 562 version = package_set[project_name][0] 563 for dep_name, dep_version, req in conflicting[project_name]: 564 message = ( 565 "{name} {version} requires {requirement}, but {you} have " 566 "{dep_name} {dep_version} which is incompatible." 567 ).format( 568 name=project_name, 569 version=version, 570 requirement=req, 571 dep_name=dep_name, 572 dep_version=dep_version, 573 you=("you" if resolver_variant == "2020-resolver" else "you'll") 574 ) 575 parts.append(message) 576 577 logger.critical("\n".join(parts)) 578 579 580def get_lib_location_guesses( 581 user=False, # type: bool 582 home=None, # type: Optional[str] 583 root=None, # type: Optional[str] 584 isolated=False, # type: bool 585 prefix=None # type: Optional[str] 586): 587 # type:(...) -> List[str] 588 scheme = distutils_scheme('', user=user, home=home, root=root, 589 isolated=isolated, prefix=prefix) 590 return [scheme['purelib'], scheme['platlib']] 591 592 593def site_packages_writable(root, isolated): 594 # type: (Optional[str], bool) -> bool 595 return all( 596 test_writable_dir(d) for d in set( 597 get_lib_location_guesses(root=root, isolated=isolated)) 598 ) 599 600 601def decide_user_install( 602 use_user_site, # type: Optional[bool] 603 prefix_path=None, # type: Optional[str] 604 target_dir=None, # type: Optional[str] 605 root_path=None, # type: Optional[str] 606 isolated_mode=False, # type: bool 607): 608 # type: (...) -> bool 609 """Determine whether to do a user install based on the input options. 610 611 If use_user_site is False, no additional checks are done. 612 If use_user_site is True, it is checked for compatibility with other 613 options. 614 If use_user_site is None, the default behaviour depends on the environment, 615 which is provided by the other arguments. 616 """ 617 # In some cases (config from tox), use_user_site can be set to an integer 618 # rather than a bool, which 'use_user_site is False' wouldn't catch. 619 if (use_user_site is not None) and (not use_user_site): 620 logger.debug("Non-user install by explicit request") 621 return False 622 623 if use_user_site: 624 if prefix_path: 625 raise CommandError( 626 "Can not combine '--user' and '--prefix' as they imply " 627 "different installation locations" 628 ) 629 if virtualenv_no_global(): 630 raise InstallationError( 631 "Can not perform a '--user' install. User site-packages " 632 "are not visible in this virtualenv." 633 ) 634 logger.debug("User install by explicit request") 635 return True 636 637 # If we are here, user installs have not been explicitly requested/avoided 638 assert use_user_site is None 639 640 # user install incompatible with --prefix/--target 641 if prefix_path or target_dir: 642 logger.debug("Non-user install due to --prefix or --target option") 643 return False 644 645 # If user installs are not enabled, choose a non-user install 646 if not site.ENABLE_USER_SITE: 647 logger.debug("Non-user install because user site-packages disabled") 648 return False 649 650 # If we have permission for a non-user install, do that, 651 # otherwise do a user install. 652 if site_packages_writable(root=root_path, isolated=isolated_mode): 653 logger.debug("Non-user install because site-packages writeable") 654 return False 655 656 logger.info("Defaulting to user installation because normal site-packages " 657 "is not writeable") 658 return True 659 660 661def reject_location_related_install_options(requirements, options): 662 # type: (List[InstallRequirement], Optional[List[str]]) -> None 663 """If any location-changing --install-option arguments were passed for 664 requirements or on the command-line, then show a deprecation warning. 665 """ 666 def format_options(option_names): 667 # type: (Iterable[str]) -> List[str] 668 return ["--{}".format(name.replace("_", "-")) for name in option_names] 669 670 offenders = [] 671 672 for requirement in requirements: 673 install_options = requirement.install_options 674 location_options = parse_distutils_args(install_options) 675 if location_options: 676 offenders.append( 677 "{!r} from {}".format( 678 format_options(location_options.keys()), requirement 679 ) 680 ) 681 682 if options: 683 location_options = parse_distutils_args(options) 684 if location_options: 685 offenders.append( 686 "{!r} from command line".format( 687 format_options(location_options.keys()) 688 ) 689 ) 690 691 if not offenders: 692 return 693 694 raise CommandError( 695 "Location-changing options found in --install-option: {}." 696 " This is unsupported, use pip-level options like --user," 697 " --prefix, --root, and --target instead.".format( 698 "; ".join(offenders) 699 ) 700 ) 701 702 703def create_env_error_message(error, show_traceback, using_user_site): 704 # type: (EnvironmentError, bool, bool) -> str 705 """Format an error message for an EnvironmentError 706 707 It may occur anytime during the execution of the install command. 708 """ 709 parts = [] 710 711 # Mention the error if we are not going to show a traceback 712 parts.append("Could not install packages due to an EnvironmentError") 713 if not show_traceback: 714 parts.append(": ") 715 parts.append(str(error)) 716 else: 717 parts.append(".") 718 719 # Spilt the error indication from a helper message (if any) 720 parts[-1] += "\n" 721 722 # Suggest useful actions to the user: 723 # (1) using user site-packages or (2) verifying the permissions 724 if error.errno == errno.EACCES: 725 user_option_part = "Consider using the `--user` option" 726 permissions_part = "Check the permissions" 727 728 if not using_user_site: 729 parts.extend([ 730 user_option_part, " or ", 731 permissions_part.lower(), 732 ]) 733 else: 734 parts.append(permissions_part) 735 parts.append(".\n") 736 737 return "".join(parts).strip() + "\n" 738