1"""Handles all VCS (version control) support""" 2from __future__ import absolute_import 3 4import errno 5import logging 6import os 7import shutil 8import sys 9 10from pip._vendor import pkg_resources 11from pip._vendor.six.moves.urllib import parse as urllib_parse 12 13from pip._internal.exceptions import BadCommand 14from pip._internal.utils.misc import ( 15 ask_path_exists, backup_dir, call_subprocess, display_path, rmtree, 16) 17from pip._internal.utils.typing import MYPY_CHECK_RUNNING 18 19if MYPY_CHECK_RUNNING: 20 from typing import ( 21 Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type 22 ) 23 from pip._internal.utils.ui import SpinnerInterface 24 25 AuthInfo = Tuple[Optional[str], Optional[str]] 26 27__all__ = ['vcs'] 28 29 30logger = logging.getLogger(__name__) 31 32 33def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): 34 """ 35 Return the URL for a VCS requirement. 36 37 Args: 38 repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). 39 project_name: the (unescaped) project name. 40 """ 41 egg_project_name = pkg_resources.to_filename(project_name) 42 req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) 43 if subdir: 44 req += '&subdirectory={}'.format(subdir) 45 46 return req 47 48 49class RemoteNotFoundError(Exception): 50 pass 51 52 53class RevOptions(object): 54 55 """ 56 Encapsulates a VCS-specific revision to install, along with any VCS 57 install options. 58 59 Instances of this class should be treated as if immutable. 60 """ 61 62 def __init__( 63 self, 64 vc_class, # type: Type[VersionControl] 65 rev=None, # type: Optional[str] 66 extra_args=None, # type: Optional[List[str]] 67 ): 68 # type: (...) -> None 69 """ 70 Args: 71 vc_class: a VersionControl subclass. 72 rev: the name of the revision to install. 73 extra_args: a list of extra options. 74 """ 75 if extra_args is None: 76 extra_args = [] 77 78 self.extra_args = extra_args 79 self.rev = rev 80 self.vc_class = vc_class 81 82 def __repr__(self): 83 return '<RevOptions {}: rev={!r}>'.format(self.vc_class.name, self.rev) 84 85 @property 86 def arg_rev(self): 87 # type: () -> Optional[str] 88 if self.rev is None: 89 return self.vc_class.default_arg_rev 90 91 return self.rev 92 93 def to_args(self): 94 # type: () -> List[str] 95 """ 96 Return the VCS-specific command arguments. 97 """ 98 args = [] # type: List[str] 99 rev = self.arg_rev 100 if rev is not None: 101 args += self.vc_class.get_base_rev_args(rev) 102 args += self.extra_args 103 104 return args 105 106 def to_display(self): 107 # type: () -> str 108 if not self.rev: 109 return '' 110 111 return ' (to revision {})'.format(self.rev) 112 113 def make_new(self, rev): 114 # type: (str) -> RevOptions 115 """ 116 Make a copy of the current instance, but with a new rev. 117 118 Args: 119 rev: the name of the revision for the new object. 120 """ 121 return self.vc_class.make_rev_options(rev, extra_args=self.extra_args) 122 123 124class VcsSupport(object): 125 _registry = {} # type: Dict[str, VersionControl] 126 schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] 127 128 def __init__(self): 129 # type: () -> None 130 # Register more schemes with urlparse for various version control 131 # systems 132 urllib_parse.uses_netloc.extend(self.schemes) 133 # Python >= 2.7.4, 3.3 doesn't have uses_fragment 134 if getattr(urllib_parse, 'uses_fragment', None): 135 urllib_parse.uses_fragment.extend(self.schemes) 136 super(VcsSupport, self).__init__() 137 138 def __iter__(self): 139 return self._registry.__iter__() 140 141 @property 142 def backends(self): 143 # type: () -> List[VersionControl] 144 return list(self._registry.values()) 145 146 @property 147 def dirnames(self): 148 # type: () -> List[str] 149 return [backend.dirname for backend in self.backends] 150 151 @property 152 def all_schemes(self): 153 # type: () -> List[str] 154 schemes = [] # type: List[str] 155 for backend in self.backends: 156 schemes.extend(backend.schemes) 157 return schemes 158 159 def register(self, cls): 160 # type: (Type[VersionControl]) -> None 161 if not hasattr(cls, 'name'): 162 logger.warning('Cannot register VCS %s', cls.__name__) 163 return 164 if cls.name not in self._registry: 165 self._registry[cls.name] = cls() 166 logger.debug('Registered VCS backend: %s', cls.name) 167 168 def unregister(self, name): 169 # type: (str) -> None 170 if name in self._registry: 171 del self._registry[name] 172 173 def get_backend_for_dir(self, location): 174 # type: (str) -> Optional[VersionControl] 175 """ 176 Return a VersionControl object if a repository of that type is found 177 at the given directory. 178 """ 179 for vcs_backend in self._registry.values(): 180 if vcs_backend.controls_location(location): 181 logger.debug('Determine that %s uses VCS: %s', 182 location, vcs_backend.name) 183 return vcs_backend 184 return None 185 186 def get_backend(self, name): 187 # type: (str) -> Optional[VersionControl] 188 """ 189 Return a VersionControl object or None. 190 """ 191 name = name.lower() 192 return self._registry.get(name) 193 194 195vcs = VcsSupport() 196 197 198class VersionControl(object): 199 name = '' 200 dirname = '' 201 repo_name = '' 202 # List of supported schemes for this Version Control 203 schemes = () # type: Tuple[str, ...] 204 # Iterable of environment variable names to pass to call_subprocess(). 205 unset_environ = () # type: Tuple[str, ...] 206 default_arg_rev = None # type: Optional[str] 207 208 @classmethod 209 def should_add_vcs_url_prefix(cls, remote_url): 210 """ 211 Return whether the vcs prefix (e.g. "git+") should be added to a 212 repository's remote url when used in a requirement. 213 """ 214 return not remote_url.lower().startswith('{}:'.format(cls.name)) 215 216 @classmethod 217 def get_subdirectory(cls, repo_dir): 218 """ 219 Return the path to setup.py, relative to the repo root. 220 """ 221 return None 222 223 @classmethod 224 def get_requirement_revision(cls, repo_dir): 225 """ 226 Return the revision string that should be used in a requirement. 227 """ 228 return cls.get_revision(repo_dir) 229 230 @classmethod 231 def get_src_requirement(cls, repo_dir, project_name): 232 """ 233 Return the requirement string to use to redownload the files 234 currently at the given repository directory. 235 236 Args: 237 project_name: the (unescaped) project name. 238 239 The return value has a form similar to the following: 240 241 {repository_url}@{revision}#egg={project_name} 242 """ 243 repo_url = cls.get_remote_url(repo_dir) 244 if repo_url is None: 245 return None 246 247 if cls.should_add_vcs_url_prefix(repo_url): 248 repo_url = '{}+{}'.format(cls.name, repo_url) 249 250 revision = cls.get_requirement_revision(repo_dir) 251 subdir = cls.get_subdirectory(repo_dir) 252 req = make_vcs_requirement_url(repo_url, revision, project_name, 253 subdir=subdir) 254 255 return req 256 257 @staticmethod 258 def get_base_rev_args(rev): 259 """ 260 Return the base revision arguments for a vcs command. 261 262 Args: 263 rev: the name of a revision to install. Cannot be None. 264 """ 265 raise NotImplementedError 266 267 @classmethod 268 def make_rev_options(cls, rev=None, extra_args=None): 269 # type: (Optional[str], Optional[List[str]]) -> RevOptions 270 """ 271 Return a RevOptions object. 272 273 Args: 274 rev: the name of a revision to install. 275 extra_args: a list of extra options. 276 """ 277 return RevOptions(cls, rev, extra_args=extra_args) 278 279 @classmethod 280 def _is_local_repository(cls, repo): 281 # type: (str) -> bool 282 """ 283 posix absolute paths start with os.path.sep, 284 win32 ones start with drive (like c:\\folder) 285 """ 286 drive, tail = os.path.splitdrive(repo) 287 return repo.startswith(os.path.sep) or bool(drive) 288 289 def export(self, location, url): 290 """ 291 Export the repository at the url to the destination location 292 i.e. only download the files, without vcs informations 293 294 :param url: the repository URL starting with a vcs prefix. 295 """ 296 raise NotImplementedError 297 298 @classmethod 299 def get_netloc_and_auth(cls, netloc, scheme): 300 """ 301 Parse the repository URL's netloc, and return the new netloc to use 302 along with auth information. 303 304 Args: 305 netloc: the original repository URL netloc. 306 scheme: the repository URL's scheme without the vcs prefix. 307 308 This is mainly for the Subversion class to override, so that auth 309 information can be provided via the --username and --password options 310 instead of through the URL. For other subclasses like Git without 311 such an option, auth information must stay in the URL. 312 313 Returns: (netloc, (username, password)). 314 """ 315 return netloc, (None, None) 316 317 @classmethod 318 def get_url_rev_and_auth(cls, url): 319 # type: (str) -> Tuple[str, Optional[str], AuthInfo] 320 """ 321 Parse the repository URL to use, and return the URL, revision, 322 and auth info to use. 323 324 Returns: (url, rev, (username, password)). 325 """ 326 scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) 327 if '+' not in scheme: 328 raise ValueError( 329 "Sorry, {!r} is a malformed VCS url. " 330 "The format is <vcs>+<protocol>://<url>, " 331 "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) 332 ) 333 # Remove the vcs prefix. 334 scheme = scheme.split('+', 1)[1] 335 netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme) 336 rev = None 337 if '@' in path: 338 path, rev = path.rsplit('@', 1) 339 url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) 340 return url, rev, user_pass 341 342 @staticmethod 343 def make_rev_args(username, password): 344 """ 345 Return the RevOptions "extra arguments" to use in obtain(). 346 """ 347 return [] 348 349 def get_url_rev_options(self, url): 350 # type: (str) -> Tuple[str, RevOptions] 351 """ 352 Return the URL and RevOptions object to use in obtain() and in 353 some cases export(), as a tuple (url, rev_options). 354 """ 355 url, rev, user_pass = self.get_url_rev_and_auth(url) 356 username, password = user_pass 357 extra_args = self.make_rev_args(username, password) 358 rev_options = self.make_rev_options(rev, extra_args=extra_args) 359 360 return url, rev_options 361 362 @staticmethod 363 def normalize_url(url): 364 # type: (str) -> str 365 """ 366 Normalize a URL for comparison by unquoting it and removing any 367 trailing slash. 368 """ 369 return urllib_parse.unquote(url).rstrip('/') 370 371 @classmethod 372 def compare_urls(cls, url1, url2): 373 # type: (str, str) -> bool 374 """ 375 Compare two repo URLs for identity, ignoring incidental differences. 376 """ 377 return (cls.normalize_url(url1) == cls.normalize_url(url2)) 378 379 def fetch_new(self, dest, url, rev_options): 380 """ 381 Fetch a revision from a repository, in the case that this is the 382 first fetch from the repository. 383 384 Args: 385 dest: the directory to fetch the repository to. 386 rev_options: a RevOptions object. 387 """ 388 raise NotImplementedError 389 390 def switch(self, dest, url, rev_options): 391 """ 392 Switch the repo at ``dest`` to point to ``URL``. 393 394 Args: 395 rev_options: a RevOptions object. 396 """ 397 raise NotImplementedError 398 399 def update(self, dest, url, rev_options): 400 """ 401 Update an already-existing repo to the given ``rev_options``. 402 403 Args: 404 rev_options: a RevOptions object. 405 """ 406 raise NotImplementedError 407 408 @classmethod 409 def is_commit_id_equal(cls, dest, name): 410 """ 411 Return whether the id of the current commit equals the given name. 412 413 Args: 414 dest: the repository directory. 415 name: a string name. 416 """ 417 raise NotImplementedError 418 419 def obtain(self, dest, url): 420 # type: (str, str) -> None 421 """ 422 Install or update in editable mode the package represented by this 423 VersionControl object. 424 425 :param dest: the repository directory in which to install or update. 426 :param url: the repository URL starting with a vcs prefix. 427 """ 428 url, rev_options = self.get_url_rev_options(url) 429 430 if not os.path.exists(dest): 431 self.fetch_new(dest, url, rev_options) 432 return 433 434 rev_display = rev_options.to_display() 435 if self.is_repository_directory(dest): 436 existing_url = self.get_remote_url(dest) 437 if self.compare_urls(existing_url, url): 438 logger.debug( 439 '%s in %s exists, and has correct URL (%s)', 440 self.repo_name.title(), 441 display_path(dest), 442 url, 443 ) 444 if not self.is_commit_id_equal(dest, rev_options.rev): 445 logger.info( 446 'Updating %s %s%s', 447 display_path(dest), 448 self.repo_name, 449 rev_display, 450 ) 451 self.update(dest, url, rev_options) 452 else: 453 logger.info('Skipping because already up-to-date.') 454 return 455 456 logger.warning( 457 '%s %s in %s exists with URL %s', 458 self.name, 459 self.repo_name, 460 display_path(dest), 461 existing_url, 462 ) 463 prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', 464 ('s', 'i', 'w', 'b')) 465 else: 466 logger.warning( 467 'Directory %s already exists, and is not a %s %s.', 468 dest, 469 self.name, 470 self.repo_name, 471 ) 472 # https://github.com/python/mypy/issues/1174 473 prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore 474 ('i', 'w', 'b')) 475 476 logger.warning( 477 'The plan is to install the %s repository %s', 478 self.name, 479 url, 480 ) 481 response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) 482 483 if response == 'a': 484 sys.exit(-1) 485 486 if response == 'w': 487 logger.warning('Deleting %s', display_path(dest)) 488 rmtree(dest) 489 self.fetch_new(dest, url, rev_options) 490 return 491 492 if response == 'b': 493 dest_dir = backup_dir(dest) 494 logger.warning( 495 'Backing up %s to %s', display_path(dest), dest_dir, 496 ) 497 shutil.move(dest, dest_dir) 498 self.fetch_new(dest, url, rev_options) 499 return 500 501 # Do nothing if the response is "i". 502 if response == 's': 503 logger.info( 504 'Switching %s %s to %s%s', 505 self.repo_name, 506 display_path(dest), 507 url, 508 rev_display, 509 ) 510 self.switch(dest, url, rev_options) 511 512 def unpack(self, location, url): 513 # type: (str, str) -> None 514 """ 515 Clean up current location and download the url repository 516 (and vcs infos) into location 517 518 :param url: the repository URL starting with a vcs prefix. 519 """ 520 if os.path.exists(location): 521 rmtree(location) 522 self.obtain(location, url=url) 523 524 @classmethod 525 def get_remote_url(cls, location): 526 """ 527 Return the url used at location 528 529 Raises RemoteNotFoundError if the repository does not have a remote 530 url configured. 531 """ 532 raise NotImplementedError 533 534 @classmethod 535 def get_revision(cls, location): 536 """ 537 Return the current commit id of the files at the given location. 538 """ 539 raise NotImplementedError 540 541 @classmethod 542 def run_command( 543 cls, 544 cmd, # type: List[str] 545 show_stdout=True, # type: bool 546 cwd=None, # type: Optional[str] 547 on_returncode='raise', # type: str 548 extra_ok_returncodes=None, # type: Optional[Iterable[int]] 549 command_desc=None, # type: Optional[str] 550 extra_environ=None, # type: Optional[Mapping[str, Any]] 551 spinner=None # type: Optional[SpinnerInterface] 552 ): 553 # type: (...) -> Text 554 """ 555 Run a VCS subcommand 556 This is simply a wrapper around call_subprocess that adds the VCS 557 command name, and checks that the VCS is available 558 """ 559 cmd = [cls.name] + cmd 560 try: 561 return call_subprocess(cmd, show_stdout, cwd, 562 on_returncode=on_returncode, 563 extra_ok_returncodes=extra_ok_returncodes, 564 command_desc=command_desc, 565 extra_environ=extra_environ, 566 unset_environ=cls.unset_environ, 567 spinner=spinner) 568 except OSError as e: 569 # errno.ENOENT = no such file or directory 570 # In other words, the VCS executable isn't available 571 if e.errno == errno.ENOENT: 572 raise BadCommand( 573 'Cannot find command %r - do you have ' 574 '%r installed and in your ' 575 'PATH?' % (cls.name, cls.name)) 576 else: 577 raise # re-raise exception if a different error occurred 578 579 @classmethod 580 def is_repository_directory(cls, path): 581 # type: (str) -> bool 582 """ 583 Return whether a directory path is a repository directory. 584 """ 585 logger.debug('Checking in %s for %s (%s)...', 586 path, cls.dirname, cls.name) 587 return os.path.exists(os.path.join(path, cls.dirname)) 588 589 @classmethod 590 def controls_location(cls, location): 591 # type: (str) -> bool 592 """ 593 Check if a location is controlled by the vcs. 594 It is meant to be overridden to implement smarter detection 595 mechanisms for specific vcs. 596 597 This can do more than is_repository_directory() alone. For example, 598 the Git override checks that Git is actually available. 599 """ 600 return cls.is_repository_directory(location) 601