1"""Backing implementation for InstallRequirement's various constructors 2 3The idea here is that these formed a major chunk of InstallRequirement's size 4so, moving them and support code dedicated to them outside of that class 5helps creates for better understandability for the rest of the code. 6 7These are meant to be used elsewhere within pip to create instances of 8InstallRequirement. 9""" 10 11import logging 12import os 13import re 14 15from pip._vendor.packaging.markers import Marker 16from pip._vendor.packaging.requirements import InvalidRequirement, Requirement 17from pip._vendor.packaging.specifiers import Specifier 18from pip._vendor.pkg_resources import RequirementParseError, parse_requirements 19 20from pip._internal.exceptions import InstallationError 21from pip._internal.models.index import PyPI, TestPyPI 22from pip._internal.models.link import Link 23from pip._internal.models.wheel import Wheel 24from pip._internal.pyproject import make_pyproject_path 25from pip._internal.req.req_install import InstallRequirement 26from pip._internal.utils.deprecation import deprecated 27from pip._internal.utils.filetypes import is_archive_file 28from pip._internal.utils.misc import is_installable_dir 29from pip._internal.utils.typing import MYPY_CHECK_RUNNING 30from pip._internal.utils.urls import path_to_url 31from pip._internal.vcs import is_url, vcs 32 33if MYPY_CHECK_RUNNING: 34 from typing import Any, Dict, Optional, Set, Tuple, Union 35 36 from pip._internal.req.req_file import ParsedRequirement 37 38 39__all__ = [ 40 "install_req_from_editable", "install_req_from_line", 41 "parse_editable" 42] 43 44logger = logging.getLogger(__name__) 45operators = Specifier._operators.keys() 46 47 48def _strip_extras(path): 49 # type: (str) -> Tuple[str, Optional[str]] 50 m = re.match(r'^(.+)(\[[^\]]+\])$', path) 51 extras = None 52 if m: 53 path_no_extras = m.group(1) 54 extras = m.group(2) 55 else: 56 path_no_extras = path 57 58 return path_no_extras, extras 59 60 61def convert_extras(extras): 62 # type: (Optional[str]) -> Set[str] 63 if not extras: 64 return set() 65 return Requirement("placeholder" + extras.lower()).extras 66 67 68def parse_editable(editable_req): 69 # type: (str) -> Tuple[Optional[str], str, Set[str]] 70 """Parses an editable requirement into: 71 - a requirement name 72 - an URL 73 - extras 74 - editable options 75 Accepted requirements: 76 svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir 77 .[some_extra] 78 """ 79 80 url = editable_req 81 82 # If a file path is specified with extras, strip off the extras. 83 url_no_extras, extras = _strip_extras(url) 84 85 if os.path.isdir(url_no_extras): 86 if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): 87 msg = ( 88 'File "setup.py" not found. Directory cannot be installed ' 89 'in editable mode: {}'.format(os.path.abspath(url_no_extras)) 90 ) 91 pyproject_path = make_pyproject_path(url_no_extras) 92 if os.path.isfile(pyproject_path): 93 msg += ( 94 '\n(A "pyproject.toml" file was found, but editable ' 95 'mode currently requires a setup.py based build.)' 96 ) 97 raise InstallationError(msg) 98 99 # Treating it as code that has already been checked out 100 url_no_extras = path_to_url(url_no_extras) 101 102 if url_no_extras.lower().startswith('file:'): 103 package_name = Link(url_no_extras).egg_fragment 104 if extras: 105 return ( 106 package_name, 107 url_no_extras, 108 Requirement("placeholder" + extras.lower()).extras, 109 ) 110 else: 111 return package_name, url_no_extras, set() 112 113 for version_control in vcs: 114 if url.lower().startswith('{}:'.format(version_control)): 115 url = '{}+{}'.format(version_control, url) 116 break 117 118 if '+' not in url: 119 raise InstallationError( 120 '{} is not a valid editable requirement. ' 121 'It should either be a path to a local project or a VCS URL ' 122 '(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req) 123 ) 124 125 vc_type = url.split('+', 1)[0].lower() 126 127 if not vcs.get_backend(vc_type): 128 backends = ", ".join([bends.name + '+URL' for bends in vcs.backends]) 129 error_message = "For --editable={}, " \ 130 "only {} are currently supported".format( 131 editable_req, backends) 132 raise InstallationError(error_message) 133 134 package_name = Link(url).egg_fragment 135 if not package_name: 136 raise InstallationError( 137 "Could not detect requirement name for '{}', please specify one " 138 "with #egg=your_package_name".format(editable_req) 139 ) 140 return package_name, url, set() 141 142 143def deduce_helpful_msg(req): 144 # type: (str) -> str 145 """Returns helpful msg in case requirements file does not exist, 146 or cannot be parsed. 147 148 :params req: Requirements file path 149 """ 150 msg = "" 151 if os.path.exists(req): 152 msg = " The path does exist. " 153 # Try to parse and check if it is a requirements file. 154 try: 155 with open(req, 'r') as fp: 156 # parse first line only 157 next(parse_requirements(fp.read())) 158 msg += ( 159 "The argument you provided " 160 "({}) appears to be a" 161 " requirements file. If that is the" 162 " case, use the '-r' flag to install" 163 " the packages specified within it." 164 ).format(req) 165 except RequirementParseError: 166 logger.debug( 167 "Cannot parse '%s' as requirements file", req, exc_info=True 168 ) 169 else: 170 msg += " File '{}' does not exist.".format(req) 171 return msg 172 173 174class RequirementParts(object): 175 def __init__( 176 self, 177 requirement, # type: Optional[Requirement] 178 link, # type: Optional[Link] 179 markers, # type: Optional[Marker] 180 extras, # type: Set[str] 181 ): 182 self.requirement = requirement 183 self.link = link 184 self.markers = markers 185 self.extras = extras 186 187 188def parse_req_from_editable(editable_req): 189 # type: (str) -> RequirementParts 190 name, url, extras_override = parse_editable(editable_req) 191 192 if name is not None: 193 try: 194 req = Requirement(name) 195 except InvalidRequirement: 196 raise InstallationError("Invalid requirement: '{}'".format(name)) 197 else: 198 req = None 199 200 link = Link(url) 201 202 return RequirementParts(req, link, None, extras_override) 203 204 205# ---- The actual constructors follow ---- 206 207 208def install_req_from_editable( 209 editable_req, # type: str 210 comes_from=None, # type: Optional[Union[InstallRequirement, str]] 211 use_pep517=None, # type: Optional[bool] 212 isolated=False, # type: bool 213 options=None, # type: Optional[Dict[str, Any]] 214 constraint=False, # type: bool 215 user_supplied=False, # type: bool 216): 217 # type: (...) -> InstallRequirement 218 219 parts = parse_req_from_editable(editable_req) 220 221 return InstallRequirement( 222 parts.requirement, 223 comes_from=comes_from, 224 user_supplied=user_supplied, 225 editable=True, 226 link=parts.link, 227 constraint=constraint, 228 use_pep517=use_pep517, 229 isolated=isolated, 230 install_options=options.get("install_options", []) if options else [], 231 global_options=options.get("global_options", []) if options else [], 232 hash_options=options.get("hashes", {}) if options else {}, 233 extras=parts.extras, 234 ) 235 236 237def _looks_like_path(name): 238 # type: (str) -> bool 239 """Checks whether the string "looks like" a path on the filesystem. 240 241 This does not check whether the target actually exists, only judge from the 242 appearance. 243 244 Returns true if any of the following conditions is true: 245 * a path separator is found (either os.path.sep or os.path.altsep); 246 * a dot is found (which represents the current directory). 247 """ 248 if os.path.sep in name: 249 return True 250 if os.path.altsep is not None and os.path.altsep in name: 251 return True 252 if name.startswith("."): 253 return True 254 return False 255 256 257def _get_url_from_path(path, name): 258 # type: (str, str) -> Optional[str] 259 """ 260 First, it checks whether a provided path is an installable directory 261 (e.g. it has a setup.py). If it is, returns the path. 262 263 If false, check if the path is an archive file (such as a .whl). 264 The function checks if the path is a file. If false, if the path has 265 an @, it will treat it as a PEP 440 URL requirement and return the path. 266 """ 267 if _looks_like_path(name) and os.path.isdir(path): 268 if is_installable_dir(path): 269 return path_to_url(path) 270 raise InstallationError( 271 "Directory {name!r} is not installable. Neither 'setup.py' " 272 "nor 'pyproject.toml' found.".format(**locals()) 273 ) 274 if not is_archive_file(path): 275 return None 276 if os.path.isfile(path): 277 return path_to_url(path) 278 urlreq_parts = name.split('@', 1) 279 if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]): 280 # If the path contains '@' and the part before it does not look 281 # like a path, try to treat it as a PEP 440 URL req instead. 282 return None 283 logger.warning( 284 'Requirement %r looks like a filename, but the ' 285 'file does not exist', 286 name 287 ) 288 return path_to_url(path) 289 290 291def parse_req_from_line(name, line_source): 292 # type: (str, Optional[str]) -> RequirementParts 293 if is_url(name): 294 marker_sep = '; ' 295 else: 296 marker_sep = ';' 297 if marker_sep in name: 298 name, markers_as_string = name.split(marker_sep, 1) 299 markers_as_string = markers_as_string.strip() 300 if not markers_as_string: 301 markers = None 302 else: 303 markers = Marker(markers_as_string) 304 else: 305 markers = None 306 name = name.strip() 307 req_as_string = None 308 path = os.path.normpath(os.path.abspath(name)) 309 link = None 310 extras_as_string = None 311 312 if is_url(name): 313 link = Link(name) 314 else: 315 p, extras_as_string = _strip_extras(path) 316 url = _get_url_from_path(p, name) 317 if url is not None: 318 link = Link(url) 319 320 # it's a local file, dir, or url 321 if link: 322 # Handle relative file URLs 323 if link.scheme == 'file' and re.search(r'\.\./', link.url): 324 link = Link( 325 path_to_url(os.path.normpath(os.path.abspath(link.path)))) 326 # wheel file 327 if link.is_wheel: 328 wheel = Wheel(link.filename) # can raise InvalidWheelFilename 329 req_as_string = "{wheel.name}=={wheel.version}".format(**locals()) 330 else: 331 # set the req to the egg fragment. when it's not there, this 332 # will become an 'unnamed' requirement 333 req_as_string = link.egg_fragment 334 335 # a requirement specifier 336 else: 337 req_as_string = name 338 339 extras = convert_extras(extras_as_string) 340 341 def with_source(text): 342 # type: (str) -> str 343 if not line_source: 344 return text 345 return '{} (from {})'.format(text, line_source) 346 347 if req_as_string is not None: 348 try: 349 req = Requirement(req_as_string) 350 except InvalidRequirement: 351 if os.path.sep in req_as_string: 352 add_msg = "It looks like a path." 353 add_msg += deduce_helpful_msg(req_as_string) 354 elif ('=' in req_as_string and 355 not any(op in req_as_string for op in operators)): 356 add_msg = "= is not a valid operator. Did you mean == ?" 357 else: 358 add_msg = '' 359 msg = with_source( 360 'Invalid requirement: {!r}'.format(req_as_string) 361 ) 362 if add_msg: 363 msg += '\nHint: {}'.format(add_msg) 364 raise InstallationError(msg) 365 else: 366 # Deprecate extras after specifiers: "name>=1.0[extras]" 367 # This currently works by accident because _strip_extras() parses 368 # any extras in the end of the string and those are saved in 369 # RequirementParts 370 for spec in req.specifier: 371 spec_str = str(spec) 372 if spec_str.endswith(']'): 373 msg = "Extras after version '{}'.".format(spec_str) 374 replace = "moving the extras before version specifiers" 375 deprecated(msg, replacement=replace, gone_in="21.0") 376 else: 377 req = None 378 379 return RequirementParts(req, link, markers, extras) 380 381 382def install_req_from_line( 383 name, # type: str 384 comes_from=None, # type: Optional[Union[str, InstallRequirement]] 385 use_pep517=None, # type: Optional[bool] 386 isolated=False, # type: bool 387 options=None, # type: Optional[Dict[str, Any]] 388 constraint=False, # type: bool 389 line_source=None, # type: Optional[str] 390 user_supplied=False, # type: bool 391): 392 # type: (...) -> InstallRequirement 393 """Creates an InstallRequirement from a name, which might be a 394 requirement, directory containing 'setup.py', filename, or URL. 395 396 :param line_source: An optional string describing where the line is from, 397 for logging purposes in case of an error. 398 """ 399 parts = parse_req_from_line(name, line_source) 400 401 return InstallRequirement( 402 parts.requirement, comes_from, link=parts.link, markers=parts.markers, 403 use_pep517=use_pep517, isolated=isolated, 404 install_options=options.get("install_options", []) if options else [], 405 global_options=options.get("global_options", []) if options else [], 406 hash_options=options.get("hashes", {}) if options else {}, 407 constraint=constraint, 408 extras=parts.extras, 409 user_supplied=user_supplied, 410 ) 411 412 413def install_req_from_req_string( 414 req_string, # type: str 415 comes_from=None, # type: Optional[InstallRequirement] 416 isolated=False, # type: bool 417 use_pep517=None, # type: Optional[bool] 418 user_supplied=False, # type: bool 419): 420 # type: (...) -> InstallRequirement 421 try: 422 req = Requirement(req_string) 423 except InvalidRequirement: 424 raise InstallationError("Invalid requirement: '{}'".format(req_string)) 425 426 domains_not_allowed = [ 427 PyPI.file_storage_domain, 428 TestPyPI.file_storage_domain, 429 ] 430 if (req.url and comes_from and comes_from.link and 431 comes_from.link.netloc in domains_not_allowed): 432 # Explicitly disallow pypi packages that depend on external urls 433 raise InstallationError( 434 "Packages installed from PyPI cannot depend on packages " 435 "which are not also hosted on PyPI.\n" 436 "{} depends on {} ".format(comes_from.name, req) 437 ) 438 439 return InstallRequirement( 440 req, 441 comes_from, 442 isolated=isolated, 443 use_pep517=use_pep517, 444 user_supplied=user_supplied, 445 ) 446 447 448def install_req_from_parsed_requirement( 449 parsed_req, # type: ParsedRequirement 450 isolated=False, # type: bool 451 use_pep517=None, # type: Optional[bool] 452 user_supplied=False, # type: bool 453): 454 # type: (...) -> InstallRequirement 455 if parsed_req.is_editable: 456 req = install_req_from_editable( 457 parsed_req.requirement, 458 comes_from=parsed_req.comes_from, 459 use_pep517=use_pep517, 460 constraint=parsed_req.constraint, 461 isolated=isolated, 462 user_supplied=user_supplied, 463 ) 464 465 else: 466 req = install_req_from_line( 467 parsed_req.requirement, 468 comes_from=parsed_req.comes_from, 469 use_pep517=use_pep517, 470 isolated=isolated, 471 options=parsed_req.options, 472 constraint=parsed_req.constraint, 473 line_source=parsed_req.line_source, 474 user_supplied=user_supplied, 475 ) 476 return req 477